@vellumai/assistant 0.10.8 → 0.10.9-dev.202607152225.ec6fdbf

package/src/persistence/migrations/335-collapse-memory-embed-backlog.ts

@@ -0,0 +1,177 @@
+import { randomUUID } from "node:crypto";
+
+import { getConfig } from "../../config/loader.js";
+import { getLogger } from "../../util/logger.js";
+import { getMemoryDbPath } from "../../util/memory-db-path.js";
+import { parseChangesFromStdout, runAsyncSqlite } from "../db-async-query.js";
+import { type DrizzleDb, getMemorySqlite } from "../db-connection.js";
+
+const log = getLogger("memory-db");
+
+/**
+ * Batch size for the pending `embed_concept_page` purge. Matches the
+ * relocation drain batch: large enough for throughput, small enough that the
+ * write lock is held only briefly per statement.
+ */
+const PURGE_BATCH = 10_000;
+
+/**
+ * `true` unless `memory.enabled` or `memory.v2.enabled` is explicitly
+ * `false`. Missing/unloadable config falls through to the schema defaults
+ * (enabled) — mirrors `isMemoryEnabled` in the jobs store and the gating of
+ * workspace migration 085.
+ */
+function memoryV2ReembedAllowed(): boolean {
+  try {
+    const config = getConfig();
+    if (config.memory?.enabled === false) {
+      return false;
+    }
+    if (config.memory?.v2?.enabled === false) {
+      return false;
+    }
+    return true;
+  } catch {
+    return true;
+  }
+}
+
+/**
+ * Collapse duplicate pending memory-v2 maintenance jobs left behind by
+ * unguarded enqueues.
+ *
+ * `memory_v2_reembed` (payload `{}`, lists every concept page at execution
+ * time) and its per-page `embed_concept_page` fan-out (payload `{slug}`, page
+ * read from disk at execution time) are enqueued with pending-row coalescing,
+ * so at most one pending row exists per reembed / per slug. A queue populated
+ * before that coalescing can hold millions of duplicates — each pending
+ * reembed fans out one embed per page as it drains, so the backlog grows even
+ * after the enqueue storm stops. All duplicates describe identical work: one
+ * full reembed pass regenerates exactly the deduplicated queue.
+ *
+ * Sequence (each step idempotent; a crash resumes safely on the next boot):
+ *   1. When the memory-v2 gates allow it and pending `embed_concept_page`
+ *      rows exist with no pending/running `memory_v2_reembed`, insert one
+ *      replacement reembed. The insert happens BEFORE any delete so a crash
+ *      mid-purge can never leave deleted embeds with no reembed to
+ *      regenerate them. When a gate is explicitly off, pending reembeds are
+ *      deleted instead (runs against the user's intent — workspace migration
+ *      085 parity) and nothing is inserted.
+ *   2. Collapse pending `memory_v2_reembed` and pending `memory_v3_maintain`
+ *      to their earliest row each. `memory_v3_maintain` is never inserted —
+ *      its handler no-ops while v3 is off and the scheduler self-heals.
+ *   3. Delete ALL pending `embed_concept_page` rows in bounded batches off
+ *      the event loop — the surviving/inserted reembed re-fans one embed per
+ *      existing page. Accepted edge: a pending embed for a since-deleted page
+ *      carried a Qdrant point deletion that a page-listing reembed will not
+ *      regenerate; the startup embedding reconcile covers the stale point.
+ *   4. Truncate the memory DB's WAL (best-effort).
+ *
+ * Only `pending` rows of exactly these three types are touched. Running rows
+ * survive (their claimer owns them), as do all other job types — notably
+ * `embed_segment`, whose pending rows carry distinct meaningful payloads. A
+ * worker left over from before the daemon restarted can re-enqueue a fan-out
+ * mid-purge; that residue is bounded by the page count and the coalescing
+ * enqueues absorb it.
+ *
+ * Throws (rather than returning) if the memory database cannot be opened, so
+ * the runner records the step as failed and retries it on a later boot. The
+ * throw is caught per-step by the runner, so startup is not aborted.
+ */
+export async function migrateCollapseMemoryEmbedBacklog(
+  _database: DrizzleDb,
+): Promise<void> {
+  const memoryRaw = getMemorySqlite();
+  if (!memoryRaw) {
+    throw new Error(
+      "memory database unavailable — deferring embed-backlog collapse",
+    );
+  }
+
+  const hasPendingEmbeds =
+    memoryRaw
+      .query(
+        `SELECT 1 FROM memory_jobs WHERE type='embed_concept_page' AND status='pending' LIMIT 1`,
+      )
+      .get() != null;
+
+  if (memoryV2ReembedAllowed()) {
+    if (hasPendingEmbeds) {
+      const existingReembed = memoryRaw
+        .query(
+          `SELECT id FROM memory_jobs WHERE type='memory_v2_reembed' AND status IN ('pending','running') LIMIT 1`,
+        )
+        .get();
+      if (!existingReembed) {
+        const now = Date.now();
+        memoryRaw
+          .query(
+            `INSERT INTO memory_jobs
+               (id, type, payload, status, attempts, deferrals, run_after, last_error, created_at, updated_at)
+             VALUES (?, 'memory_v2_reembed', '{}', 'pending', 0, 0, ?, NULL, ?, ?)`,
+          )
+          .run(randomUUID(), now, now, now);
+      }
+    }
+    memoryRaw
+      .query(
+        `DELETE FROM memory_jobs
+         WHERE type='memory_v2_reembed' AND status='pending'
+           AND rowid != (SELECT MIN(rowid) FROM memory_jobs WHERE type='memory_v2_reembed' AND status='pending')`,
+      )
+      .run();
+  } else {
+    memoryRaw
+      .query(
+        `DELETE FROM memory_jobs WHERE type='memory_v2_reembed' AND status='pending'`,
+      )
+      .run();
+  }
+
+  memoryRaw
+    .query(
+      `DELETE FROM memory_jobs
+       WHERE type='memory_v3_maintain' AND status='pending'
+         AND rowid != (SELECT MIN(rowid) FROM memory_jobs WHERE type='memory_v3_maintain' AND status='pending')`,
+    )
+    .run();
+
+  if (hasPendingEmbeds) {
+    const dbPath = getMemoryDbPath();
+    let totalPurged = 0;
+    for (;;) {
+      const res = await runAsyncSqlite(
+        `DELETE FROM memory_jobs WHERE rowid IN (` +
+          `SELECT rowid FROM memory_jobs WHERE type='embed_concept_page' AND status='pending' LIMIT ${PURGE_BATCH});\n` +
+          `SELECT changes();`,
+        "embed-backlog-collapse:purge-batch",
+        { dbPath },
+      );
+      if (!res.ok) {
+        throw new Error(`embed-backlog purge batch failed: ${res.error}`);
+      }
+      const purged = parseChangesFromStdout(res.stdout);
+      if (purged === 0) {
+        break;
+      }
+      totalPurged += purged;
+      log.info(
+        { purged, totalPurged },
+        "embed-backlog collapse: purge progressed",
+      );
+    }
+
+    const finalizeRes = await runAsyncSqlite(
+      `PRAGMA wal_checkpoint(TRUNCATE);`,
+      "embed-backlog-collapse:wal-truncate",
+      { dbPath },
+    );
+    if (!finalizeRes.ok) {
+      log.warn(
+        { error: finalizeRes.error },
+        "embed-backlog collapse: WAL truncate failed (best-effort)",
+      );
+    }
+    log.info({ totalPurged }, "embed-backlog collapse: complete");
+  }
+}

package/src/persistence/migrations/__tests__/327-create-flush-checkpoints.test.ts

@@ -0,0 +1,124 @@
+/**
+ * Tests for migration 327 — the `flush_checkpoints` table on the dedicated
+ * telemetry database, and the one-time move of the telemetry reporter's
+ * watermark cursors out of the main DB's `memory_checkpoints` ledger.
+ *
+ * The step is idempotent, so each test can drive it directly.
+ */
+import { beforeEach, describe, expect, test } from "bun:test";
+
+const { getDb, getSqlite, getTelemetrySqlite } =
+  await import("../../db-connection.js");
+const { initializeDb } = await import("../../db-init.js");
+const { createFlushCheckpointsTable } =
+  await import("../327-create-flush-checkpoints.js");
+
+await initializeDb();
+
+function telemetrySqlite() {
+  const db = getTelemetrySqlite();
+  if (!db) {
+    throw new Error("telemetry DB unavailable in test");
+  }
+  return db;
+}
+
+function readTelemetryValue(key: string): string | null {
+  const row = telemetrySqlite()
+    .query(`SELECT value FROM flush_checkpoints WHERE key = ?`)
+    .get(key) as { value: string } | null;
+  return row?.value ?? null;
+}
+
+function readMainValue(key: string): string | null {
+  const row = getSqlite()
+    .query(`SELECT value FROM memory_checkpoints WHERE key = ?`)
+    .get(key) as { value: string } | null;
+  return row?.value ?? null;
+}
+
+function seedMainCheckpoint(key: string, value: string): void {
+  getSqlite()
+    .query(
+      `INSERT OR REPLACE INTO memory_checkpoints (key, value, updated_at) VALUES (?, ?, ?)`,
+    )
+    .run(key, value, Date.now());
+}
+
+describe("migration 327 — flush_checkpoints on the telemetry database", () => {
+  beforeEach(() => {
+    telemetrySqlite().query(`DELETE FROM flush_checkpoints`).run();
+    getSqlite()
+      .query(`DELETE FROM memory_checkpoints WHERE key LIKE 'telemetry:%'`)
+      .run();
+  });
+
+  test("after init, the table exists on the telemetry DB", () => {
+    const row = telemetrySqlite()
+      .query(
+        `SELECT name FROM sqlite_master WHERE type='table' AND name='flush_checkpoints'`,
+      )
+      .get();
+    expect(row).not.toBeNull();
+  });
+
+  test("moves legacy watermark keys from memory_checkpoints", () => {
+    seedMainCheckpoint("telemetry:usage:last_reported_at", "1700000000000");
+    seedMainCheckpoint("telemetry:usage:last_reported_id", "row-abc");
+
+    createFlushCheckpointsTable(getDb());
+
+    expect(readTelemetryValue("telemetry:usage:last_reported_at")).toBe(
+      "1700000000000",
+    );
+    expect(readTelemetryValue("telemetry:usage:last_reported_id")).toBe(
+      "row-abc",
+    );
+    expect(readMainValue("telemetry:usage:last_reported_at")).toBeNull();
+    expect(readMainValue("telemetry:usage:last_reported_id")).toBeNull();
+  });
+
+  test("leaves non-watermark telemetry-prefixed checkpoints in place", () => {
+    seedMainCheckpoint("telemetry:installation_id", "inst-123");
+    seedMainCheckpoint(
+      "telemetry:watchers:inventory_last_recorded",
+      "1700000000000",
+    );
+
+    createFlushCheckpointsTable(getDb());
+
+    expect(readMainValue("telemetry:installation_id")).toBe("inst-123");
+    expect(readMainValue("telemetry:watchers:inventory_last_recorded")).toBe(
+      "1700000000000",
+    );
+    expect(readTelemetryValue("telemetry:installation_id")).toBeNull();
+  });
+
+  test("re-run never overwrites a value the reporter has since advanced", () => {
+    seedMainCheckpoint("telemetry:turns:last_reported_at", "1000");
+    createFlushCheckpointsTable(getDb());
+    expect(readTelemetryValue("telemetry:turns:last_reported_at")).toBe("1000");
+
+    // The reporter advances the moved cursor, then a crash re-runs the
+    // migration with a stale main-DB copy still present (crash between copy
+    // and delete on the first run). INSERT OR IGNORE must keep the advanced
+    // value.
+    telemetrySqlite()
+      .query(`UPDATE flush_checkpoints SET value = '2000' WHERE key = ?`)
+      .run("telemetry:turns:last_reported_at");
+    seedMainCheckpoint("telemetry:turns:last_reported_at", "1000");
+
+    createFlushCheckpointsTable(getDb());
+
+    expect(readTelemetryValue("telemetry:turns:last_reported_at")).toBe("2000");
+    expect(readMainValue("telemetry:turns:last_reported_at")).toBeNull();
+  });
+
+  test("no-op when there are no legacy keys", () => {
+    createFlushCheckpointsTable(getDb());
+    const count = telemetrySqlite()
+      .query(`SELECT COUNT(*) as c FROM flush_checkpoints`)
+      .get() as { c: number };
+    expect(count.c).toBe(0);
+  });
+});

package/src/persistence/raw-query.ts

@@ -29,7 +29,12 @@
 
 import type { Database, SQLQueryBindings } from "bun:sqlite";
 
-import { getLogsSqlite, getMemorySqlite, getSqlite } from "./db-connection.js";
+import {
+  getLogsSqlite,
+  getMemorySqlite,
+  getSqlite,
+  getTelemetrySqlite,
+} from "./db-connection.js";
 
 type SqlParam = SQLQueryBindings;
 
@@ -120,6 +125,13 @@ function logsSqlite(): Database {
   return sqlite;
 }
 
+/** The telemetry connection, or a thrown error when it cannot be opened. */
+function telemetrySqlite(): Database {
+  const sqlite = getTelemetrySqlite();
+  if (!sqlite) throw new Error("telemetry database unavailable");
+  return sqlite;
+}
+
 /** {@link rawAll} against the memory connection. */
 export function rawMemoryAll<T>(
   label: string,
@@ -166,6 +178,20 @@ export function rawLogsRun(
   return (sqlite.query("SELECT changes() AS c").get() as { c: number }).c;
 }
 
+/** {@link rawRun} against the telemetry connection. */
+export function rawTelemetryRun(
+  label: string,
+  sql: string,
+  ...params: SqlParam[]
+): number {
+  const sqlite = telemetrySqlite();
+  sqlite
+    .label(label)
+    .query(sql)
+    .run(...params);
+  return (sqlite.query("SELECT changes() AS c").get() as { c: number }).c;
+}
+
 /**
  * Delete all rows from the given tables in a single transaction.
  *

package/src/persistence/schema/calls.ts

@@ -90,29 +90,6 @@ export const externalConversationBindings = sqliteTable(
   },
 );
 
-export const channelGuardianApprovalRequests = sqliteTable(
-  "channel_guardian_approval_requests",
-  {
-    id: text("id").primaryKey(),
-    runId: text("run_id").notNull(),
-    requestId: text("request_id"),
-    conversationId: text("conversation_id").notNull(),
-    channel: text("channel").notNull(),
-    requesterExternalUserId: text("requester_external_user_id").notNull(),
-    requesterChatId: text("requester_chat_id").notNull(),
-    guardianExternalUserId: text("guardian_external_user_id").notNull(),
-    guardianChatId: text("guardian_chat_id").notNull(),
-    toolName: text("tool_name").notNull(),
-    riskLevel: text("risk_level"),
-    reason: text("reason"),
-    status: text("status").notNull().default("pending"),
-    decidedByExternalUserId: text("decided_by_external_user_id"),
-    expiresAt: integer("expires_at").notNull(),
-    createdAt: integer("created_at").notNull(),
-    updatedAt: integer("updated_at").notNull(),
-  },
-);
-
 export const mediaAssets = sqliteTable("media_assets", {
   id: text("id").primaryKey(),
   title: text("title").notNull(),

package/src/persistence/schema/conversation-starters.ts

@@ -0,0 +1,31 @@
+import { index, integer, sqliteTable, text } from "drizzle-orm/sqlite-core";
+
+// Proactive conversation-starter suggestions (home-surface chips/cards),
+// regenerated in batches by the home job handler
+// (`home/job-handlers/conversation-starters.ts`) and served by
+// `runtime/routes/conversation-starter-routes.ts`. Product state owned by the
+// home surface — the generator draws on memory content, but the table is not
+// part of the memory plugin's storage.
+export const conversationStarters = sqliteTable(
+  "conversation_starters",
+  {
+    id: text("id").primaryKey(),
+    label: text("label").notNull(),
+    prompt: text("prompt").notNull(),
+    generationBatch: integer("generation_batch").notNull(),
+    sourceMemoryKinds: text("source_memory_kinds"),
+    category: text("category"),
+    icon: text("icon"),
+    description: text("description"),
+    tags: text("tags"),
+    cardType: text("card_type").notNull().default("chip"),
+    createdAt: integer("created_at").notNull(),
+  },
+  (table) => [
+    index("idx_conversation_starters_batch").on(
+      table.generationBatch,
+      table.createdAt,
+    ),
+    index("idx_conversation_starters_card_type").on(table.cardType),
+  ],
+);

package/src/persistence/schema/conversations.ts

@@ -107,9 +107,9 @@ export const messages = sqliteTable(
     /**
      * Union of two JSON shapes: an inline `ContentBlock[]` (or a legacy
      * plain string), or `{ ref: "<workspace-relative path>" }` pointing at
-     * a file-backed content payload. Always read through the message row
-     * mapper (which applies `resolveStoredMessageContent`) — never parse
-     * this column raw.
+     * a file-backed content payload. Resolve to typed blocks with
+     * `resolveMessageContentBlocks` (message-content-file.ts) — never
+     * interpret this column's shape by hand.
      */
     content: text("content").notNull(),
     createdAt: integer("created_at").notNull(),

package/src/persistence/schema/guardian.ts

@@ -1,155 +1,5 @@
 import { index, integer, sqliteTable, text } from "drizzle-orm/sqlite-core";
 
-import { callPendingQuestions, callSessions } from "./calls.js";
-
-export const guardianActionRequests = sqliteTable(
-  "guardian_action_requests",
-  {
-    id: text("id").primaryKey(),
-    kind: text("kind").notNull(), // 'ask_guardian'
-    sourceChannel: text("source_channel").notNull(), // 'phone'
-    sourceConversationId: text("source_conversation_id").notNull(),
-    callSessionId: text("call_session_id")
-      .notNull()
-      .references(() => callSessions.id, { onDelete: "cascade" }),
-    pendingQuestionId: text("pending_question_id")
-      .notNull()
-      .references(() => callPendingQuestions.id, { onDelete: "cascade" }),
-    questionText: text("question_text").notNull(),
-    requestCode: text("request_code").notNull(), // short human-readable code for routing replies
-    status: text("status").notNull().default("pending"), // pending | answered | expired | cancelled
-    answerText: text("answer_text"),
-    answeredByChannel: text("answered_by_channel"),
-    answeredByExternalUserId: text("answered_by_external_user_id"),
-    answeredAt: integer("answered_at"),
-    expiresAt: integer("expires_at").notNull(),
-    expiredReason: text("expired_reason"), // call_timeout | sweep_timeout | cancelled
-    followupState: text("followup_state").notNull().default("none"), // none | awaiting_guardian_choice | dispatching | completed | declined | failed
-    lateAnswerText: text("late_answer_text"),
-    lateAnsweredAt: integer("late_answered_at"),
-    followupAction: text("followup_action"), // call_back | decline
-    followupCompletedAt: integer("followup_completed_at"),
-    toolName: text("tool_name"), // tool identity for tool-approval requests
-    inputDigest: text("input_digest"), // canonical SHA-256 digest of tool input
-    supersededByRequestId: text("superseded_by_request_id"), // links to the request that replaced this one
-    supersededAt: integer("superseded_at"), // epoch ms when supersession occurred
-    createdAt: integer("created_at").notNull(),
-    updatedAt: integer("updated_at").notNull(),
-  },
-  (table) => [
-    index("idx_guardian_action_requests_session_status_created").on(
-      table.callSessionId,
-      table.status,
-      table.createdAt,
-    ),
-  ],
-);
-
-export const guardianActionDeliveries = sqliteTable(
-  "guardian_action_deliveries",
-  {
-    id: text("id").primaryKey(),
-    requestId: text("request_id")
-      .notNull()
-      .references(() => guardianActionRequests.id, { onDelete: "cascade" }),
-    destinationChannel: text("destination_channel").notNull(), // 'telegram' | 'vellum'
-    destinationConversationId: text("destination_conversation_id"),
-    destinationChatId: text("destination_chat_id"),
-    destinationExternalUserId: text("destination_external_user_id"),
-    status: text("status").notNull().default("pending"), // pending | sent | failed | answered | expired | cancelled
-    sentAt: integer("sent_at"),
-    respondedAt: integer("responded_at"),
-    lastError: text("last_error"),
-    createdAt: integer("created_at").notNull(),
-    updatedAt: integer("updated_at").notNull(),
-  },
-  (table) => [
-    index("idx_guardian_action_deliveries_dest_conversation").on(
-      table.destinationConversationId,
-    ),
-  ],
-);
-
-export const canonicalGuardianRequests = sqliteTable(
-  "canonical_guardian_requests",
-  {
-    id: text("id").primaryKey(),
-    kind: text("kind").notNull(),
-    sourceType: text("source_type").notNull(),
-    sourceChannel: text("source_channel"),
-    conversationId: text("conversation_id"),
-    requesterExternalUserId: text("requester_external_user_id"),
-    requesterChatId: text("requester_chat_id"),
-    guardianExternalUserId: text("guardian_external_user_id"),
-    guardianPrincipalId: text("guardian_principal_id"),
-    callSessionId: text("call_session_id"),
-    pendingQuestionId: text("pending_question_id"),
-    questionText: text("question_text"),
-    requestCode: text("request_code"),
-    toolName: text("tool_name"),
-    inputDigest: text("input_digest"),
-    commandPreview: text("command_preview"),
-    riskLevel: text("risk_level"),
-    activityText: text("activity_text"),
-    executionTarget: text("execution_target"),
-    // JSON-encoded RequesterIdentitySignals ({isBot,isStranger,isRestricted})
-    // captured at creation so decision-time policy reads the same identity
-    // facts the introduction card was rendered from.
-    requesterSignals: text("requester_signals"),
-    // What prompted an access request: 'denied' (refused sender awaiting a
-    // let-them-in decision) or 'admitted' (floor-admitted sender nudged for
-    // trust assignment). NULL means 'denied'. Resolvers and the expiry sweep
-    // suppress requester-facing lifecycle notices for 'admitted'.
-    trigger: text("request_trigger"),
-    status: text("status").notNull().default("pending"),
-    answerText: text("answer_text"),
-    decidedByExternalUserId: text("decided_by_external_user_id"),
-    decidedByPrincipalId: text("decided_by_principal_id"),
-    followupState: text("followup_state"),
-    expiresAt: integer("expires_at"),
-    createdAt: integer("created_at").notNull(),
-    updatedAt: integer("updated_at").notNull(),
-  },
-  (table) => [
-    index("idx_canonical_guardian_requests_status").on(table.status),
-    index("idx_canonical_guardian_requests_guardian").on(
-      table.guardianExternalUserId,
-      table.status,
-    ),
-    index("idx_canonical_guardian_requests_conversation").on(
-      table.conversationId,
-      table.status,
-    ),
-    index("idx_canonical_guardian_requests_source").on(
-      table.sourceType,
-      table.status,
-    ),
-    index("idx_canonical_guardian_requests_kind").on(table.kind, table.status),
-    index("idx_canonical_guardian_requests_request_code").on(table.requestCode),
-  ],
-);
-
-export const canonicalGuardianDeliveries = sqliteTable(
-  "canonical_guardian_deliveries",
-  {
-    id: text("id").primaryKey(),
-    requestId: text("request_id")
-      .notNull()
-      .references(() => canonicalGuardianRequests.id, { onDelete: "cascade" }),
-    destinationChannel: text("destination_channel").notNull(),
-    destinationConversationId: text("destination_conversation_id"),
-    destinationChatId: text("destination_chat_id"),
-    destinationMessageId: text("destination_message_id"),
-    status: text("status").notNull().default("pending"),
-    createdAt: integer("created_at").notNull(),
-    updatedAt: integer("updated_at").notNull(),
-  },
-  (table) => [
-    index("idx_canonical_guardian_deliveries_request_id").on(table.requestId),
-    index("idx_canonical_guardian_deliveries_status").on(table.status),
-  ],
-);
-
 export const scopedApprovalGrants = sqliteTable(
   "scoped_approval_grants",
   {

package/src/persistence/schema/index.ts

@@ -4,6 +4,7 @@ export * from "./bookmarks.js";
 export * from "./calls.js";
 export * from "./contacts.js";
 export * from "./conversation-groups.js";
+export * from "./conversation-starters.js";
 export * from "./conversations.js";
 export * from "./documents.js";
 export * from "./guardian.js";