@vellumai/assistant 0.10.2-dev.202606250318.5e7cfb0 → 0.10.2

package/src/memory/migrations/run-migrations.ts

@@ -4,45 +4,10 @@ import { type DrizzleDb, getSqliteFrom } from "../db-connection.js";
 
 const log = getLogger("db-init");
 
-/**
- * Rollback metadata for a migration step that can be reversed.
- *
- * A single step may carry multiple rollback entries (e.g. migration 162 has
- * two: one for the timestamp conversion and one for the table rebuild). Each
- * entry has its own monotonic version number for rollback ordering and its
- * own `down()` function.
- */
-export interface RollbackEntry {
-  /** Monotonic version used for rollback ordering. */
-  version: number;
-  /** Human-readable description for diagnostics. */
-  description: string;
-  /** Reverse the migration. Must be idempotent — safe to re-run. */
-  down: (database: DrizzleDb) => void;
-}
-
-/**
- * Object form of a migration step, carrying optional rollback metadata and
- * dependency declarations inline — no separate registry needed.
- */
-export interface MigrationStepObject {
-  /** Step name — used for `step:*` checkpoint key, logging, and skip logic. */
-  name: string;
-  /** Forward migration body. */
-  run: (database: DrizzleDb) => void | Promise<void>;
-  /** Step names that must complete before this step runs. */
-  dependsOn?: string[];
-  /** Rollback entries — present if this step has down() function(s). */
-  rollback?: RollbackEntry[];
-}
-
 /**
  * A single forward migration step, identified for checkpointing and logging by
  * its `.name`. Anonymous steps (empty `.name`) cannot be tracked and always run.
  *
- * Bare function steps are identified by `Function.name`. Object-form steps
- * carry an explicit `name` plus optional `rollback` and `dependsOn` metadata.
- *
  * A step may be synchronous or return a promise. The runner awaits an async
  * step to completion before checkpointing it and moving on, so ordering is
  * preserved exactly as for sync steps: step N+1 never starts — and is never
@@ -51,56 +16,7 @@ export interface MigrationStepObject {
  * thread between batches while still guaranteeing later migrations observe its
  * completed result.
  */
-export type MigrationStep =
-  | MigrationStepObject
-  | ((database: DrizzleDb) => void | Promise<void>);
-
-/**
- * Normalize a {@link MigrationStep} (which may be a bare function or an object)
- * into its object form so callers don't need to type-narrow.
- */
-export function normalizeStep(step: MigrationStep): MigrationStepObject {
-  if (typeof step === "function") {
-    return { name: step.name, run: step };
-  }
-  return step;
-}
-
-/**
- * Extract the step name from either form of {@link MigrationStep}.
- */
-export function getStepName(step: MigrationStep): string {
-  return typeof step === "function" ? step.name : step.name;
-}
-
-/**
- * Get the maximum rollback version across all steps that carry rollback
- * metadata. Returns 0 if no steps have rollback entries.
- */
-export function getMaxRollbackVersion(steps: MigrationStep[]): number {
-  let max = 0;
-  for (const step of steps) {
-    if (typeof step === "function") continue;
-    if (!step.rollback) continue;
-    for (const entry of step.rollback) {
-      if (entry.version > max) max = entry.version;
-    }
-  }
-  return max;
-}
-
-/**
- * Collect all step names that appear in the steps array (both bare-function
- * and object-form). Used by {@link validateMigrationState} to detect
- * checkpoints from a newer daemon version.
- */
-export function getKnownStepNames(steps: MigrationStep[]): Set<string> {
-  const names = new Set<string>();
-  for (const step of steps) {
-    names.add(getStepName(step));
-  }
-  return names;
-}
+export type MigrationStep = (database: DrizzleDb) => void | Promise<void>;
 
 export interface MigrationRunResult {
   /** Steps that ran and completed successfully this boot. */
@@ -114,8 +30,9 @@ export interface MigrationRunResult {
 /**
  * Prefix under which forward-step completions are recorded in the shared
  * `memory_checkpoints` ledger. A distinct namespace keeps step bookkeeping from
- * colliding with legacy checkpoint keys (`migration_*`, `backfill_*`,
- * `drop_*`) left by older migration functions.
+ * colliding with registry checkpoint keys (`migration_*`, `backfill_*`,
+ * `drop_*`) and is deliberately chosen so `validateMigrationState` does not
+ * mistake a step record for an unknown registry migration.
  */
 export const STEP_CHECKPOINT_PREFIX = "step:";
 
@@ -244,8 +161,7 @@ export async function runMigrationSteps(
   const ran: string[] = [];
 
   for (const step of steps) {
-    const obj = normalizeStep(step);
-    const name = obj.name;
+    const name = step.name;
     const checkpointable = name !== "";
 
     if (checkpointable && applied.has(name)) {
@@ -259,7 +175,7 @@ export async function runMigrationSteps(
         markStarted.run(`${STEP_CHECKPOINT_PREFIX}${name}`, Date.now());
       }
       log.info({ migration: name }, `Starting migration: ${name}`);
-      const result = obj.run(database);
+      const result = step(database);
       if (result instanceof Promise) {
         await result;
       }

package/src/memory/migrations/validate-migration-state.ts

@@ -3,25 +3,17 @@ import { getLogger } from "../../util/logger.js";
 import { getDbPath } from "../../util/platform.js";
 import { type DrizzleDb, getSqliteFrom } from "../db-connection.js";
 import {
-  getStepName,
-  type MigrationStep,
-  normalizeStep,
+  MIGRATION_REGISTRY,
+  type MigrationValidationResult,
+} from "./registry.js";
+import {
   STEP_CHECKPOINT_PREFIX,
 } from "./run-migrations.js";
 
 const log = getLogger("memory-db");
 
-export interface MigrationValidationResult {
-  /** Keys of migrations whose checkpoint has value 'started' — started but never completed. */
-  crashed: string[];
-  /** Pairs where a completed migration's declared prerequisite is missing from checkpoints. */
-  dependencyViolations: Array<{ migration: string; missingDependency: string }>;
-  /** Checkpoint keys present in the database but absent from the known step names — likely from a newer version. */
-  unknownCheckpoints: string[];
-}
-
 /**
- * Validate the applied migration state against the known step list at startup.
+ * Validate the applied migration state against the registry at startup.
  *
  * Logs a prominent error when a migration started but never completed (crash
  * detected) — startup continues so the migration can be retried.
@@ -33,15 +25,9 @@ export interface MigrationValidationResult {
  *
  * Call this AFTER all DDL and migration functions have run so that the final
  * state is inspected.
- *
- * @param database  The Drizzle database instance.
- * @param steps     The full ordered migration step array (same one passed to
- *   `runMigrationSteps`). Used to determine known step names for unknown-checkpoint
- *   detection and to check `dependsOn` declarations.
  */
 export function validateMigrationState(
   database: DrizzleDb,
-  steps: MigrationStep[],
 ): MigrationValidationResult {
   const raw = getSqliteFrom(database);
 
@@ -78,6 +64,8 @@ export function validateMigrationState(
   }
 
   // Build a set of completed step names from `step:*` checkpoints with value '1'.
+  // The step runner writes these — `migration_*` registry keys are no longer
+  // written by migration functions (Phase 2 removed withCrashRecovery).
   const completedStepNames = new Set(
     rows
       .filter(
@@ -87,24 +75,32 @@ export function validateMigrationState(
       .map((r) => r.key.slice(STEP_CHECKPOINT_PREFIX.length)),
   );
 
-  // Check dependency ordering for object-form steps that declare dependsOn.
-  // A step is "completed" if its name appears in the completedStepNames set.
+  // Map registry entries to their step names to determine which migrations
+  // are completed. Multiple registry entries can share the same step name
+  // (e.g., 162's two entries both map to migrateGuardianTimestampsEpochMs).
+  const completed = new Set<string>();
+  for (const entry of MIGRATION_REGISTRY) {
+    if (completedStepNames.has(entry.stepName)) {
+      completed.add(entry.key);
+    }
+  }
+
   const dependencyViolations: Array<{
     migration: string;
     missingDependency: string;
   }> = [];
 
-  for (const step of steps) {
-    const obj = normalizeStep(step);
-    if (!obj.dependsOn || obj.dependsOn.length === 0) continue;
-    // Only check steps that have been completed — unapplied or in-progress
+  // Validate dependency ordering.
+  for (const entry of MIGRATION_REGISTRY) {
+    if (!entry.dependsOn || entry.dependsOn.length === 0) continue;
+    // Only check entries that have been completed — unapplied or in-progress
     // migrations have not had a chance to violate their prerequisites yet.
-    if (!completedStepNames.has(obj.name)) continue;
+    if (!completed.has(entry.key)) continue;
 
-    for (const dep of obj.dependsOn) {
-      if (!completedStepNames.has(dep)) {
+    for (const dep of entry.dependsOn) {
+      if (!completed.has(dep)) {
         dependencyViolations.push({
-          migration: obj.name,
+          migration: entry.key,
           missingDependency: dep,
         });
       }
@@ -125,11 +121,10 @@ export function validateMigrationState(
   }
 
   // Detect step checkpoints that exist in the database but have no
-  // corresponding entry in the known step list — these are from a newer
-  // version of the daemon.
-  const knownStepNames = new Set(steps.map((s) => getStepName(s)));
+  // corresponding registry entry — these are from a newer version of the daemon.
+  const registryStepNames = new Set(MIGRATION_REGISTRY.map((e) => e.stepName));
   const unknownCheckpoints = [...completedStepNames].filter(
-    (name) => !knownStepNames.has(name),
+    (name) => !registryStepNames.has(name),
   );
 
   if (unknownCheckpoints.length > 0) {
@@ -145,26 +140,23 @@ export function validateMigrationState(
 /**
  * Roll back all completed memory (database) migrations with version > targetVersion.
  *
- * Iterates eligible migration steps in reverse version order. For each
- * rollback entry on a completed step:
- * 1. Marks the step checkpoint as `"rolling_back"` for crash recovery.
+ * Iterates eligible migrations in reverse version order. For each:
+ * 1. Marks the checkpoint as `"rolling_back"` for crash recovery.
  * 2. Calls `entry.down(database)` — each down() manages its own transactions.
- * 3. Deletes the step checkpoint from `memory_checkpoints`.
- *
- * A single step may carry multiple rollback entries (e.g. migration 162 has
- * two). All entries with version > target are rolled back in reverse version
- * order. The step checkpoint is only deleted after the last rollback entry
- * for that step completes.
+ *    (`down` is required on `MigrationRegistryEntry` at the type level.)
+ * 3. Deletes the checkpoint from `memory_checkpoints`.
  *
  * **Usage**: Pass the target version number you want to roll back *to*. All
- * rollback entries with a higher version number on completed steps will be
- * reversed. For example, `rollbackMemoryMigration(db, 5, steps)` rolls back
- * all rollback entries with version > 5.
+ * migrations with a higher version number that have been applied will be
+ * reversed. For example, `rollbackMemoryMigration(db, 5)` rolls back all
+ * applied migrations with version > 5.
  *
- * **Checkpoint state**: Each rolled-back step's `step:*` checkpoint is deleted
- * from `memory_checkpoints` after its down() functions succeed. If the process
- * crashes mid-rollback, the `"rolling_back"` marker is detected and cleared by
- * `recoverCrashedMigrations` on the next startup.
+ * **Checkpoint state**: Each rolled-back migration's checkpoint is deleted
+ * from `memory_checkpoints`. If the process crashes mid-rollback, the
+ * `"rolling_back"` marker is detected and cleared by
+ * `recoverCrashedMigrations` on the next startup. The forward-step checkpoints
+ * recorded by the migration runner (the `step:` namespace) for the rolled-back
+ * entries are also discarded so a later upgrade re-applies them.
  *
  * **Warning — data loss**: Some down() migrations may not fully restore the
  * original state (e.g., DROP TABLE migrations recreate the table but cannot
@@ -176,15 +168,13 @@ export function validateMigrationState(
  * query failures, or data corruption.
  *
  * @param database  The Drizzle database instance.
- * @param targetVersion  Roll back to this version (exclusive — all rollback
- *   entries with version > targetVersion are reversed).
- * @param steps     The full ordered migration step array.
- * @returns The list of rolled-back step names.
+ * @param targetVersion  Roll back to this version (exclusive — all migrations
+ *   with version > targetVersion are reversed).
+ * @returns The list of rolled-back migration keys.
  */
 export function rollbackMemoryMigration(
   database: DrizzleDb,
   targetVersion: number,
-  steps: MigrationStep[],
 ): string[] {
   const raw = getSqliteFrom(database);
 
@@ -208,66 +198,41 @@ export function rollbackMemoryMigration(
       .map((r) => r.key.slice(STEP_CHECKPOINT_PREFIX.length)),
   );
 
-  // Collect all rollback entries with version > targetVersion on completed steps.
-  // Each entry references its parent step name so we can clear the checkpoint
-  // after all rollback entries for that step have been executed.
-  interface RollbackItem {
-    stepName: string;
-    version: number;
-    description: string;
-    down: (db: DrizzleDb) => void;
-  }
-
-  const toRollback: RollbackItem[] = [];
-  for (const step of steps) {
-    const obj = normalizeStep(step);
-    if (!obj.rollback) continue;
-    if (!completedStepNames.has(obj.name)) continue;
+  // Find registry entries with version > targetVersion whose step is completed.
+  // Deduplicate by stepName — multiple registry entries can share the same step
+  // (e.g., 162's two entries), and we only need to clear the step checkpoint once.
+  const toRollback = MIGRATION_REGISTRY.filter(
+    (entry) =>
+      entry.version > targetVersion &&
+      completedStepNames.has(entry.stepName),
+  ).sort((a, b) => b.version - a.version); // reverse version order
 
-    for (const entry of obj.rollback) {
-      if (entry.version > targetVersion) {
-        toRollback.push({
-          stepName: obj.name,
-          version: entry.version,
-          description: entry.description,
-          down: entry.down,
-        });
-      }
-    }
-  }
-
-  // Sort in reverse version order — children (higher version) before parents.
-  toRollback.sort((a, b) => b.version - a.version);
-
-  // Group by stepName so we only delete the checkpoint after all rollback
-  // entries for that step have been executed.
-  const stepNamesToRollback = new Set(toRollback.map((r) => r.stepName));
   const rolledBack: string[] = [];
 
-  for (const item of toRollback) {
-    const stepKey = `${STEP_CHECKPOINT_PREFIX}${item.stepName}`;
+  for (const entry of toRollback) {
+    const stepKey = `${STEP_CHECKPOINT_PREFIX}${entry.stepName}`;
 
-    // Mark as rolling_back for crash recovery.
+    // Mark as rolling_back for crash recovery — if the process crashes here,
+    // recoverCrashedMigrations will clear this checkpoint on next startup.
     raw
       .query(
         `INSERT OR REPLACE INTO memory_checkpoints (key, value, updated_at) VALUES (?, 'rolling_back', ?)`,
       )
       .run(stepKey, Date.now());
 
-    // Execute the down migration.
-    item.down(database);
+    // Execute the down migration — let it manage its own transaction lifecycle.
+    // Many down() functions call BEGIN/COMMIT internally or use PRAGMA statements
+    // that are no-ops inside a transaction.
+    entry.down(database);
+
+    // Delete the step checkpoint after down() succeeds.
+    raw.query(`DELETE FROM memory_checkpoints WHERE key = ?`).run(stepKey);
 
     log.info(
-      { step: item.stepName, version: item.version, description: item.description },
-      `Rolled back migration "${item.stepName}" (version ${item.version})`,
+      { key: entry.key, version: entry.version },
+      `Rolled back migration "${entry.key}" (version ${entry.version})`,
     );
-  }
-
-  // After all down() calls succeed, delete checkpoints for the rolled-back steps.
-  for (const stepName of stepNamesToRollback) {
-    const stepKey = `${STEP_CHECKPOINT_PREFIX}${stepName}`;
-    raw.query(`DELETE FROM memory_checkpoints WHERE key = ?`).run(stepKey);
-    rolledBack.push(stepName);
+    rolledBack.push(entry.key);
   }
 
   return rolledBack;

package/src/memory/schema/conversations.ts

@@ -52,15 +52,6 @@ export const conversations = sqliteTable(
     inferenceProfileSessionId: text("inference_profile_session_id"),
     inferenceProfileExpiresAt: integer("inference_profile_expires_at"),
     lastNotifiedInferenceProfile: text("last_notified_inference_profile"),
-    /**
-     * Epoch-ms timestamp set when the agent loop starts a turn for this
-     * conversation, cleared (NULL) when the turn ends. NULL means not
-     * processing. This is the cross-process source of truth for processing
-     * state — the in-memory `Conversation._processing` flag is the hot-path
-     * read for resident conversations, but CLI-side and other out-of-process
-     * callers read this column directly.
-     */
-    processingStartedAt: integer("processing_started_at"),
   },
   (table) => [
     index("idx_conversations_updated_at").on(table.updatedAt),

package/src/memory/schema/infrastructure.ts

@@ -337,26 +337,6 @@ export const skillLoadedEvents = sqliteTable(
   ],
 );
 
-// One row per `watchdog` telemetry event, emitted when a daemon watchdog
-// check fires (event-loop block, stream-idle stall, restart, ...) — see
-// watchdog-events-store.ts for the data contract. Flushed by the usage
-// telemetry reporter. `value` is a REAL (BQ FLOAT) so the daemon need not
-// distinguish int vs float; the platform serializer coerces ints to float.
-// `detail` is a JSON bag stored as text and forwarded verbatim.
-export const watchdogEvents = sqliteTable(
-  "watchdog_events",
-  {
-    id: text("id").primaryKey(),
-    createdAt: integer("created_at").notNull(),
-    checkName: text("check_name").notNull(),
-    value: real("value"),
-    detail: text("detail"),
-  },
-  (table) => [
-    index("idx_watchdog_events_created_at_id").on(table.createdAt, table.id),
-  ],
-);
-
 export const traceEvents = sqliteTable(
   "trace_events",
   {

package/src/memory/v2/__tests__/cli-command-store.test.ts

@@ -371,31 +371,6 @@ describe("seedV2CliCommandEntries", () => {
     ).rejects.toThrow("backend exploded");
   });
 
-  test("populates the cache from the Commander tree on the first seed even when the embedding backend is unavailable (cold-start needle resilience)", async () => {
-    // Cold-start race: the startup seed runs before the managed embedding
-    // credential is provisioned, so the first `embedWithBackend` throws. The
-    // in-memory cache (read by the v3 needle lane and the page index) must
-    // still populate from the local Commander tree so CLI commands are
-    // discoverable from first boot; only the dense Qdrant upsert is deferred.
-    state.commands = [
-      { name: "config", description: "Manage configuration", helpText: "..." },
-    ];
-    state.embedThrows = new Error(
-      'Embedding backend "gemini" is not configured',
-    );
-
-    await expect(seedV2CliCommandEntries()).resolves.toBeUndefined();
-
-    const entry = getCliCommandCapability("config");
-    expect(entry).not.toBeNull();
-    expect(entry?.id).toBe("config");
-    expect(listCliCommandEntries().map((e) => e.id)).toEqual(["config"]);
-
-    // No dense vectors were produced, so the Qdrant write is skipped entirely.
-    expect(state.upsertCalls).toHaveLength(0);
-    expect(state.pruneCalls).toHaveLength(0);
-  });
-
   test("skips stale in-flight seed results when a newer refresh is requested", async () => {
     state.commands = [
       { name: "config", description: "Manage configuration", helpText: "..." },

package/src/memory/v2/__tests__/skill-store.test.ts

@@ -619,86 +619,6 @@ Write a local article draft.
     expect(after).toEqual(before);
   });
 
-  test("populates the cache from the local catalog on the first seed even when the embedding backend is unavailable (cold-start needle resilience)", async () => {
-    // Regression: on a brand-new managed assistant the startup seed runs before
-    // the platform provisions the managed embedding credential, so the very
-    // first `embedWithBackend` throws. The in-memory cache (which the v3 needle
-    // lane and the page index read) must still populate from the local catalog
-    // so skills are discoverable from first boot; only the dense Qdrant upsert
-    // is deferred until the backend recovers.
-    const skillA = makeSummary({
-      id: "example-skill-a",
-      displayName: "Skill A",
-    });
-    state.catalog = [skillA];
-    state.resolved = [{ summary: skillA, state: "enabled" }];
-    // No prior successful seed — the backend is unconfigured from the start.
-    state.embedThrows = new Error(
-      'Embedding backend "gemini" is not configured',
-    );
-
-    // Best-effort callers (the startup seed) must resolve.
-    await expect(seedV2SkillEntries()).resolves.toBeUndefined();
-
-    // The needle-lane cache is populated despite the dense-embed failure.
-    const entry = getSkillCapability("example-skill-a");
-    expect(entry).not.toBeNull();
-    expect(entry?.id).toBe("example-skill-a");
-    expect(entry?.content).toContain("Skill A");
-    expect(listSkillEntries().map((e) => e.id)).toEqual(["example-skill-a"]);
-
-    // No dense vectors were produced, so the Qdrant write is skipped entirely.
-    expect(state.upsertCalls).toHaveLength(0);
-    expect(state.pruneCalls).toHaveLength(0);
-  });
-
-  test("surfaces the dense-embed failure to throwOnError callers while still populating the needle cache", async () => {
-    // The managed-credential reseed and the operator reembed route pass
-    // `throwOnError` so they learn the dense lane didn't complete and the
-    // retry/maintain machinery backfills it — but the needle cache is fixed
-    // regardless before the error propagates.
-    const skillA = makeSummary({ id: "example-skill-a" });
-    state.catalog = [skillA];
-    state.resolved = [{ summary: skillA, state: "enabled" }];
-    state.embedThrows = new Error("backend down");
-
-    await expect(seedV2SkillEntries({ throwOnError: true })).rejects.toThrow(
-      "backend down",
-    );
-
-    expect(getSkillCapability("example-skill-a")).not.toBeNull();
-    expect(state.upsertCalls).toHaveLength(0);
-  });
-
-  test("drops a locally-disabled installed skill even when the catalog is unavailable (local state is authoritative)", async () => {
-    // Regression: the local resolution is authoritative, so a skill that is
-    // still installed but explicitly disabled must NOT be kept alive by remote-
-    // catalog uncertainty — `getSkillCapability` and the page index must drop it.
-    const skillA = makeSummary({ id: "example-skill-a" });
-    state.catalog = [skillA];
-    state.resolved = [{ summary: skillA, state: "enabled" }];
-    state.fullCatalog = [
-      { id: "example-skill-a", name: "example-skill-a", description: "A" },
-    ];
-    state.embedReturn = [[0.1, 0.2, 0.3]];
-
-    // First run: skillA enabled and cached.
-    await seedV2SkillEntries();
-    expect(getSkillCapability("example-skill-a")).not.toBeNull();
-
-    // Second run: skillA is still locally installed but now disabled; the remote
-    // catalog is unavailable. It stays in `installedIds`, so the carry-forward
-    // must skip it and the cache drops it.
-    state.resolved = [{ summary: skillA, state: "disabled" }];
-    state.fullCatalog = [];
-    state.fullCatalogThrows = new Error("catalog fetch failed");
-
-    await seedV2SkillEntries();
-
-    expect(getSkillCapability("example-skill-a")).toBeNull();
-    expect(listSkillEntries()).toEqual([]);
-  });
-
   test("no enabled skills yields empty cache and no prune when catalog is empty", async () => {
     state.catalog = [];
     state.resolved = [];