@vellumai/assistant 0.10.1 → 0.10.2-dev.202606241651.2d2b40d

package/src/memory/migrations/validate-migration-state.ts

@@ -3,17 +3,25 @@ import { getLogger } from "../../util/logger.js";
 import { getDbPath } from "../../util/platform.js";
 import { type DrizzleDb, getSqliteFrom } from "../db-connection.js";
 import {
-  MIGRATION_REGISTRY,
-  type MigrationValidationResult,
-} from "./registry.js";
-import {
+  getStepName,
+  type MigrationStep,
+  normalizeStep,
   STEP_CHECKPOINT_PREFIX,
 } from "./run-migrations.js";
 
 const log = getLogger("memory-db");
 
+export interface MigrationValidationResult {
+  /** Keys of migrations whose checkpoint has value 'started' — started but never completed. */
+  crashed: string[];
+  /** Pairs where a completed migration's declared prerequisite is missing from checkpoints. */
+  dependencyViolations: Array<{ migration: string; missingDependency: string }>;
+  /** Checkpoint keys present in the database but absent from the known step names — likely from a newer version. */
+  unknownCheckpoints: string[];
+}
+
 /**
- * Validate the applied migration state against the registry at startup.
+ * Validate the applied migration state against the known step list at startup.
  *
  * Logs a prominent error when a migration started but never completed (crash
  * detected) — startup continues so the migration can be retried.
@@ -25,9 +33,15 @@ const log = getLogger("memory-db");
  *
  * Call this AFTER all DDL and migration functions have run so that the final
  * state is inspected.
+ *
+ * @param database  The Drizzle database instance.
+ * @param steps     The full ordered migration step array (same one passed to
+ *   `runMigrationSteps`). Used to determine known step names for unknown-checkpoint
+ *   detection and to check `dependsOn` declarations.
  */
 export function validateMigrationState(
   database: DrizzleDb,
+  steps: MigrationStep[],
 ): MigrationValidationResult {
   const raw = getSqliteFrom(database);
 
@@ -64,8 +78,6 @@ export function validateMigrationState(
   }
 
   // Build a set of completed step names from `step:*` checkpoints with value '1'.
-  // The step runner writes these — `migration_*` registry keys are no longer
-  // written by migration functions (Phase 2 removed withCrashRecovery).
   const completedStepNames = new Set(
     rows
       .filter(
@@ -75,32 +87,24 @@ export function validateMigrationState(
       .map((r) => r.key.slice(STEP_CHECKPOINT_PREFIX.length)),
   );
 
-  // Map registry entries to their step names to determine which migrations
-  // are completed. Multiple registry entries can share the same step name
-  // (e.g., 162's two entries both map to migrateGuardianTimestampsEpochMs).
-  const completed = new Set<string>();
-  for (const entry of MIGRATION_REGISTRY) {
-    if (completedStepNames.has(entry.stepName)) {
-      completed.add(entry.key);
-    }
-  }
-
+  // Check dependency ordering for object-form steps that declare dependsOn.
+  // A step is "completed" if its name appears in the completedStepNames set.
   const dependencyViolations: Array<{
     migration: string;
     missingDependency: string;
   }> = [];
 
-  // Validate dependency ordering.
-  for (const entry of MIGRATION_REGISTRY) {
-    if (!entry.dependsOn || entry.dependsOn.length === 0) continue;
-    // Only check entries that have been completed — unapplied or in-progress
+  for (const step of steps) {
+    const obj = normalizeStep(step);
+    if (!obj.dependsOn || obj.dependsOn.length === 0) continue;
+    // Only check steps that have been completed — unapplied or in-progress
     // migrations have not had a chance to violate their prerequisites yet.
-    if (!completed.has(entry.key)) continue;
+    if (!completedStepNames.has(obj.name)) continue;
 
-    for (const dep of entry.dependsOn) {
-      if (!completed.has(dep)) {
+    for (const dep of obj.dependsOn) {
+      if (!completedStepNames.has(dep)) {
         dependencyViolations.push({
-          migration: entry.key,
+          migration: obj.name,
           missingDependency: dep,
         });
       }
@@ -121,10 +125,11 @@ export function validateMigrationState(
   }
 
   // Detect step checkpoints that exist in the database but have no
-  // corresponding registry entry — these are from a newer version of the daemon.
-  const registryStepNames = new Set(MIGRATION_REGISTRY.map((e) => e.stepName));
+  // corresponding entry in the known step list — these are from a newer
+  // version of the daemon.
+  const knownStepNames = new Set(steps.map((s) => getStepName(s)));
   const unknownCheckpoints = [...completedStepNames].filter(
-    (name) => !registryStepNames.has(name),
+    (name) => !knownStepNames.has(name),
   );
 
   if (unknownCheckpoints.length > 0) {
@@ -140,23 +145,26 @@ export function validateMigrationState(
 /**
  * Roll back all completed memory (database) migrations with version > targetVersion.
  *
- * Iterates eligible migrations in reverse version order. For each:
- * 1. Marks the checkpoint as `"rolling_back"` for crash recovery.
+ * Iterates eligible migration steps in reverse version order. For each
+ * rollback entry on a completed step:
+ * 1. Marks the step checkpoint as `"rolling_back"` for crash recovery.
  * 2. Calls `entry.down(database)` — each down() manages its own transactions.
- *    (`down` is required on `MigrationRegistryEntry` at the type level.)
- * 3. Deletes the checkpoint from `memory_checkpoints`.
+ * 3. Deletes the step checkpoint from `memory_checkpoints`.
+ *
+ * A single step may carry multiple rollback entries (e.g. migration 162 has
+ * two). All entries with version > target are rolled back in reverse version
+ * order. The step checkpoint is only deleted after the last rollback entry
+ * for that step completes.
  *
  * **Usage**: Pass the target version number you want to roll back *to*. All
- * migrations with a higher version number that have been applied will be
- * reversed. For example, `rollbackMemoryMigration(db, 5)` rolls back all
- * applied migrations with version > 5.
+ * rollback entries with a higher version number on completed steps will be
+ * reversed. For example, `rollbackMemoryMigration(db, 5, steps)` rolls back
+ * all rollback entries with version > 5.
  *
- * **Checkpoint state**: Each rolled-back migration's checkpoint is deleted
- * from `memory_checkpoints`. If the process crashes mid-rollback, the
- * `"rolling_back"` marker is detected and cleared by
- * `recoverCrashedMigrations` on the next startup. The forward-step checkpoints
- * recorded by the migration runner (the `step:` namespace) for the rolled-back
- * entries are also discarded so a later upgrade re-applies them.
+ * **Checkpoint state**: Each rolled-back step's `step:*` checkpoint is deleted
+ * from `memory_checkpoints` after its down() functions succeed. If the process
+ * crashes mid-rollback, the `"rolling_back"` marker is detected and cleared by
+ * `recoverCrashedMigrations` on the next startup.
  *
  * **Warning — data loss**: Some down() migrations may not fully restore the
  * original state (e.g., DROP TABLE migrations recreate the table but cannot
@@ -168,13 +176,15 @@ export function validateMigrationState(
  * query failures, or data corruption.
  *
  * @param database  The Drizzle database instance.
- * @param targetVersion  Roll back to this version (exclusive — all migrations
- *   with version > targetVersion are reversed).
- * @returns The list of rolled-back migration keys.
+ * @param targetVersion  Roll back to this version (exclusive — all rollback
+ *   entries with version > targetVersion are reversed).
+ * @param steps     The full ordered migration step array.
+ * @returns The list of rolled-back step names.
  */
 export function rollbackMemoryMigration(
   database: DrizzleDb,
   targetVersion: number,
+  steps: MigrationStep[],
 ): string[] {
   const raw = getSqliteFrom(database);
 
@@ -198,41 +208,66 @@ export function rollbackMemoryMigration(
       .map((r) => r.key.slice(STEP_CHECKPOINT_PREFIX.length)),
   );
 
-  // Find registry entries with version > targetVersion whose step is completed.
-  // Deduplicate by stepName — multiple registry entries can share the same step
-  // (e.g., 162's two entries), and we only need to clear the step checkpoint once.
-  const toRollback = MIGRATION_REGISTRY.filter(
-    (entry) =>
-      entry.version > targetVersion &&
-      completedStepNames.has(entry.stepName),
-  ).sort((a, b) => b.version - a.version); // reverse version order
+  // Collect all rollback entries with version > targetVersion on completed steps.
+  // Each entry references its parent step name so we can clear the checkpoint
+  // after all rollback entries for that step have been executed.
+  interface RollbackItem {
+    stepName: string;
+    version: number;
+    description: string;
+    down: (db: DrizzleDb) => void;
+  }
+
+  const toRollback: RollbackItem[] = [];
+  for (const step of steps) {
+    const obj = normalizeStep(step);
+    if (!obj.rollback) continue;
+    if (!completedStepNames.has(obj.name)) continue;
 
+    for (const entry of obj.rollback) {
+      if (entry.version > targetVersion) {
+        toRollback.push({
+          stepName: obj.name,
+          version: entry.version,
+          description: entry.description,
+          down: entry.down,
+        });
+      }
+    }
+  }
+
+  // Sort in reverse version order — children (higher version) before parents.
+  toRollback.sort((a, b) => b.version - a.version);
+
+  // Group by stepName so we only delete the checkpoint after all rollback
+  // entries for that step have been executed.
+  const stepNamesToRollback = new Set(toRollback.map((r) => r.stepName));
   const rolledBack: string[] = [];
 
-  for (const entry of toRollback) {
-    const stepKey = `${STEP_CHECKPOINT_PREFIX}${entry.stepName}`;
+  for (const item of toRollback) {
+    const stepKey = `${STEP_CHECKPOINT_PREFIX}${item.stepName}`;
 
-    // Mark as rolling_back for crash recovery — if the process crashes here,
-    // recoverCrashedMigrations will clear this checkpoint on next startup.
+    // Mark as rolling_back for crash recovery.
     raw
       .query(
         `INSERT OR REPLACE INTO memory_checkpoints (key, value, updated_at) VALUES (?, 'rolling_back', ?)`,
       )
       .run(stepKey, Date.now());
 
-    // Execute the down migration — let it manage its own transaction lifecycle.
-    // Many down() functions call BEGIN/COMMIT internally or use PRAGMA statements
-    // that are no-ops inside a transaction.
-    entry.down(database);
-
-    // Delete the step checkpoint after down() succeeds.
-    raw.query(`DELETE FROM memory_checkpoints WHERE key = ?`).run(stepKey);
+    // Execute the down migration.
+    item.down(database);
 
     log.info(
-      { key: entry.key, version: entry.version },
-      `Rolled back migration "${entry.key}" (version ${entry.version})`,
+      { step: item.stepName, version: item.version, description: item.description },
+      `Rolled back migration "${item.stepName}" (version ${item.version})`,
     );
-    rolledBack.push(entry.key);
+  }
+
+  // After all down() calls succeed, delete checkpoints for the rolled-back steps.
+  for (const stepName of stepNamesToRollback) {
+    const stepKey = `${STEP_CHECKPOINT_PREFIX}${stepName}`;
+    raw.query(`DELETE FROM memory_checkpoints WHERE key = ?`).run(stepKey);
+    rolledBack.push(stepName);
   }
 
   return rolledBack;

package/src/memory/prompt-override.ts

@@ -0,0 +1,129 @@
+/**
+ * Shared loader for file-based prompt overrides.
+ *
+ * The memory router, consolidation, and v3 selector prompts each ship a bundled
+ * default that operators may replace by pointing a config field at a file. The
+ * load-with-fallback semantics are identical across all three, so they live
+ * here: path resolution, the size guard, and the permissive fallback that keeps
+ * retrieval/consolidation working when an override is missing or malformed.
+ *
+ * The caller owns the bundled default and any placeholder substitution — this
+ * module only decides whether a usable override file exists and returns its raw
+ * contents, or `null` to mean "fall back to the bundled prompt."
+ */
+
+import { lstatSync, readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { isAbsolute, join } from "node:path";
+
+/**
+ * Minimal logger surface the loader needs. Structurally compatible with
+ * `pino.Logger`, so callers pass their module logger directly and the warnings
+ * keep that logger's namespace; tests pass a lightweight recorder.
+ */
+export interface PromptOverrideLogger {
+  warn(obj: object, msg: string): void;
+}
+
+/**
+ * Hard upper bound on an override file. The bundled prompts are kilobytes; 1 MiB
+ * is generous headroom for a hand-edit while preventing a `settings.write`
+ * principal from pointing the field at a giant file (or an unsizeable stream
+ * `lstat` can't cap on its own) and slurping it into memory on every call.
+ */
+export const MAX_PROMPT_OVERRIDE_BYTES = 1 * 1024 * 1024;
+
+/**
+ * Resolve a configured override path to an absolute path: a leading `~/` expands
+ * to the home directory, an absolute path is used as-is, and a relative path
+ * resolves under `workspaceDir`.
+ */
+export function resolveOverridePath(
+  overridePath: string,
+  workspaceDir: string,
+): string {
+  if (overridePath.startsWith("~/")) {
+    return join(homedir(), overridePath.slice(2));
+  }
+  if (isAbsolute(overridePath)) return overridePath;
+  return join(workspaceDir, overridePath);
+}
+
+/**
+ * Load a prompt-override file, returning its raw contents when the override is
+ * present and usable, or `null` (after logging a warning describing why) when
+ * the caller should fall back to its bundled prompt. A nullish `overridePath`
+ * (`null` or `undefined`) returns `null` without touching the filesystem.
+ *
+ * Fallback is intentionally total — an absent override, a missing, non-regular,
+ * oversized, unreadable, or empty/whitespace-only file all degrade to `null`.
+ * Memory retrieval and consolidation must never break because of a bad
+ * override, so `undefined` (an unset config field) is treated as "no override"
+ * rather than crashing on path resolution.
+ *
+ * `label` names the prompt in the warning messages (e.g. `"router prompt"`).
+ */
+export function loadPromptOverride(opts: {
+  overridePath: string | null | undefined;
+  workspaceDir: string;
+  log: PromptOverrideLogger;
+  label: string;
+}): string | null {
+  const { overridePath, workspaceDir, log, label } = opts;
+  if (overridePath == null) return null;
+
+  const resolvedPath = resolveOverridePath(overridePath, workspaceDir);
+  let contents: string;
+  try {
+    const stat = lstatSync(resolvedPath);
+    if (!stat.isFile()) {
+      log.warn(
+        {
+          configuredPath: overridePath,
+          resolvedPath,
+          reason: "not_regular_file",
+          fallback: "bundled",
+        },
+        `${label} override is not a regular file; using bundled prompt`,
+      );
+      return null;
+    }
+    if (stat.size > MAX_PROMPT_OVERRIDE_BYTES) {
+      log.warn(
+        {
+          configuredPath: overridePath,
+          resolvedPath,
+          size: stat.size,
+          limit: MAX_PROMPT_OVERRIDE_BYTES,
+          reason: "oversized_override",
+          fallback: "bundled",
+        },
+        `${label} override exceeds size limit; using bundled prompt`,
+      );
+      return null;
+    }
+    contents = readFileSync(resolvedPath, "utf-8");
+  } catch (err) {
+    const code = (err as NodeJS.ErrnoException).code;
+    log.warn(
+      { configuredPath: overridePath, resolvedPath, code, fallback: "bundled" },
+      `${label} override unreadable; using bundled prompt`,
+    );
+    return null;
+  }
+
+  if (contents.trim().length === 0) {
+    log.warn(
+      {
+        configuredPath: overridePath,
+        resolvedPath,
+        reason: "empty_override",
+        fallback: "bundled",
+      },
+      `${label} override is empty; using bundled prompt`,
+    );
+    return null;
+  }
+
+  return contents;
+}

package/src/memory/schema/conversations.ts

@@ -52,6 +52,15 @@ export const conversations = sqliteTable(
     inferenceProfileSessionId: text("inference_profile_session_id"),
     inferenceProfileExpiresAt: integer("inference_profile_expires_at"),
     lastNotifiedInferenceProfile: text("last_notified_inference_profile"),
+    /**
+     * Epoch-ms timestamp set when the agent loop starts a turn for this
+     * conversation, cleared (NULL) when the turn ends. NULL means not
+     * processing. This is the cross-process source of truth for processing
+     * state — the in-memory `Conversation._processing` flag is the hot-path
+     * read for resident conversations, but CLI-side and other out-of-process
+     * callers read this column directly.
+     */
+    processingStartedAt: integer("processing_started_at"),
   },
   (table) => [
     index("idx_conversations_updated_at").on(table.updatedAt),

package/src/memory/schema/infrastructure.ts

@@ -337,6 +337,26 @@ export const skillLoadedEvents = sqliteTable(
   ],
 );
 
+// One row per `watchdog` telemetry event, emitted when a daemon watchdog
+// check fires (event-loop block, stream-idle stall, restart, ...) — see
+// watchdog-events-store.ts for the data contract. Flushed by the usage
+// telemetry reporter. `value` is a REAL (BQ FLOAT) so the daemon need not
+// distinguish int vs float; the platform serializer coerces ints to float.
+// `detail` is a JSON bag stored as text and forwarded verbatim.
+export const watchdogEvents = sqliteTable(
+  "watchdog_events",
+  {
+    id: text("id").primaryKey(),
+    createdAt: integer("created_at").notNull(),
+    checkName: text("check_name").notNull(),
+    value: real("value"),
+    detail: text("detail"),
+  },
+  (table) => [
+    index("idx_watchdog_events_created_at_id").on(table.createdAt, table.id),
+  ],
+);
+
 export const traceEvents = sqliteTable(
   "trace_events",
   {