opencandle 0.4.0 → 0.6.0

package/src/memory/index.ts

@@ -1,10 +1,10 @@
-export { initDatabase, initDefaultDatabase, getTableNames, getSchemaVersion } from "./sqlite.js";
-export { MemoryStorage } from "./storage.js";
-export type { WorkflowPreferences } from "./storage.js";
-export { buildMemoryContext } from "./retrieval.js";
-export { extractPreferences } from "./preference-extractor.js";
 export { MemoryManager } from "./manager.js";
-export { getAllDefaults, getDefaults, setDefault, clearDefault } from "./tool-defaults.js";
+export { extractPreferences } from "./preference-extractor.js";
+export { buildMemoryContext } from "./retrieval.js";
+export { getSchemaVersion, getTableNames, initDatabase, initDefaultDatabase } from "./sqlite.js";
+export type { WorkflowPreferences } from "./storage.js";
+export { MemoryStorage } from "./storage.js";
 export type { ToolDefaults } from "./tool-defaults.js";
+export { clearDefault, getAllDefaults, getDefaults, setDefault } from "./tool-defaults.js";
 export type { MemoryCategory, MemoryEntry } from "./types.js";
-export { isStale, STALENESS_THRESHOLDS, NEVER_TRUST_FROM_MEMORY } from "./types.js";
+export { isStale, NEVER_TRUST_FROM_MEMORY, STALENESS_THRESHOLDS } from "./types.js";

package/src/memory/manager.ts

@@ -1,12 +1,22 @@
 import type { MemoryStorage } from "./storage.js";
 import type { MemoryCategory, MemoryEntry } from "./types.js";
 import {
+  isStale,
   KEY_TO_CATEGORY,
-  WORKFLOW_RELEVANT_CATEGORIES,
   NEVER_TRUST_FROM_MEMORY,
-  isStale,
+  WORKFLOW_RELEVANT_CATEGORIES,
 } from "./types.js";
 
+export interface FilteredMemoryEntry {
+  entry: MemoryEntry;
+  reason: "suppressed_by_user_slot" | "never_trust" | "stale" | "irrelevant_category";
+}
+
+export interface MemoryRetrievalResult {
+  entries: MemoryEntry[];
+  filtered: FilteredMemoryEntry[];
+}
+
 /** Slot name → preference key(s) mapping for suppression. */
 const SLOT_TO_PREF_KEYS: Record<string, string[]> = {
   riskProfile: ["risk_profile"],
@@ -36,31 +46,39 @@ export class MemoryManager {
     overriddenSlots?: string[],
     now: Date = new Date(),
   ): MemoryEntry[] {
-    const relevantCategories = WORKFLOW_RELEVANT_CATEGORIES[workflowType] ??
-      WORKFLOW_RELEVANT_CATEGORIES["unclassified"];
+    return this.retrieveDetailed(workflowType, overriddenSlots, now).entries;
+  }
+
+  retrieveDetailed(
+    workflowType: string,
+    overriddenSlots?: string[],
+    now: Date = new Date(),
+  ): MemoryRetrievalResult {
+    const relevantCategories =
+      WORKFLOW_RELEVANT_CATEGORIES[workflowType] ?? WORKFLOW_RELEVANT_CATEGORIES.unclassified;
 
     // Build set of preference keys to suppress
     const suppressedKeys = new Set<string>();
     if (overriddenSlots) {
       for (const slot of overriddenSlots) {
         const keys = SLOT_TO_PREF_KEYS[slot];
-        if (keys) keys.forEach((k) => suppressedKeys.add(k));
+        if (keys) {
+          for (const key of keys) {
+            suppressedKeys.add(key);
+          }
+        }
       }
     }
 
     const entries: MemoryEntry[] = [];
+    const filtered: FilteredMemoryEntry[] = [];
 
     // Preferences as investor_profile entries
     if (relevantCategories.includes("investor_profile")) {
       const prefs = this.storage.getPreferencesByNamespace("global");
       for (const pref of prefs) {
         const key = String(pref.key);
-        if (suppressedKeys.has(key)) continue;
-        if (NEVER_TRUST_FROM_MEMORY.has(key)) continue;
-
         const category = KEY_TO_CATEGORY[key] ?? "investor_profile";
-        if (!relevantCategories.includes(category)) continue;
-
         const entry: MemoryEntry = {
           key,
           value: tryParseValue(String(pref.value_json ?? "")),
@@ -70,9 +88,23 @@ export class MemoryManager {
           source: pref.source != null ? String(pref.source) : undefined,
         };
 
-        if (!isStale(entry, now)) {
-          entries.push(entry);
+        if (suppressedKeys.has(key)) {
+          filtered.push({ entry, reason: "suppressed_by_user_slot" });
+          continue;
+        }
+        if (NEVER_TRUST_FROM_MEMORY.has(key)) {
+          filtered.push({ entry, reason: "never_trust" });
+          continue;
         }
+        if (!relevantCategories.includes(category)) {
+          filtered.push({ entry, reason: "irrelevant_category" });
+          continue;
+        }
+        if (isStale(entry, now)) {
+          filtered.push({ entry, reason: "stale" });
+          continue;
+        }
+        entries.push(entry);
       }
     }
 
@@ -90,30 +122,29 @@ export class MemoryManager {
         const recordedAt = String(run.created_at ?? now.toISOString());
         const entry: MemoryEntry = {
           key: `workflow_run_${run.id}`,
-          value: run.output_summary
-            ? `${wfType}: ${run.output_summary}`
-            : wfType,
+          value: run.output_summary ? `${wfType}: ${run.output_summary}` : wfType,
           category: "workflow_history",
           recordedAt,
         };
 
-        if (!isStale(entry, now)) {
-          entries.push(entry);
+        if (isStale(entry, now)) {
+          filtered.push({ entry, reason: "stale" });
+          continue;
         }
+        entries.push(entry);
       }
     }
 
-    return entries.slice(0, MAX_PREFERENCE_LINES + MAX_WORKFLOW_HISTORY_PER_TYPE * 4);
+    return {
+      entries: entries.slice(0, MAX_PREFERENCE_LINES + MAX_WORKFLOW_HISTORY_PER_TYPE * 4),
+      filtered,
+    };
   }
 
   /**
    * Build compact text context from retrieved memory entries.
    */
-  buildContext(
-    workflowType: string,
-    overriddenSlots?: string[],
-    now: Date = new Date(),
-  ): string {
+  buildContext(workflowType: string, overriddenSlots?: string[], now: Date = new Date()): string {
     const entries = this.retrieve(workflowType, overriddenSlots, now);
     if (entries.length === 0) return "";
 
@@ -130,19 +161,19 @@ export class MemoryManager {
     const profileEntries = byCategory.get("investor_profile");
     if (profileEntries && profileEntries.length > 0) {
       const lines = profileEntries.map((e) => `- ${e.key}: ${e.value}`);
-      sections.push("User Preferences:\n" + lines.join("\n"));
+      sections.push(`User Preferences:\n${lines.join("\n")}`);
     }
 
     const historyEntries = byCategory.get("workflow_history");
     if (historyEntries && historyEntries.length > 0) {
       const lines = historyEntries.map((e) => `- ${e.value} (${e.recordedAt})`);
-      sections.push("Recent Workflows:\n" + lines.join("\n"));
+      sections.push(`Recent Workflows:\n${lines.join("\n")}`);
     }
 
     const feedbackEntries = byCategory.get("interaction_feedback");
     if (feedbackEntries && feedbackEntries.length > 0) {
       const lines = feedbackEntries.map((e) => `- ${e.key}: ${e.value}`);
-      sections.push("Feedback:\n" + lines.join("\n"));
+      sections.push(`Feedback:\n${lines.join("\n")}`);
     }
 
     return sections.join("\n\n");

package/src/memory/retrieval.ts

@@ -19,10 +19,7 @@ const SLOT_TO_PREF_KEYS: Record<string, string[]> = {
  *   The corresponding preference keys will be excluded from memory context to avoid
  *   conflicting provenance signals.
  */
-export function buildMemoryContext(
-  storage: MemoryStorage,
-  overriddenSlots?: string[],
-): string {
+export function buildMemoryContext(storage: MemoryStorage, overriddenSlots?: string[]): string {
   const sections: string[] = [];
 
   // Build set of preference keys to suppress
@@ -30,7 +27,11 @@ export function buildMemoryContext(
   if (overriddenSlots) {
     for (const slot of overriddenSlots) {
       const keys = SLOT_TO_PREF_KEYS[slot];
-      if (keys) keys.forEach((k) => suppressedKeys.add(k));
+      if (keys) {
+        for (const key of keys) {
+          suppressedKeys.add(key);
+        }
+      }
     }
   }
 
@@ -43,7 +44,7 @@ export function buildMemoryContext(
         const value = tryParseJson(p.value_json as string);
         return `- ${p.key}: ${value}`;
       });
-      sections.push("User Preferences:\n" + lines.join("\n"));
+      sections.push(`User Preferences:\n${lines.join("\n")}`);
     }
   }
 
@@ -54,7 +55,7 @@ export function buildMemoryContext(
       const summary = r.output_summary ? ` — ${r.output_summary}` : "";
       return `- ${r.workflow_type} (${r.created_at})${summary}`;
     });
-    sections.push("Recent Workflows:\n" + lines.join("\n"));
+    sections.push(`Recent Workflows:\n${lines.join("\n")}`);
   }
 
   return sections.join("\n\n");

package/src/memory/sqlite.ts

@@ -1,9 +1,9 @@
 import { mkdirSync } from "node:fs";
 import { dirname } from "node:path";
 import Database from "better-sqlite3";
-import { getStateDbPath } from "../infra/opencandle-paths.js";
+import { ensureOpenCandleHomeDir, getStateDbPath } from "../infra/opencandle-paths.js";
 
-const CURRENT_SCHEMA_VERSION = 4;
+const CURRENT_SCHEMA_VERSION = 7;
 
 const CURRENT_SCHEMA = `
   CREATE TABLE IF NOT EXISTS schema_version (
@@ -62,6 +62,294 @@ const CURRENT_SCHEMA = `
     set_at TEXT NOT NULL,
     PRIMARY KEY (tool_name, param_path)
   );
+
+  CREATE TABLE IF NOT EXISTS instruments (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    symbol TEXT NOT NULL,
+    asset_type TEXT NOT NULL,
+    name TEXT,
+    exchange TEXT,
+    currency TEXT,
+    provider TEXT NOT NULL,
+    provider_metadata_json TEXT,
+    last_resolved_at TEXT NOT NULL,
+    created_at TEXT NOT NULL,
+    updated_at TEXT NOT NULL
+  );
+
+  CREATE INDEX IF NOT EXISTS idx_instruments_symbol ON instruments(symbol);
+  CREATE UNIQUE INDEX IF NOT EXISTS idx_instruments_provider_identity
+    ON instruments(provider, symbol, asset_type, IFNULL(exchange, ''));
+
+  CREATE TABLE IF NOT EXISTS instrument_aliases (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    instrument_id INTEGER NOT NULL,
+    source TEXT NOT NULL,
+    source_symbol TEXT NOT NULL,
+    source_exchange TEXT,
+    source_asset_type TEXT,
+    source_id TEXT,
+    raw_json TEXT,
+    created_at TEXT NOT NULL,
+    updated_at TEXT NOT NULL,
+    FOREIGN KEY (instrument_id) REFERENCES instruments(id) ON DELETE RESTRICT
+  );
+
+  CREATE UNIQUE INDEX IF NOT EXISTS idx_instrument_aliases_source_id
+    ON instrument_aliases(source, source_id)
+    WHERE source_id IS NOT NULL;
+  CREATE UNIQUE INDEX IF NOT EXISTS idx_instrument_aliases_source_symbol
+    ON instrument_aliases(source, source_symbol, IFNULL(source_exchange, ''), IFNULL(source_asset_type, ''))
+    WHERE source_id IS NULL;
+
+  CREATE TABLE IF NOT EXISTS watchlists (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    name TEXT NOT NULL,
+    is_default INTEGER NOT NULL DEFAULT 0,
+    created_at TEXT NOT NULL,
+    updated_at TEXT NOT NULL
+  );
+
+  CREATE UNIQUE INDEX IF NOT EXISTS idx_watchlists_one_default
+    ON watchlists(is_default)
+    WHERE is_default = 1;
+
+  CREATE TABLE IF NOT EXISTS watchlist_items (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    watchlist_id INTEGER NOT NULL,
+    instrument_id INTEGER NOT NULL,
+    thesis TEXT,
+    notes TEXT,
+    tags_json TEXT,
+    target_price REAL,
+    stop_price REAL,
+    price_currency TEXT,
+    source TEXT,
+    source_row_id TEXT,
+    source_metadata_json TEXT,
+    created_at TEXT NOT NULL,
+    updated_at TEXT NOT NULL,
+    UNIQUE(watchlist_id, instrument_id),
+    FOREIGN KEY (watchlist_id) REFERENCES watchlists(id) ON DELETE CASCADE,
+    FOREIGN KEY (instrument_id) REFERENCES instruments(id) ON DELETE RESTRICT
+  );
+
+  CREATE INDEX IF NOT EXISTS idx_watchlist_items_source_row
+    ON watchlist_items(source, source_row_id)
+    WHERE source IS NOT NULL AND source_row_id IS NOT NULL;
+
+  CREATE TABLE IF NOT EXISTS portfolios (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    name TEXT NOT NULL,
+    base_currency TEXT NOT NULL DEFAULT 'USD',
+    is_default INTEGER NOT NULL DEFAULT 0,
+    created_at TEXT NOT NULL,
+    updated_at TEXT NOT NULL
+  );
+
+  CREATE UNIQUE INDEX IF NOT EXISTS idx_portfolios_one_default
+    ON portfolios(is_default)
+    WHERE is_default = 1;
+
+  CREATE TABLE IF NOT EXISTS portfolio_lots (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    portfolio_id INTEGER NOT NULL,
+    instrument_id INTEGER NOT NULL,
+    quantity REAL NOT NULL,
+    avg_cost REAL NOT NULL,
+    currency TEXT NOT NULL,
+    opened_at TEXT,
+    notes TEXT,
+    source TEXT,
+    source_account_ref TEXT,
+    source_lot_id TEXT,
+    source_row_id TEXT,
+    source_metadata_json TEXT,
+    created_at TEXT NOT NULL,
+    updated_at TEXT NOT NULL,
+    FOREIGN KEY (portfolio_id) REFERENCES portfolios(id) ON DELETE CASCADE,
+    FOREIGN KEY (instrument_id) REFERENCES instruments(id) ON DELETE RESTRICT
+  );
+
+  CREATE INDEX IF NOT EXISTS idx_portfolio_lots_source_row
+    ON portfolio_lots(source, source_row_id)
+    WHERE source IS NOT NULL AND source_row_id IS NOT NULL;
+  CREATE INDEX IF NOT EXISTS idx_portfolio_lots_source_lot
+    ON portfolio_lots(source, source_lot_id)
+    WHERE source IS NOT NULL AND source_lot_id IS NOT NULL;
+
+  CREATE TABLE IF NOT EXISTS prediction_records (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    instrument_id INTEGER NOT NULL,
+    direction TEXT NOT NULL,
+    conviction REAL NOT NULL,
+    entry_price REAL NOT NULL,
+    target_price REAL,
+    opened_at TEXT NOT NULL,
+    expires_at TEXT NOT NULL,
+    status TEXT NOT NULL,
+    resolved_at TEXT,
+    result_json TEXT,
+    created_at TEXT NOT NULL,
+    updated_at TEXT NOT NULL,
+    FOREIGN KEY (instrument_id) REFERENCES instruments(id) ON DELETE RESTRICT
+  );
+
+  CREATE TABLE IF NOT EXISTS alert_rules (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    scope_type TEXT NOT NULL,
+    scope_id INTEGER,
+    instrument_id INTEGER,
+    condition_type TEXT NOT NULL,
+    condition_version INTEGER NOT NULL,
+    condition_json TEXT NOT NULL,
+    timeframe TEXT NOT NULL,
+    enabled INTEGER NOT NULL DEFAULT 1,
+    check_interval_seconds INTEGER,
+    next_check_at TEXT,
+    last_checked_at TEXT,
+    last_observed_json TEXT,
+    status TEXT NOT NULL DEFAULT 'active',
+    retrigger_mode TEXT NOT NULL DEFAULT 'recurring',
+    last_condition_state TEXT NOT NULL DEFAULT 'unknown',
+    rule_revision INTEGER NOT NULL DEFAULT 1,
+    arm_cycle_id INTEGER NOT NULL DEFAULT 1,
+    cooldown_seconds INTEGER,
+    last_triggered_at TEXT,
+    created_at TEXT NOT NULL,
+    updated_at TEXT NOT NULL,
+    FOREIGN KEY (instrument_id) REFERENCES instruments(id) ON DELETE RESTRICT
+  );
+
+  CREATE TABLE IF NOT EXISTS alert_events (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    alert_rule_id INTEGER NOT NULL,
+    instrument_id INTEGER,
+    observed_value_json TEXT,
+    triggered_at TEXT NOT NULL,
+    observed_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    provider_data_at TEXT,
+    source_provider TEXT,
+    cache_status TEXT NOT NULL DEFAULT 'live',
+    data_delay_ms INTEGER,
+    trigger_source TEXT NOT NULL DEFAULT 'manual',
+    dedupe_key TEXT,
+    status TEXT NOT NULL,
+    message TEXT,
+    FOREIGN KEY (alert_rule_id) REFERENCES alert_rules(id) ON DELETE CASCADE,
+    FOREIGN KEY (instrument_id) REFERENCES instruments(id) ON DELETE RESTRICT
+  );
+
+  CREATE UNIQUE INDEX IF NOT EXISTS idx_alert_events_dedupe
+    ON alert_events(dedupe_key)
+    WHERE dedupe_key IS NOT NULL;
+
+  CREATE TABLE IF NOT EXISTS alert_check_runs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    started_at TEXT NOT NULL,
+    completed_at TEXT,
+    status TEXT NOT NULL,
+    trigger_type TEXT NOT NULL,
+    checked_count INTEGER NOT NULL DEFAULT 0,
+    triggered_count INTEGER NOT NULL DEFAULT 0,
+    unavailable_count INTEGER NOT NULL DEFAULT 0,
+    owner_id TEXT,
+    error_json TEXT,
+    provider_status_json TEXT
+  );
+
+  CREATE TABLE IF NOT EXISTS report_templates (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    name TEXT NOT NULL,
+    report_type TEXT NOT NULL,
+    cadence TEXT NOT NULL,
+    timezone TEXT NOT NULL,
+    local_time TEXT NOT NULL,
+    config_json TEXT NOT NULL,
+    enabled INTEGER NOT NULL DEFAULT 1,
+    last_run_at TEXT,
+    next_run_at TEXT,
+    created_at TEXT NOT NULL,
+    updated_at TEXT NOT NULL
+  );
+
+  CREATE TABLE IF NOT EXISTS report_runs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    template_id INTEGER,
+    started_at TEXT NOT NULL,
+    completed_at TEXT,
+    status TEXT NOT NULL,
+    trigger_type TEXT NOT NULL DEFAULT 'manual',
+    scheduled_for TEXT,
+    owner_id TEXT,
+    artifact_path TEXT,
+    summary_json TEXT,
+    errors_json TEXT,
+    FOREIGN KEY (template_id) REFERENCES report_templates(id) ON DELETE SET NULL
+  );
+
+  CREATE TABLE IF NOT EXISTS automation_runner_leases (
+    id INTEGER PRIMARY KEY CHECK (id = 1),
+    owner_id TEXT NOT NULL,
+    owner_kind TEXT NOT NULL,
+    acquired_at TEXT NOT NULL,
+    heartbeat_at TEXT NOT NULL,
+    expires_at TEXT NOT NULL
+  );
+
+  CREATE TABLE IF NOT EXISTS notification_events (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    source_type TEXT NOT NULL,
+    source_id INTEGER,
+    severity TEXT NOT NULL,
+    title TEXT NOT NULL,
+    body TEXT NOT NULL,
+    payload_json TEXT,
+    status TEXT NOT NULL,
+    created_at TEXT NOT NULL,
+    acknowledged_at TEXT
+  );
+
+  CREATE TABLE IF NOT EXISTS notification_delivery_attempts (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    notification_event_id INTEGER NOT NULL,
+    channel TEXT NOT NULL,
+    status TEXT NOT NULL,
+    attempted_at TEXT NOT NULL,
+    completed_at TEXT,
+    response_json TEXT,
+    error TEXT,
+    FOREIGN KEY (notification_event_id) REFERENCES notification_events(id) ON DELETE CASCADE
+  );
+
+  CREATE TABLE IF NOT EXISTS import_batches (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    source TEXT NOT NULL,
+    source_label TEXT,
+    imported_at TEXT NOT NULL,
+    status TEXT NOT NULL,
+    raw_metadata_json TEXT
+  );
+
+  CREATE TABLE IF NOT EXISTS import_rows (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    batch_id INTEGER NOT NULL,
+    row_type TEXT NOT NULL,
+    source_symbol TEXT,
+    source_row_id TEXT,
+    source_account_ref TEXT,
+    normalized_instrument_id INTEGER,
+    status TEXT NOT NULL,
+    error TEXT,
+    source_metadata_json TEXT,
+    raw_json TEXT,
+    FOREIGN KEY (batch_id) REFERENCES import_batches(id) ON DELETE CASCADE,
+    FOREIGN KEY (normalized_instrument_id) REFERENCES instruments(id) ON DELETE SET NULL
+  );
+
+  CREATE UNIQUE INDEX IF NOT EXISTS idx_import_rows_batch_source_row
+    ON import_rows(batch_id, source_row_id)
+    WHERE source_row_id IS NOT NULL;
 `;
 
 export function initDatabase(path: string): Database.Database {
@@ -70,6 +358,7 @@ export function initDatabase(path: string): Database.Database {
   }
   const db = new Database(path);
   db.pragma("journal_mode = WAL");
+  db.pragma("busy_timeout = 5000");
   db.pragma("foreign_keys = ON");
   ensureCurrentSchema(db);
 
@@ -77,6 +366,7 @@ export function initDatabase(path: string): Database.Database {
 }
 
 export function initDefaultDatabase(): Database.Database {
+  ensureOpenCandleHomeDir();
   return initDatabase(getStateDbPath());
 }
 
@@ -90,15 +380,39 @@ function ensureCurrentSchema(db: Database.Database): void {
     return;
   }
 
+  if (currentVersion === 6) {
+    migrateV6ToV7(db);
+    return;
+  }
+
+  if (currentVersion === 5) {
+    migrateV5ToV6(db);
+    migrateV6ToV7(db);
+    return;
+  }
+
+  if (currentVersion === 4) {
+    migrateV4ToV5(db);
+    migrateV5ToV6(db);
+    migrateV6ToV7(db);
+    return;
+  }
+
   if (currentVersion === 3) {
     migrateV3ToV4(db);
+    migrateV4ToV5(db);
+    migrateV5ToV6(db);
+    migrateV6ToV7(db);
     return;
   }
 
-  // Additive v2 → v3 → v4 migration without dropping data.
+  // Additive v2 → v3 → v4 → v5 → v6 migration without dropping data.
   if (currentVersion === 2) {
     migrateV2ToV3(db);
     migrateV3ToV4(db);
+    migrateV4ToV5(db);
+    migrateV5ToV6(db);
+    migrateV6ToV7(db);
     return;
   }
 
@@ -112,9 +426,7 @@ function migrateV2ToV3(db: Database.Database): void {
   );
 
   if (!cols.includes("turn_type")) {
-    db.exec(
-      `ALTER TABLE workflow_runs ADD COLUMN turn_type TEXT NOT NULL DEFAULT 'workflow'`,
-    );
+    db.exec(`ALTER TABLE workflow_runs ADD COLUMN turn_type TEXT NOT NULL DEFAULT 'workflow'`);
   }
 
   // Ensure any tables or indexes added between versions are present.
@@ -125,12 +437,84 @@ function migrateV2ToV3(db: Database.Database): void {
 }
 
 function migrateV3ToV4(db: Database.Database): void {
+  // CURRENT_SCHEMA indexes alert_events(dedupe_key); pre-v7 tables lack the column.
+  addColumnIfMissing(db, "alert_events", "dedupe_key", "TEXT");
+  db.exec(CURRENT_SCHEMA);
+
+  db.prepare("DELETE FROM schema_version").run();
+  db.prepare("INSERT INTO schema_version (version) VALUES (?)").run(4);
+}
+
+function migrateV4ToV5(db: Database.Database): void {
+  addColumnIfMissing(db, "alert_events", "dedupe_key", "TEXT");
+  db.exec(CURRENT_SCHEMA);
+
+  db.prepare("DELETE FROM schema_version").run();
+  db.prepare("INSERT INTO schema_version (version) VALUES (?)").run(5);
+}
+
+function migrateV5ToV6(db: Database.Database): void {
+  addColumnIfMissing(db, "import_rows", "source_row_id", "TEXT");
+  addColumnIfMissing(db, "import_rows", "source_account_ref", "TEXT");
+  addColumnIfMissing(db, "import_rows", "source_metadata_json", "TEXT");
+
+  db.prepare("DELETE FROM schema_version").run();
+  db.prepare("INSERT INTO schema_version (version) VALUES (?)").run(6);
+}
+
+function migrateV6ToV7(db: Database.Database): void {
+  addColumnIfMissing(db, "alert_rules", "status", "TEXT NOT NULL DEFAULT 'active'");
+  addColumnIfMissing(db, "alert_rules", "retrigger_mode", "TEXT NOT NULL DEFAULT 'recurring'");
+  addColumnIfMissing(db, "alert_rules", "last_condition_state", "TEXT NOT NULL DEFAULT 'unknown'");
+  addColumnIfMissing(db, "alert_rules", "rule_revision", "INTEGER NOT NULL DEFAULT 1");
+  addColumnIfMissing(db, "alert_rules", "arm_cycle_id", "INTEGER NOT NULL DEFAULT 1");
+
+  addColumnIfMissing(db, "alert_events", "observed_at", "TEXT");
+  addColumnIfMissing(db, "alert_events", "provider_data_at", "TEXT");
+  addColumnIfMissing(db, "alert_events", "source_provider", "TEXT");
+  addColumnIfMissing(db, "alert_events", "cache_status", "TEXT NOT NULL DEFAULT 'live'");
+  addColumnIfMissing(db, "alert_events", "data_delay_ms", "INTEGER");
+  addColumnIfMissing(db, "alert_events", "trigger_source", "TEXT NOT NULL DEFAULT 'manual'");
+  addColumnIfMissing(db, "alert_events", "dedupe_key", "TEXT");
+
+  addColumnIfMissing(db, "report_runs", "trigger_type", "TEXT NOT NULL DEFAULT 'manual'");
+  addColumnIfMissing(db, "report_runs", "scheduled_for", "TEXT");
+  addColumnIfMissing(db, "report_runs", "owner_id", "TEXT");
+
+  if (tableExists(db, "alert_events")) {
+    db.exec("UPDATE alert_events SET observed_at = triggered_at WHERE observed_at IS NULL");
+  }
+
   db.exec(CURRENT_SCHEMA);
 
   db.prepare("DELETE FROM schema_version").run();
   db.prepare("INSERT INTO schema_version (version) VALUES (?)").run(CURRENT_SCHEMA_VERSION);
 }
 
+function addColumnIfMissing(
+  db: Database.Database,
+  tableName: string,
+  columnName: string,
+  definition: string,
+): void {
+  const cols = (db.pragma(`table_info(${tableName})`) as Array<{ name: string }>).map(
+    (c) => c.name,
+  );
+  if (cols.length === 0) {
+    if (!tableExists(db, tableName)) return;
+  }
+  if (!cols.includes(columnName)) {
+    db.exec(`ALTER TABLE ${tableName} ADD COLUMN ${columnName} ${definition}`);
+  }
+}
+
+function tableExists(db: Database.Database, tableName: string): boolean {
+  const table = db
+    .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?")
+    .get(tableName) as { name: string } | undefined;
+  return table != null;
+}
+
 function readSchemaVersion(db: Database.Database): number | null {
   const table = db
     .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'schema_version'")
@@ -147,6 +531,23 @@ function readSchemaVersion(db: Database.Database): number | null {
 
 function resetSchema(db: Database.Database): void {
   db.exec(`
+    DROP TABLE IF EXISTS import_rows;
+    DROP TABLE IF EXISTS import_batches;
+    DROP TABLE IF EXISTS notification_delivery_attempts;
+    DROP TABLE IF EXISTS notification_events;
+    DROP TABLE IF EXISTS automation_runner_leases;
+    DROP TABLE IF EXISTS alert_check_runs;
+    DROP TABLE IF EXISTS report_runs;
+    DROP TABLE IF EXISTS report_templates;
+    DROP TABLE IF EXISTS alert_events;
+    DROP TABLE IF EXISTS alert_rules;
+    DROP TABLE IF EXISTS prediction_records;
+    DROP TABLE IF EXISTS portfolio_lots;
+    DROP TABLE IF EXISTS portfolios;
+    DROP TABLE IF EXISTS watchlist_items;
+    DROP TABLE IF EXISTS watchlists;
+    DROP TABLE IF EXISTS instrument_aliases;
+    DROP TABLE IF EXISTS instruments;
     DROP TABLE IF EXISTS recommendations;
     DROP TABLE IF EXISTS workflow_runs;
     DROP TABLE IF EXISTS user_preferences;