@rohirik/openltm-core 2.8.0

package/src/migrations.ts

@@ -0,0 +1,415 @@
+#!/usr/bin/env bun
+/**
+ * migrations.ts — Versioned schema migration runner for openltm.db
+ * Uses bun:sqlite and Bun file APIs exclusively.
+ *
+ * CLI: bun migrations.ts [--status | --up | --down | --reset]
+ */
+import { Database } from "bun:sqlite";
+import { readdirSync, existsSync, statSync, unlinkSync } from "fs";
+import { join, dirname } from "path";
+import { createHash } from "crypto";
+import { getDbPath, getMigrationsDir } from "./paths.js";
+
+const DB_PATH = getDbPath();
+const MIGRATIONS_DIR = getMigrationsDir();
+
+function openDb(): Database {
+  const db = new Database(DB_PATH, { create: true });
+  db.exec("PRAGMA journal_mode=WAL; PRAGMA foreign_keys=ON;");
+  return db;
+}
+
+// ── Helpers ────────────────────────────────────────────────────────────────────
+
+export function ensureMigrationsTable(db: Database): void {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS _schema_version (
+      id          INTEGER PRIMARY KEY AUTOINCREMENT,
+      version     INTEGER NOT NULL UNIQUE,
+      name        TEXT NOT NULL,
+      applied_at  TEXT NOT NULL DEFAULT (datetime('now')),
+      checksum    TEXT NOT NULL
+    )
+  `);
+}
+
+export interface MigrationFile {
+  version: number;
+  name: string;
+  path: string;
+  content: string;
+}
+
+export async function getMigrationFiles(): Promise<MigrationFile[]> {
+  if (!existsSync(MIGRATIONS_DIR)) return [];
+
+  const files = readdirSync(MIGRATIONS_DIR)
+    .filter((f) => f.endsWith(".sql"))
+    .sort();
+
+  const results: MigrationFile[] = [];
+  for (const file of files) {
+    const match = file.match(/^(\d+)_(.+)\.sql$/);
+    if (!match) throw new Error(`Migration filename must match NNN_name.sql: ${file}`);
+    const version = parseInt(match[1]!, 10);
+    const name = match[2]!;
+    const filePath = join(MIGRATIONS_DIR, file);
+    const content = await Bun.file(filePath).text();
+    results.push({ version, name, path: filePath, content });
+  }
+  return results;
+}
+
+export function getAppliedVersions(db: Database): Set<number> {
+  ensureMigrationsTable(db);
+  const rows = db
+    .query<{ version: number }, []>("SELECT version FROM _schema_version ORDER BY version")
+    .all();
+  return new Set(rows.map((r) => r.version));
+}
+
+export function computeChecksum(content: string): string {
+  return createHash("sha256").update(content).digest("hex");
+}
+
+export async function backupDb(): Promise<string> {
+  const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+  const backupPath = `${DB_PATH}.bak-${timestamp}`;
+  // Skip backup when the source DB doesn't exist (e.g. in-process test DBs
+  // that were created on a different path than the module-level DB_PATH).
+  if (!existsSync(DB_PATH)) return backupPath;
+  const bytes = await Bun.file(DB_PATH).arrayBuffer();
+  await Bun.write(backupPath, bytes);
+  return backupPath;
+}
+
+/**
+ * Read the default backup retention count from `LTM_BACKUP_RETENTION` env var.
+ * Falls back to 1. Negative or non-numeric values also default to 1.
+ */
+export function getRetentionDefault(): number {
+  const env = process.env["LTM_BACKUP_RETENTION"];
+  if (!env) return 1;
+  const n = parseInt(env, 10);
+  if (Number.isNaN(n) || n < 0) return 1;
+  return n;
+}
+
+export interface EnforceRetentionOptions {
+  /** Override the db path (defaults to getDbPath() result). */
+  dbPath?: string;
+  /** Path of a .bak file just created by backupDb() — never delete it. */
+  currentBackupPath?: string;
+  /** Skip files whose mtime is newer than this many ms (default 60_000 = 60s). */
+  gracePeriodMs?: number;
+}
+
+export interface EnforceRetentionResult {
+  /** Absolute paths of files successfully deleted. */
+  deleted: string[];
+  /** Absolute paths of files that remain (within the retention window). */
+  kept: string[];
+  /** Non-fatal issues encountered (e.g. stat/unlink permission errors). */
+  warnings: string[];
+}
+
+/**
+ * Enforce a maximum number of .bak files alongside the database.
+ *
+ * Filenames are `openltm.db.bak-<ISO-timestamp>`. ISO timestamps sort lexically,
+ * so sorting by filename = sorting by creation time. We keep the newest N
+ * and delete the rest. Files newer than `gracePeriodMs` (default 60s) are
+ * skipped to avoid racing with a concurrent writer.
+ *
+ * The function is idempotent and never throws — every error becomes a
+ * warning in the result.
+ */
+export async function enforceRetention(
+  maxBackups: number,
+  options: EnforceRetentionOptions = {},
+): Promise<EnforceRetentionResult> {
+  const dbPath = options.dbPath ?? getDbPath();
+  const gracePeriodMs = options.gracePeriodMs ?? 60_000;
+  const result: EnforceRetentionResult = { deleted: [], kept: [], warnings: [] };
+
+  const dir = dirname(dbPath);
+  if (!existsSync(dir)) {
+    result.warnings.push(`Directory does not exist: ${dir}`);
+    return result;
+  }
+
+  const files = readdirSync(dir).filter((f) => f.startsWith("openltm.db.bak-"));
+  if (files.length === 0) return result;
+
+  const now = Date.now();
+  const eligible: string[] = [];
+  const excluded: string[] = [];
+  for (const f of files) {
+    const fullPath = join(dir, f);
+    try {
+      const mtime = statSync(fullPath).mtimeMs;
+      if (now - mtime < gracePeriodMs) {
+        excluded.push(fullPath);
+        continue;
+      }
+    } catch (err: unknown) {
+      const code = (err as NodeJS.ErrnoException).code;
+      if (code === "ENOENT") continue;
+      result.warnings.push(`stat failed for ${fullPath}: ${(err as Error).message}`);
+      continue;
+    }
+    if (options.currentBackupPath && fullPath === options.currentBackupPath) {
+      excluded.push(fullPath);
+      continue;
+    }
+    eligible.push(fullPath);
+  }
+
+  if (eligible.length === 0) {
+    result.kept = excluded;
+    return result;
+  }
+
+  eligible.sort();
+
+  const toDelete = eligible.slice(0, Math.max(0, eligible.length - maxBackups));
+  const toKeep = eligible.slice(toDelete.length);
+
+  for (const file of toDelete) {
+    try {
+      unlinkSync(file);
+      result.deleted.push(file);
+    } catch (err: unknown) {
+      const code = (err as NodeJS.ErrnoException).code;
+      if (code === "ENOENT") {
+        result.deleted.push(file);
+      } else {
+        result.warnings.push(`unlink failed for ${file}: ${(err as Error).message}`);
+      }
+    }
+  }
+
+  result.kept = [...excluded, ...toKeep];
+  return result;
+}
+
+export interface ParsedMigration {
+  up: string;
+  down: string;
+}
+
+export function parseMigration(content: string): ParsedMigration {
+  const parts = content.split(/^--\s*DOWN\s*$/m);
+  const up = (parts[0] ?? "")
+    .replace(/^--.*$/gm, "")
+    .trim();
+  const down = (parts[1] ?? "")
+    .replace(/^--.*$/gm, "")
+    .trim();
+  return { up, down };
+}
+
+// ── Core migration actions ─────────────────────────────────────────────────────
+
+export interface MigrationResult {
+  version: number;
+  name: string;
+  action: "applied" | "skipped" | "rolled_back";
+}
+
+export async function runPendingMigrations(db?: Database): Promise<MigrationResult[]> {
+  const _db = db ?? openDb();
+  ensureMigrationsTable(_db);
+
+  const files = await getMigrationFiles();
+  const applied = getAppliedVersions(_db);
+  const pending = files.filter((f) => !applied.has(f.version));
+
+  if (pending.length === 0) return [];
+
+  const newBackupPath = await backupDb();
+
+  const results: MigrationResult[] = [];
+  for (const file of pending) {
+    const { up } = parseMigration(file.content);
+    const checksum = computeChecksum(file.content);
+
+    try {
+      _db.transaction(() => {
+        // Skip exec for marker migrations that have no DDL (e.g. 001_baseline).
+        if (up) _db.exec(up);
+        _db.run(
+          `INSERT INTO _schema_version (version, name, checksum) VALUES (?, ?, ?)`,
+          [file.version, file.name, checksum],
+        );
+      })();
+    } catch (err: unknown) {
+      const msg = (err as { message?: string }).message ?? "";
+      // Column was already added by shared-db.ts runMigrations() — self-heal by recording as applied
+      if (msg.includes("duplicate column name") || msg.includes("already exists")) {
+        _db.run(
+          `INSERT OR IGNORE INTO _schema_version (version, name, checksum) VALUES (?, ?, ?)`,
+          [file.version, file.name, checksum],
+        );
+      } else {
+        throw err;
+      }
+    }
+
+    results.push({ version: file.version, name: file.name, action: "applied" });
+  }
+
+  try {
+    await enforceRetention(getRetentionDefault(), { currentBackupPath: newBackupPath });
+  } catch (err: unknown) {
+    console.warn(`[migrations] retention enforcement failed: ${(err as Error).message}`);
+  }
+
+  return results;
+}
+
+export interface MigrationStatus {
+  version: number;
+  name: string;
+  status: "applied" | "pending";
+  applied_at?: string;
+}
+
+export async function getMigrationStatus(db?: Database): Promise<MigrationStatus[]> {
+  const _db = db ?? openDb();
+  ensureMigrationsTable(_db);
+
+  const files = await getMigrationFiles();
+  const rows = _db
+    .query<{ version: number; applied_at: string }, []>(
+      "SELECT version, applied_at FROM _schema_version ORDER BY version",
+    )
+    .all();
+  const appliedMap = new Map(rows.map((r) => [r.version, r.applied_at]));
+
+  return files.map((f) => {
+    const applied_at = appliedMap.get(f.version);
+    return {
+      version: f.version,
+      name: f.name,
+      status: (applied_at ? "applied" : "pending") as "applied" | "pending",
+      ...(applied_at ? { applied_at } : {}),
+    };
+  });
+}
+
+export async function rollbackLast(db?: Database): Promise<MigrationResult | null> {
+  const _db = db ?? openDb();
+  ensureMigrationsTable(_db);
+
+  const last = _db
+    .query<{ version: number; name: string }, []>(
+      "SELECT version, name FROM _schema_version ORDER BY version DESC LIMIT 1",
+    )
+    .get();
+
+  if (!last) return null;
+
+  const files = await getMigrationFiles();
+  const file = files.find((f) => f.version === last.version);
+  if (!file) throw new Error(`Migration file for version ${last.version} not found`);
+
+  const { down } = parseMigration(file.content);
+  if (!down) throw new Error(`Migration ${last.version} has no DOWN section`);
+
+  _db.transaction(() => {
+    _db.exec(down);
+    _db.run("DELETE FROM _schema_version WHERE version = ?", [last.version]);
+  })();
+
+  return { version: last.version, name: last.name, action: "rolled_back" };
+}
+
+export async function resetAll(db?: Database): Promise<MigrationResult[]> {
+  const _db = db ?? openDb();
+  ensureMigrationsTable(_db);
+
+  // Read files once, then roll back in reverse order without repeated disk reads
+  const files = await getMigrationFiles();
+  const fileMap = new Map(files.map((f) => [f.version, f]));
+
+  const applied = _db
+    .query<{ version: number; name: string }, []>(
+      "SELECT version, name FROM _schema_version ORDER BY version DESC",
+    )
+    .all();
+
+  const results: MigrationResult[] = [];
+  for (const row of applied) {
+    const file = fileMap.get(row.version);
+    if (!file) throw new Error(`Migration file for version ${row.version} not found`);
+    const { down } = parseMigration(file.content);
+    if (!down) throw new Error(`Migration ${row.version} has no DOWN section`);
+    _db.transaction(() => {
+      _db.exec(down);
+      _db.run("DELETE FROM _schema_version WHERE version = ?", [row.version]);
+    })();
+    results.push({ version: row.version, name: row.name, action: "rolled_back" });
+  }
+  return results;
+}
+
+// ── CLI entry point ────────────────────────────────────────────────────────────
+
+if (import.meta.main) {
+  void (async () => {
+    const arg = process.argv[2] ?? "--status";
+    const db = openDb();
+
+    try {
+      if (arg === "--status") {
+        const statuses = await getMigrationStatus(db);
+        if (statuses.length === 0) {
+          console.error("No migration files found in migrations/");
+        } else {
+          console.error("\nMigration Status:");
+          console.error("─".repeat(60));
+          for (const s of statuses) {
+            const tag = s.status === "applied" ? "[applied]" : "[pending]";
+            const date = s.applied_at ? `  (${s.applied_at})` : "";
+            console.error(`  ${tag.padEnd(10)} v${String(s.version).padStart(3, "0")} ${s.name}${date}`);
+          }
+          console.error("");
+        }
+      } else if (arg === "--up") {
+        const results = await runPendingMigrations(db);
+        if (results.length === 0) {
+          console.error("No pending migrations.");
+        } else {
+          for (const r of results) {
+            console.error(`Applied: v${String(r.version).padStart(3, "0")} ${r.name}`);
+          }
+        }
+      } else if (arg === "--down") {
+        const result = await rollbackLast(db);
+        if (!result) {
+          console.error("No applied migrations to roll back.");
+        } else {
+          console.error(`Rolled back: v${String(result.version).padStart(3, "0")} ${result.name}`);
+        }
+      } else if (arg === "--reset") {
+        const results = await resetAll(db);
+        if (results.length === 0) {
+          console.error("Nothing to reset.");
+        } else {
+          for (const r of results) {
+            console.error(`Rolled back: v${String(r.version).padStart(3, "0")} ${r.name}`);
+          }
+        }
+      } else {
+        console.error(`Unknown argument: ${arg}`);
+        console.error("Usage: bun migrations.ts [--status | --up | --down | --reset]");
+        process.exit(1);
+      }
+    } catch (err) {
+      console.error("Migration error:", err);
+      process.exit(1);
+    }
+  })();
+}

package/src/paths.ts

@@ -0,0 +1,22 @@
+/**
+ * paths.ts (openltm-core) — Path resolution without any host-specific defaults.
+ * Priority: LTM_DB_PATH env var > CLAUDE_PLUGIN_DATA env var > dev fallback.
+ * The CLAUDE_DIR constant is intentionally absent — adapters inject paths via LtmCoreConfig.
+ */
+import { join } from "path";
+
+export function getDbPath(): string {
+  if (process.env["LTM_DB_PATH"]) return process.env["LTM_DB_PATH"];
+  if (process.env["CLAUDE_PLUGIN_DATA"]) return join(process.env["CLAUDE_PLUGIN_DATA"], "openltm.db");
+  return join(import.meta.dir, "..", "..", "..", "data", "openltm.db");
+}
+
+export function getSchemaPath(): string {
+  return join(import.meta.dir, "schema.sql");
+}
+
+export function getMigrationsDir(): string {
+  // Resolves to project-root migrations/ during development inside the monorepo.
+  // Adapters should set LtmCoreConfig.migrationsDir when deploying standalone.
+  return join(import.meta.dir, "..", "..", "..", "migrations");
+}

package/src/proposals.ts

@@ -0,0 +1,120 @@
+/**
+ * proposals.ts — Review interface for memory proposals written by EvaluateSession.
+ *
+ * Proposals live as JSON files in ${CLAUDE_PLUGIN_DATA}/proposals/<session-id>.json.
+ * Each file: { proposals: MemoryProposal[], generatedAt: number }
+ */
+import { existsSync, readdirSync, readFileSync, writeFileSync, unlinkSync } from "fs";
+import { join } from "path";
+import { homedir } from "os";
+import { learn } from "./db.js";
+import type { MemoryCategory } from "./db.js";
+
+interface MemoryProposal {
+  content: string;
+  category: string;
+  importance: number;
+  source: string;
+}
+
+interface ProposalFile {
+  proposals: MemoryProposal[];
+  generatedAt: number;
+}
+
+export interface PendingProposal {
+  sessionId: string;
+  index: number;
+  content: string;
+  category: string;
+  importance: number;
+  source: string;
+  generatedAt: number;
+}
+
+function getProposalsDir(): string {
+  const base = process.env.CLAUDE_PLUGIN_DATA
+    ?? join(homedir(), ".claude", "plugins", "data", "OpenLtm-openltm");
+  return join(base, "proposals");
+}
+
+function readProposalFile(path: string): ProposalFile | null {
+  try {
+    const raw = readFileSync(path, "utf-8");
+    const parsed = JSON.parse(raw) as ProposalFile;
+    if (!Array.isArray(parsed.proposals)) return null;
+    return parsed;
+  } catch {
+    return null; // silent: malformed or missing file
+  }
+}
+
+export function listPendingProposals(): PendingProposal[] {
+  const dir = getProposalsDir();
+  if (!existsSync(dir)) return [];
+
+  const results: PendingProposal[] = [];
+  for (const file of readdirSync(dir)) {
+    if (!file.endsWith(".json")) continue;
+    const sessionId = file.replace(/\.json$/, "");
+    const data = readProposalFile(join(dir, file));
+    if (!data) continue;
+    for (let i = 0; i < data.proposals.length; i++) {
+      const p = data.proposals[i]!;
+      results.push({
+        sessionId,
+        index: i,
+        content: p.content,
+        category: p.category,
+        importance: p.importance,
+        source: p.source,
+        generatedAt: data.generatedAt,
+      });
+    }
+  }
+  return results.sort((a, b) => b.importance - a.importance || b.generatedAt - a.generatedAt);
+}
+
+export function acceptProposal(sessionId: string, index: number): boolean {
+  const dir = getProposalsDir();
+  const filePath = join(dir, `${sessionId}.json`);
+  const data = readProposalFile(filePath);
+  if (!data) return false;
+
+  const proposal = data.proposals[index];
+  if (!proposal) return false;
+
+  learn({
+    content: proposal.content,
+    category: proposal.category as MemoryCategory,
+    importance: proposal.importance,
+    source: proposal.source,
+    skipExport: true,
+  });
+
+  // Remove the accepted proposal from the file; delete file if empty
+  const remaining = data.proposals.filter((_, i) => i !== index);
+  if (remaining.length === 0) {
+    try { unlinkSync(filePath); } catch {}
+  } else {
+    writeFileSync(filePath, JSON.stringify({ ...data, proposals: remaining }, null, 2), "utf-8");
+  }
+  return true;
+}
+
+export function rejectProposal(sessionId: string, index: number): boolean {
+  const dir = getProposalsDir();
+  const filePath = join(dir, `${sessionId}.json`);
+  const data = readProposalFile(filePath);
+  if (!data) return false;
+
+  if (index < 0 || index >= data.proposals.length) return false;
+
+  const remaining = data.proposals.filter((_, i) => i !== index);
+  if (remaining.length === 0) {
+    try { unlinkSync(filePath); } catch {}
+  } else {
+    writeFileSync(filePath, JSON.stringify({ ...data, proposals: remaining }, null, 2), "utf-8");
+  }
+  return true;
+}

package/src/providers/disabled.ts

@@ -0,0 +1,19 @@
+/**
+ * providers/disabled.ts — No-op embedding provider.
+ * Default when no provider is configured. Zero I/O, zero API key required.
+ */
+import type { EmbeddingProvider } from "./embeddingProvider.js";
+
+export class DisabledProvider implements EmbeddingProvider {
+  readonly name = "disabled" as const;
+  readonly model = "none";
+  readonly dim = 0;
+
+  available(): Promise<boolean> {
+    return Promise.resolve(false);
+  }
+
+  generate(_text: string): Promise<Float32Array | null> {
+    return Promise.resolve(null);
+  }
+}

package/src/providers/embeddingProvider.ts

@@ -0,0 +1,49 @@
+/**
+ * providers/embeddingProvider.ts — Pluggable embedding provider interface.
+ * loadProvider() is the single factory used by learn() and recall().
+ * The "disabled" provider is the default — no API key required.
+ */
+
+export type EmbeddingProviderName = "gemini" | "openai" | "ollama" | "disabled";
+
+export interface EmbeddingConfig {
+  provider: EmbeddingProviderName;
+  apiKey?: string;
+  model?: string;
+  baseUrl?: string;
+  confidenceThreshold: number;
+}
+
+export const DEFAULT_EMBEDDING_CONFIG: EmbeddingConfig = {
+  provider: "disabled",
+  confidenceThreshold: 0.6,
+};
+
+export interface EmbeddingProvider {
+  readonly name: EmbeddingProviderName;
+  readonly model: string;
+  readonly dim: number;
+  available(): Promise<boolean>;
+  generate(text: string): Promise<Float32Array | null>;
+}
+
+/** Factory — returns the configured provider. Defaults to "disabled" if unrecognised. */
+export async function loadProvider(config?: Partial<EmbeddingConfig>): Promise<EmbeddingProvider> {
+  const providerName = config?.provider ?? "disabled";
+
+  if (providerName === "gemini") {
+    const { GeminiProvider } = await import("./gemini.js");
+    return new GeminiProvider(config ?? {});
+  }
+  if (providerName === "openai") {
+    const { OpenAIProvider } = await import("./openai.js");
+    return new OpenAIProvider(config ?? {});
+  }
+  if (providerName === "ollama") {
+    const { OllamaProvider } = await import("./ollama.js");
+    return new OllamaProvider(config ?? {});
+  }
+
+  const { DisabledProvider } = await import("./disabled.js");
+  return new DisabledProvider();
+}

package/src/providers/gemini.ts

@@ -0,0 +1,37 @@
+/**
+ * providers/gemini.ts — Gemini embedding provider (EmbeddingProvider interface).
+ * Filled in by T5. Stub here to satisfy tsc for T3.
+ */
+import type { EmbeddingProvider, EmbeddingProviderName, EmbeddingConfig } from "./embeddingProvider.js";
+
+export class GeminiProvider implements EmbeddingProvider {
+  readonly name: EmbeddingProviderName = "gemini";
+  readonly model: string;
+  readonly dim = 768;
+
+  private apiKey: string;
+
+  constructor(config: Partial<EmbeddingConfig>) {
+    this.model = config.model ?? "text-embedding-004";
+    this.apiKey = config.apiKey ?? process.env["GEMINI_API_KEY"] ?? "";
+  }
+
+  async available(): Promise<boolean> {
+    return this.apiKey.length > 0;
+  }
+
+  async generate(text: string): Promise<Float32Array | null> {
+    if (!this.apiKey) return null;
+    const url = `https://generativelanguage.googleapis.com/v1beta/models/${this.model}:embedContent?key=${this.apiKey}`;
+    const res = await fetch(url, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ model: `models/${this.model}`, content: { parts: [{ text }] } }),
+    });
+    if (!res.ok) return null;
+    const json = await res.json() as { embedding?: { values?: number[] } };
+    const values = json?.embedding?.values;
+    if (!values) return null;
+    return new Float32Array(values);
+  }
+}

package/src/providers/index.ts

@@ -0,0 +1,2 @@
+export * from "./embeddingProvider.js";
+export * from "./disabled.js";