@rohirik/openltm-core 2.8.0

package/src/janitor/index.ts

@@ -0,0 +1,215 @@
+/**
+ * janitor/index.ts — Janitor orchestrator.
+ * Runs inside the server.ts process, sharing the DB instance.
+ * Coordinates decay, promote, dedup, and embedding generation.
+ */
+import { getSetting, setSetting, getDb } from "../shared-db.js";
+import { runDecay, type DecayResult } from "./decay.js";
+import { findDuplicates, saveDedupCandidates, type DedupResult } from "./dedup.js";
+import { embedMissingMemories } from "./embeddings.js";
+import { runPromote, type PromoteResult } from "./promote.js";
+import { runArchive, type ArchiveResult } from "./archive.js";
+import { SETTING_KEYS, getDefault } from "./providers/types.js";
+
+export interface JanitorStatus {
+  /** Whether the janitor is currently running. */
+  running: boolean;
+  /** Last run timestamp (ISO string) or null if never run. */
+  lastRun: string | null;
+  /** Result of the last run. */
+  lastResult: JanitorRunResult | null;
+  /** Auto-run interval in minutes (0 = disabled). */
+  intervalMinutes: number;
+  /** Next scheduled run (ISO string) or null if auto-run disabled. */
+  nextRun: string | null;
+}
+
+export interface JanitorRunResult {
+  timestamp: string;
+  /** Duration in milliseconds. */
+  durationMs: number;
+  embed: { embedded: number };
+  decay: DecayResult;
+  archive: ArchiveResult;
+  promote: PromoteResult;
+  dedup: { pairsCompared: number; candidatesFound: number };
+  errors: string[];
+}
+
+let _running = false;
+let _lastRun: string | null = null;
+let _lastResult: JanitorRunResult | null = null;
+let _interval: ReturnType<typeof setInterval> | null = null;
+
+/**
+ * Run all janitor tasks in sequence:
+ * 1. Embed missing memories (required for dedup)
+ * 2. Decay stale memories
+ * 3. Promote eligible context_items
+ * 4. Find duplicates (no auto-merge without LLM verification)
+ */
+export async function runJanitor(): Promise<JanitorRunResult> {
+  if (_running) {
+    throw new Error("Janitor is already running");
+  }
+
+  _running = true;
+  const startTime = Date.now();
+  const errors: string[] = [];
+
+  let embedCount = 0;
+  let decayResult: DecayResult = { refreshed: 0, deprecated: 0 };
+  let archiveResult: ArchiveResult = { archived: 0 };
+  let promoteResult: PromoteResult = {
+    promoted: 0,
+    skipped: 0,
+    scanned: 0,
+  };
+  let dedupResult: DedupResult = {
+    pairsCompared: 0,
+    candidates: [],
+    autoMerged: 0,
+  };
+
+  try {
+    // 1. Embed missing memories
+    try {
+      embedCount = await embedMissingMemories();
+    } catch (e) {
+      errors.push(`embed: ${String(e)}`);
+    }
+
+    // 2. Decay stale memories (batch SQL refresh + deprecation)
+    try {
+      decayResult = runDecay();
+    } catch (e) {
+      errors.push(`decay: ${String(e)}`);
+    }
+
+    // 3. Archive evicted deprecated memories
+    try {
+      archiveResult = runArchive();
+    } catch (e) {
+      errors.push(`archive: ${String(e)}`);
+    }
+
+    // 4. Promote context_items
+    try {
+      promoteResult = runPromote();
+    } catch (e) {
+      errors.push(`promote: ${String(e)}`);
+    }
+
+    // 5. Find duplicates and save as pending for review
+    try {
+      dedupResult = await findDuplicates(0.85, true);
+      if (dedupResult.candidates.length > 0) {
+        saveDedupCandidates(dedupResult.candidates);
+      }
+    } catch (e) {
+      errors.push(`dedup: ${String(e)}`);
+    }
+  } finally {
+    _running = false;
+  }
+
+  const result: JanitorRunResult = {
+    timestamp: new Date().toISOString(),
+    durationMs: Date.now() - startTime,
+    embed: { embedded: embedCount },
+    decay: decayResult,
+    archive: archiveResult,
+    promote: promoteResult,
+    dedup: {
+      pairsCompared: dedupResult.pairsCompared,
+      candidatesFound: dedupResult.candidates.length,
+    },
+    errors,
+  };
+
+  // Persist run stats to settings for /openltm:health
+  await Promise.all([
+    setSetting(SETTING_KEYS.JANITOR_LAST_RUN_AT,          result.timestamp),
+    setSetting(SETTING_KEYS.JANITOR_LAST_DECAY_REFRESHED, String(decayResult.refreshed)),
+    setSetting(SETTING_KEYS.JANITOR_LAST_DEPRECATED,      String(decayResult.deprecated)),
+    setSetting(SETTING_KEYS.JANITOR_LAST_ARCHIVED,        String(archiveResult.archived)),
+  ]);
+
+  // WAL hygiene + query planner refresh
+  try {
+    const db = getDb();
+    db.exec("PRAGMA wal_checkpoint(TRUNCATE)");
+    db.exec("PRAGMA analysis_limit=400");
+    db.exec("ANALYZE");
+  } catch { /* non-fatal — next run will retry */ }
+
+  _lastRun = result.timestamp;
+  _lastResult = result;
+
+  return result;
+}
+
+/** Get current janitor status. */
+export function getJanitorStatus(): JanitorStatus {
+  const intervalMinutes = Number.parseInt(
+    getSetting(SETTING_KEYS.JANITOR_INTERVAL_MINUTES) ||
+      getDefault(SETTING_KEYS.JANITOR_INTERVAL_MINUTES),
+    10,
+  );
+
+  let nextRun: string | null = null;
+  if (intervalMinutes > 0 && _lastRun) {
+    const lastRunTime = new Date(_lastRun).getTime();
+    const nextRunTime = lastRunTime + intervalMinutes * 60 * 1000;
+    nextRun = new Date(nextRunTime).toISOString();
+  }
+
+  return {
+    running: _running,
+    lastRun: _lastRun,
+    lastResult: _lastResult,
+    intervalMinutes,
+    nextRun,
+  };
+}
+
+/**
+ * Start auto-run interval. Called by server.ts on startup if interval > 0.
+ * Safe to call multiple times — clears any existing interval first.
+ */
+export function startAutoRun(): void {
+  stopAutoRun();
+
+  const intervalMinutes = Number.parseInt(
+    getSetting(SETTING_KEYS.JANITOR_INTERVAL_MINUTES) ||
+      getDefault(SETTING_KEYS.JANITOR_INTERVAL_MINUTES),
+    10,
+  );
+
+  if (intervalMinutes <= 0) return;
+
+  const intervalMs = intervalMinutes * 60 * 1000;
+  _interval = setInterval(async () => {
+    try {
+      await runJanitor();
+    } catch {
+      // Logged in result.errors — don't crash the interval
+    }
+  }, intervalMs);
+}
+
+/** Stop auto-run interval. */
+export function stopAutoRun(): void {
+  if (_interval) {
+    clearInterval(_interval);
+    _interval = null;
+  }
+}
+
+// Re-export sub-modules for direct access from server routes
+export { approveMemory, getPendingMemories, rejectMemory } from "./promote.js";
+export { mergeMemories, parseDedupSource } from "./dedup.js";
+export { supersede } from "./supersedes.js";
+export { touchMemory } from "./decay.js";
+export { getEmbeddingProvider, semanticSearch, findSimilarMemories } from "./embeddings.js";
+export { runArchive } from "./archive.js";

package/src/janitor/promote.ts

@@ -0,0 +1,188 @@
+/**
+ * promote.ts — Auto-promote context_items (decisions/gotchas) to pending memories.
+ * Unlike the existing context.ts promote() which creates active memories immediately,
+ * this creates them as 'pending' for review in the approval UI.
+ */
+import { getDb, getSetting } from "../shared-db.js";
+import { normalizeKey } from "../dedup.js";
+import { SETTING_KEYS, getDefault } from "./providers/types.js";
+
+export interface PromoteResult {
+  /** Number of context_items promoted to pending memories. */
+  promoted: number;
+  /** Number of context_items skipped (already promoted or duplicate). */
+  skipped: number;
+  /** Total eligible items scanned. */
+  scanned: number;
+}
+
+/**
+ * Scan context_items for decisions/gotchas that should be promoted to memories.
+ *
+ * Criteria:
+ * 1. Type is 'decision' or 'gotcha' (not goals or progress)
+ * 2. Status is 'active' (not already pending_promotion or promoted)
+ * 3. Not already linked to a memory (memory_id IS NULL)
+ * 4. Content is substantial enough (> 20 chars)
+ *
+ * Creates memories with status='pending' for review/approval.
+ */
+export function runPromote(): PromoteResult {
+  const db = getDb();
+  const result: PromoteResult = { promoted: 0, skipped: 0, scanned: 0 };
+
+  const minImportance = Number.parseInt(
+    getSetting(SETTING_KEYS.PROMOTE_MIN_IMPORTANCE) ||
+      getDefault(SETTING_KEYS.PROMOTE_MIN_IMPORTANCE),
+    10,
+  );
+
+  // Find eligible context items
+  const items = db
+    .query<
+      {
+        id: number;
+        project_name: string;
+        type: string;
+        content: string;
+      },
+      []
+    >(
+      `SELECT id, project_name, type, content
+       FROM context_items
+       WHERE type IN ('decision', 'gotcha')
+         AND status = 'active'
+         AND memory_id IS NULL
+         AND LENGTH(content) > 20
+       ORDER BY id ASC`,
+    )
+    .all();
+
+  result.scanned = items.length;
+
+  for (const item of items) {
+    const dedupKey = normalizeKey(item.content);
+
+    // Check if a memory with this dedup_key already exists
+    const existing = db
+      .query<{ id: number }, [string]>(
+        "SELECT id FROM memories WHERE dedup_key = ?",
+      )
+      .get(dedupKey);
+
+    if (existing) {
+      // Link the context_item to the existing memory
+      db.run(
+        "UPDATE context_items SET memory_id = ?, status = 'promoted' WHERE id = ?",
+        [existing.id, item.id],
+      );
+      result.skipped++;
+      continue;
+    }
+
+    // Map context type to memory category
+    const category = item.type === "decision" ? "architecture" : "gotcha";
+    const importance = item.type === "gotcha" ? 4 : minImportance;
+
+    // Create a pending memory
+    const insertResult = db.run(
+      `INSERT INTO memories (content, category, importance, confidence, source, project_scope, dedup_key, status)
+       VALUES (?, ?, ?, 0.8, 'auto-promote', ?, ?, 'pending')`,
+      [item.content, category, importance, item.project_name, dedupKey],
+    );
+
+    const memoryId = Number(insertResult.lastInsertRowid);
+
+    // Update the context_item to link it and mark as pending_promotion
+    db.run(
+      "UPDATE context_items SET memory_id = ?, status = 'pending_promotion' WHERE id = ?",
+      [memoryId, item.id],
+    );
+
+    result.promoted++;
+  }
+
+  return result;
+}
+
+/**
+ * Approve a pending memory — set status to 'active'.
+ * Also updates the linked context_item status to 'promoted'.
+ */
+export function approveMemory(memoryId: number): boolean {
+  const db = getDb();
+  const mem = db
+    .query<{ id: number; status: string }, [number]>(
+      "SELECT id, status FROM memories WHERE id = ?",
+    )
+    .get(memoryId);
+
+  if (!mem || mem.status !== "pending") return false;
+
+  db.run("UPDATE memories SET status = 'active' WHERE id = ?", [memoryId]);
+  db.run(
+    "UPDATE context_items SET status = 'promoted' WHERE memory_id = ?",
+    [memoryId],
+  );
+
+  return true;
+}
+
+/**
+ * Reject a pending memory — delete it and reset the context_item.
+ */
+export function rejectMemory(memoryId: number): boolean {
+  const db = getDb();
+  const mem = db
+    .query<{ id: number; status: string }, [number]>(
+      "SELECT id, status FROM memories WHERE id = ?",
+    )
+    .get(memoryId);
+
+  if (!mem || mem.status !== "pending") return false;
+
+  // Reset linked context_items back to active
+  db.run(
+    "UPDATE context_items SET memory_id = NULL, status = 'active' WHERE memory_id = ?",
+    [memoryId],
+  );
+
+  // Delete the pending memory
+  db.run("DELETE FROM memories WHERE id = ?", [memoryId]);
+
+  return true;
+}
+
+/**
+ * Get all pending memories for the review UI.
+ */
+export function getPendingMemories(): Array<{
+  id: number;
+  content: string;
+  category: string;
+  importance: number;
+  confidence: number;
+  project_scope: string | null;
+  source: string | null;
+  created_at: string;
+}> {
+  const db = getDb();
+  return db
+    .query<
+      {
+        id: number;
+        content: string;
+        category: string;
+        importance: number;
+        confidence: number;
+        project_scope: string | null;
+        source: string | null;
+        created_at: string;
+      },
+      []
+    >(
+      `SELECT id, content, category, importance, confidence, project_scope, source, created_at
+       FROM memories WHERE status = 'pending' ORDER BY created_at DESC`,
+    )
+    .all();
+}

package/src/janitor/providers/anthropic.ts

@@ -0,0 +1,91 @@
+/**
+ * anthropic.ts — Anthropic Claude provider for LLM chat (no embedding API).
+ */
+import {
+  SETTING_KEYS,
+  type ChatInput,
+  type ChatResult,
+  type LLMProvider,
+} from "./types.js";
+import { httpErrorResult, makeApiKeyGetter, makeModelGetter } from "./utils.js";
+
+const ANTHROPIC_API_BASE = "https://api.anthropic.com/v1";
+const ANTHROPIC_VERSION = "2023-06-01";
+
+const getApiKey = makeApiKeyGetter(SETTING_KEYS.ANTHROPIC_API_KEY, "ANTHROPIC_API_KEY", "Anthropic");
+const getLlmModel = makeModelGetter(SETTING_KEYS.ANTHROPIC_LLM_MODEL);
+
+function authHeaders(apiKey: string) {
+  return {
+    "Content-Type": "application/json",
+    "x-api-key": apiKey,
+    "anthropic-version": ANTHROPIC_VERSION,
+  };
+}
+
+export const anthropicLLM: LLMProvider = {
+  name: "anthropic",
+
+  async chat(input: ChatInput): Promise<ChatResult> {
+    const apiKey = getApiKey();
+    const model = getLlmModel();
+
+    // Anthropic uses a single `system` string; only the first system message is used.
+    const systemMsg = input.messages.find((m) => m.role === "system");
+    const messages = input.messages
+      .filter((m) => m.role !== "system")
+      .map((m) => ({ role: m.role as "user" | "assistant", content: m.content }));
+
+    const body: Record<string, unknown> = {
+      model,
+      messages,
+      max_tokens: input.maxTokens ?? 1024,
+      temperature: input.temperature ?? 0.1,
+    };
+    if (systemMsg) body.system = systemMsg.content;
+
+    const res = await fetch(`${ANTHROPIC_API_BASE}/messages`, {
+      method: "POST",
+      headers: authHeaders(apiKey),
+      body: JSON.stringify(body),
+    });
+
+    if (!res.ok) {
+      const { error } = await httpErrorResult(res);
+      throw new Error(`Anthropic chat failed: ${error}`);
+    }
+
+    const data = (await res.json()) as {
+      content: Array<{ type: string; text: string }>;
+      model: string;
+      usage: { input_tokens: number; output_tokens: number };
+    };
+
+    return {
+      content: data.content.find((b) => b.type === "text")?.text ?? "",
+      model: data.model,
+      promptTokens: data.usage.input_tokens,
+      completionTokens: data.usage.output_tokens,
+    };
+  },
+
+  async verify(): Promise<{ ok: boolean; error?: string }> {
+    try {
+      const apiKey = getApiKey();
+      const model = getLlmModel();
+      const res = await fetch(`${ANTHROPIC_API_BASE}/messages`, {
+        method: "POST",
+        headers: authHeaders(apiKey),
+        body: JSON.stringify({
+          model,
+          messages: [{ role: "user", content: "Reply with OK" }],
+          max_tokens: 5,
+        }),
+      });
+      if (!res.ok) return httpErrorResult(res);
+      return { ok: true };
+    } catch (e) {
+      return { ok: false, error: String(e) };
+    }
+  },
+};

package/src/janitor/providers/cohere.ts

@@ -0,0 +1,135 @@
+/**
+ * cohere.ts — Cohere provider for embeddings and LLM chat (v2 API).
+ */
+import {
+  SETTING_KEYS,
+  type ChatInput,
+  type ChatResult,
+  type EmbedInput,
+  type EmbedResult,
+  type EmbeddingProvider,
+  type EmbeddingVector,
+  type LLMProvider,
+} from "./types.js";
+import { httpErrorResult, makeApiKeyGetter, makeModelGetter } from "./utils.js";
+
+const COHERE_API_BASE = "https://api.cohere.com/v2";
+
+const getApiKey = makeApiKeyGetter(SETTING_KEYS.COHERE_API_KEY, "COHERE_API_KEY", "Cohere");
+const getEmbedModel = makeModelGetter(SETTING_KEYS.COHERE_EMBED_MODEL);
+const getLlmModel = makeModelGetter(SETTING_KEYS.COHERE_LLM_MODEL);
+
+function authHeaders(apiKey: string) {
+  return { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}` };
+}
+
+export const cohereEmbedding: EmbeddingProvider = {
+  name: "cohere",
+
+  async embed(input: EmbedInput): Promise<EmbedResult> {
+    const apiKey = getApiKey();
+    const model = getEmbedModel();
+
+    const res = await fetch(`${COHERE_API_BASE}/embed`, {
+      method: "POST",
+      headers: authHeaders(apiKey),
+      body: JSON.stringify({
+        model,
+        texts: input.texts,
+        input_type: "search_document",
+        embedding_types: ["float"],
+      }),
+    });
+
+    if (!res.ok) {
+      const { error } = await httpErrorResult(res);
+      throw new Error(`Cohere embed failed: ${error}`);
+    }
+
+    const data = (await res.json()) as {
+      embeddings: { float: number[][] };
+      meta?: { billed_units?: { input_tokens?: number } };
+    };
+
+    const vectors: EmbeddingVector[] = data.embeddings.float.map(
+      (e) => new Float32Array(e),
+    );
+
+    return {
+      vectors,
+      model,
+      dimensions: vectors[0]?.length ?? 0,
+      totalTokens: data.meta?.billed_units?.input_tokens ?? 0,
+    };
+  },
+
+  async verify(): Promise<{ ok: boolean; error?: string }> {
+    try {
+      await this.embed({ texts: ["test"] });
+      return { ok: true };
+    } catch (e) {
+      return { ok: false, error: String(e) };
+    }
+  },
+};
+
+export const cohereLLM: LLMProvider = {
+  name: "cohere",
+
+  async chat(input: ChatInput): Promise<ChatResult> {
+    const apiKey = getApiKey();
+    const model = getLlmModel();
+
+    const res = await fetch(`${COHERE_API_BASE}/chat`, {
+      method: "POST",
+      headers: authHeaders(apiKey),
+      body: JSON.stringify({
+        model,
+        messages: input.messages.map((m) => ({
+          role: m.role === "assistant" ? "assistant" : m.role === "system" ? "system" : "user",
+          content: m.content,
+        })),
+        max_tokens: input.maxTokens ?? 1024,
+        temperature: input.temperature ?? 0.1,
+        ...(input.jsonMode ? { response_format: { type: "json_object" } } : {}),
+      }),
+    });
+
+    if (!res.ok) {
+      const { error } = await httpErrorResult(res);
+      throw new Error(`Cohere chat failed: ${error}`);
+    }
+
+    const data = (await res.json()) as {
+      message: { content: Array<{ type: string; text: string }> };
+      usage?: { billed_units?: { input_tokens?: number; output_tokens?: number } };
+    };
+
+    return {
+      content: data.message.content.find((b) => b.type === "text")?.text ?? "",
+      model,
+      promptTokens: data.usage?.billed_units?.input_tokens ?? 0,
+      completionTokens: data.usage?.billed_units?.output_tokens ?? 0,
+    };
+  },
+
+  async verify(): Promise<{ ok: boolean; error?: string }> {
+    try {
+      const apiKey = getApiKey();
+      const model = getLlmModel();
+      const res = await fetch(`${COHERE_API_BASE}/chat`, {
+        method: "POST",
+        headers: authHeaders(apiKey),
+        body: JSON.stringify({
+          model,
+          messages: [{ role: "user", content: "Reply with OK" }],
+          max_tokens: 5,
+        }),
+      });
+      if (!res.ok) return httpErrorResult(res);
+      return { ok: true };
+    } catch (e) {
+      return { ok: false, error: String(e) };
+    }
+  },
+};