@rohirik/openltm-core 2.8.0

package/src/providers/ollama.ts

@@ -0,0 +1,43 @@
+/**
+ * providers/ollama.ts — Ollama embedding provider (EmbeddingProvider interface).
+ */
+import type { EmbeddingProvider, EmbeddingProviderName, EmbeddingConfig } from "./embeddingProvider.js";
+
+export class OllamaProvider implements EmbeddingProvider {
+  readonly name: EmbeddingProviderName = "ollama";
+  readonly model: string;
+  readonly dim = 768;
+
+  private baseUrl: string;
+
+  constructor(config: Partial<EmbeddingConfig>) {
+    this.model = config.model ?? "nomic-embed-text";
+    this.baseUrl = config.baseUrl ?? process.env["OLLAMA_BASE_URL"] ?? "http://localhost:11434";
+  }
+
+  async available(): Promise<boolean> {
+    try {
+      const res = await fetch(`${this.baseUrl}/api/tags`, { signal: AbortSignal.timeout(2000) });
+      return res.ok;
+    } catch {
+      return false;
+    }
+  }
+
+  async generate(text: string): Promise<Float32Array | null> {
+    try {
+      const res = await fetch(`${this.baseUrl}/api/embed`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ model: this.model, input: text }),
+      });
+      if (!res.ok) return null;
+      const json = await res.json() as { embeddings?: number[][] };
+      const values = json?.embeddings?.[0];
+      if (!values) return null;
+      return new Float32Array(values);
+    } catch {
+      return null;
+    }
+  }
+}

package/src/providers/openai.ts

@@ -0,0 +1,35 @@
+/**
+ * providers/openai.ts — OpenAI embedding provider (EmbeddingProvider interface).
+ */
+import type { EmbeddingProvider, EmbeddingProviderName, EmbeddingConfig } from "./embeddingProvider.js";
+
+export class OpenAIProvider implements EmbeddingProvider {
+  readonly name: EmbeddingProviderName = "openai";
+  readonly model: string;
+  readonly dim = 1536;
+
+  private apiKey: string;
+
+  constructor(config: Partial<EmbeddingConfig>) {
+    this.model = config.model ?? "text-embedding-3-small";
+    this.apiKey = config.apiKey ?? process.env["OPENAI_API_KEY"] ?? "";
+  }
+
+  async available(): Promise<boolean> {
+    return this.apiKey.length > 0;
+  }
+
+  async generate(text: string): Promise<Float32Array | null> {
+    if (!this.apiKey) return null;
+    const res = await fetch("https://api.openai.com/v1/embeddings", {
+      method: "POST",
+      headers: { "Content-Type": "application/json", Authorization: `Bearer ${this.apiKey}` },
+      body: JSON.stringify({ model: this.model, input: text }),
+    });
+    if (!res.ok) return null;
+    const json = await res.json() as { data?: { embedding?: number[] }[] };
+    const values = json?.data?.[0]?.embedding;
+    if (!values) return null;
+    return new Float32Array(values);
+  }
+}

package/src/queue/index.ts

@@ -0,0 +1,53 @@
+/**
+ * queue/index.ts — Honker durable queue for embedding generation.
+ *
+ * Replaces the fire-and-forget embedMemory() call with a durable job: learn()
+ * enqueues { memoryId }, a long-lived worker (queue/worker.ts) claims, embeds,
+ * and acks with retry/backoff + dead-letter. Every export degrades to a no-op
+ * (null / false) when Honker is unavailable; callers keep the inline path.
+ */
+import type { HonkerEnqueueOptions, HonkerQueue, HonkerTransaction } from "../lib/honkerTypes.js";
+import { getHonker } from "../lib/honker.js";
+
+/** Durable queue name for embedding jobs. */
+export const EMBED_QUEUE = "ltm-embeddings";
+
+/** Retries before a job is dead-lettered (honker_retry moves to dead at maxAttempts). */
+export const EMBED_MAX_ATTEMPTS = 5;
+
+/** Payload shape for an embedding job. */
+export interface EmbeddingJob {
+  memoryId: number;
+}
+
+/** The embedding Queue handle, or null when Honker is unavailable. */
+export function getEmbeddingQueue(): HonkerQueue | null {
+  const h = getHonker();
+  return h ? h.queue(EMBED_QUEUE, { maxAttempts: EMBED_MAX_ATTEMPTS }) : null;
+}
+
+/**
+ * Enqueue an embedding job for a memory. Returns the job id, or null when the
+ * queue is unavailable (caller should embed inline instead). Pass `tx` to make
+ * the enqueue atomic with the memory INSERT — the job only becomes visible on
+ * commit and is dropped on rollback.
+ */
+export function enqueueEmbedding(memoryId: number, opts?: { tx?: HonkerTransaction }): number | null {
+  const q = getEmbeddingQueue();
+  if (!q) return null;
+  try {
+    const enqOpts: HonkerEnqueueOptions = opts?.tx ? { tx: opts.tx } : {};
+    return q.enqueue({ memoryId } satisfies EmbeddingJob, enqOpts);
+  } catch {
+    return null;
+  }
+}
+
+/** Narrow an unknown job payload to an EmbeddingJob, or null if malformed. */
+export function parseEmbeddingJob(payload: unknown): EmbeddingJob | null {
+  if (payload && typeof payload === "object" && "memoryId" in payload) {
+    const id = (payload as { memoryId: unknown }).memoryId;
+    if (typeof id === "number" && Number.isInteger(id) && id > 0) return { memoryId: id };
+  }
+  return null;
+}

package/src/queue/worker.ts

@@ -0,0 +1,77 @@
+/**
+ * queue/worker.ts — long-lived embedding worker.
+ *
+ * Claims { memoryId } jobs from the Honker embedding queue, generates the
+ * embedding via embedMemory() (which writes memory_embeddings + the vec0 index),
+ * and acks. Provider/transient errors retry with exponential backoff; Honker
+ * dead-letters once attempts exceed the queue's maxAttempts. Malformed payloads
+ * are failed immediately (no retry).
+ *
+ * Lifecycle: start ONCE in the long-lived process (graph-server). Short-lived
+ * hook processes only enqueue — they must never start a worker. Returns an inert
+ * handle when Honker is unavailable.
+ */
+import { getEmbeddingQueue, parseEmbeddingJob } from "./index.js";
+
+export interface EmbeddingWorkerHandle {
+  /** Whether a live worker loop is running (false when Honker is unavailable). */
+  readonly running: boolean;
+  /** Stop the loop and wait for the current iteration to settle. */
+  stop(): Promise<void>;
+}
+
+const INERT: EmbeddingWorkerHandle = { running: false, stop: async () => {} };
+
+const BASE_BACKOFF_S = 5;
+function backoffSeconds(attempts: number): number {
+  return Math.ceil(BASE_BACKOFF_S * 2 ** Math.max(0, attempts - 1));
+}
+
+/**
+ * Start the embedding worker. No-op (inert handle) when Honker is unavailable.
+ */
+export function startEmbeddingWorker(opts?: { workerId?: string }): EmbeddingWorkerHandle {
+  const queue = getEmbeddingQueue();
+  if (!queue) return INERT;
+
+  const workerId = opts?.workerId ?? `ltm-embed-${process.pid}`;
+  const controller = new AbortController();
+  const waker = queue.claimWaker();
+
+  const loop = (async () => {
+    const { embedMemory } = await import("../embeddings.js");
+    const { getDb } = await import("../shared-db.js");
+    const db = getDb();
+    try {
+      while (!controller.signal.aborted) {
+        const job = await waker.next(workerId, { signal: controller.signal });
+        if (!job) return; // aborted
+        const parsed = parseEmbeddingJob(job.payload);
+        if (!parsed) {
+          job.fail("invalid embedding job payload");
+          continue;
+        }
+        try {
+          await embedMemory(db, parsed.memoryId);
+          job.ack();
+        } catch (err) {
+          if (controller.signal.aborted) return;
+          const msg = err instanceof Error ? (err.stack ?? err.message) : String(err);
+          job.retry(backoffSeconds(job.attempts), msg);
+        }
+      }
+    } finally {
+      waker.close();
+    }
+  })();
+  // Prevent unhandled rejection if the loop throws on teardown.
+  loop.catch(() => {});
+
+  return {
+    running: true,
+    stop: async () => {
+      controller.abort();
+      try { await loop; } catch { /* settled */ }
+    },
+  };
+}

package/src/recall/categorise.ts

@@ -0,0 +1,139 @@
+/**
+ * recall/categorise.ts — Heuristic + LLM-fallback category classifier.
+ *
+ * Strategy:
+ *   1. Score each MemoryCategory by keyword density in the content.
+ *   2. If the top score meets or exceeds confidenceThreshold → return it.
+ *   3. Otherwise call Anthropic Messages API (claude-haiku) to classify.
+ *   4. If LLM call fails, fall back to the best heuristic guess.
+ */
+
+import type { MemoryCategory } from "../dao/types.js";
+
+export interface CategoriseResult {
+  category: MemoryCategory;
+  confidence: number;
+  source: "heuristic" | "llm";
+}
+
+// ── Keyword tables ─────────────────────────────────────────────────────────────
+
+const KEYWORDS: Record<MemoryCategory, string[]> = {
+  preference: [
+    "prefer", "always use", "convention", "style", "format", "naming",
+    "use .* not", "default", "standard", "i like", "we use",
+  ],
+  architecture: [
+    "architecture", "design", "structure", "system", "component", "module",
+    "layer", "schema", "database", "api", "interface", "integration",
+    "dependency", "abstraction", "service",
+  ],
+  gotcha: [
+    "warning", "pitfall", "bug", "avoid", "never", "don't", "dont",
+    "broken", "fail", "trap", "careful", "watch out", "⚠", "issue",
+    "error", "problem", "beware",
+  ],
+  pattern: [
+    "pattern", "approach", "recipe", "template", "boilerplate",
+    "solution", "technique", "method", "strategy", "implement",
+    "how to", "example", "reusable",
+  ],
+  workflow: [
+    "process", "step", "procedure", "workflow", "pipeline",
+    "sequence", "flow", "first", "then", "finally", "checklist",
+    "before", "after", "when to",
+  ],
+  constraint: [
+    "must", "required", "mandatory", "cannot", "should not",
+    "limit", "restriction", "enforce", "rule", "policy", "compliance",
+    "always", "never allowed", "forbidden",
+  ],
+};
+
+// ── Heuristic scorer ───────────────────────────────────────────────────────────
+
+function scoreHeuristic(content: string): { category: MemoryCategory; confidence: number } {
+  const lower = content.toLowerCase();
+
+  const scores = Object.entries(KEYWORDS).map(([cat, keywords]) => {
+    let hits = 0;
+    for (const kw of keywords) {
+      const re = new RegExp(kw, "gi");
+      const matches = lower.match(re);
+      if (matches) hits += matches.length;
+    }
+    return { category: cat as MemoryCategory, hits };
+  });
+
+  const total = scores.reduce((s, x) => s + x.hits, 0);
+  const best = scores.sort((a, b) => b.hits - a.hits)[0];
+
+  if (!best || best.hits === 0) {
+    return { category: "pattern", confidence: 0 };
+  }
+
+  // Confidence = share of total hits, clamped to [0, 1].
+  // A dominant category scores high; if two categories are equal, confidence halves.
+  const confidence = Math.min(best.hits / Math.max(total, 1), 1);
+  return { category: best.category, confidence };
+}
+
+// ── LLM fallback ───────────────────────────────────────────────────────────────
+
+const CATEGORIES_LIST = Object.keys(KEYWORDS).join(", ");
+const VALID_CATEGORIES = new Set<string>(Object.keys(KEYWORDS));
+
+async function classifyWithLlm(content: string): Promise<MemoryCategory | null> {
+  const apiKey = process.env["ANTHROPIC_API_KEY"];
+  if (!apiKey) return null;
+
+  try {
+    const res = await fetch("https://api.anthropic.com/v1/messages", {
+      method: "POST",
+      headers: {
+        "x-api-key": apiKey,
+        "anthropic-version": "2023-06-01",
+        "content-type": "application/json",
+      },
+      body: JSON.stringify({
+        model: "claude-haiku-4-5-20251001",
+        max_tokens: 20,
+        messages: [{
+          role: "user",
+          content: `Classify this memory into exactly one category. Reply with only the category name, nothing else.\n\nCategories: ${CATEGORIES_LIST}\n\nMemory: ${content.slice(0, 500)}`,
+        }],
+      }),
+      signal: AbortSignal.timeout(8_000),
+    });
+
+    if (!res.ok) return null;
+
+    const json = await res.json() as { content?: Array<{ type: string; text?: string }> };
+    const text = json.content?.[0]?.text?.trim().toLowerCase() ?? "";
+
+    return VALID_CATEGORIES.has(text) ? (text as MemoryCategory) : null;
+  } catch {
+    return null;
+  }
+}
+
+// ── Public API ─────────────────────────────────────────────────────────────────
+
+export async function categorise(
+  content: string,
+  confidenceThreshold = 0.6,
+): Promise<CategoriseResult> {
+  const heuristic = scoreHeuristic(content);
+
+  if (heuristic.confidence >= confidenceThreshold) {
+    return { ...heuristic, source: "heuristic" };
+  }
+
+  const llmCategory = await classifyWithLlm(content);
+  if (llmCategory) {
+    return { category: llmCategory, confidence: 1.0, source: "llm" };
+  }
+
+  // LLM unavailable or failed — return best heuristic guess regardless of confidence
+  return { ...heuristic, source: "heuristic" };
+}

package/src/recall/explainer.ts

@@ -0,0 +1,76 @@
+/**
+ * recall/explainer.ts — Pure function that builds a score breakdown for each recall result.
+ * No DB access. No side effects. Deterministic.
+ *
+ * Score formula:
+ *   totalScore = (fts * 0.40) + (semantic * 0.35) + (importance * 0.15) + (recency * 0.10)
+ *   When provider is disabled, FTS weight absorbs the semantic slot:
+ *   totalScore = (fts * 0.75) + (importance * 0.15) + (recency * 0.10)
+ */
+
+export type MemoryTemperature = "hot" | "warm" | "cool" | "cold";
+
+export interface RecallExplainer {
+  ftsRank: number | null;
+  semanticScore: number | null;
+  importanceBoost: number;
+  recencyBoost: number;
+  totalScore: number;
+  temperature: MemoryTemperature;
+}
+
+export interface ExplainerInput {
+  importance: number;
+  recall_count: number;
+  last_recalled_at: string | null | undefined;
+  ftsRank?: number | null;
+  semanticScore?: number | null;
+}
+
+/** Compute memory temperature from access pattern. */
+export function computeTemperature(
+  recallCount: number,
+  lastRecalledAt: string | null | undefined,
+): MemoryTemperature {
+  const daysSince = lastRecalledAt
+    ? (Date.now() - new Date(lastRecalledAt).getTime()) / 86_400_000
+    : Infinity;
+
+  if (recallCount >= 10 || daysSince <= 7) return "hot";
+  if (recallCount >= 3 || daysSince <= 30) return "warm";
+  if (recallCount >= 1 || daysSince <= 90) return "cool";
+  return "cold";
+}
+
+/** Build a RecallExplainer for a single memory result. Pure function — no I/O. */
+export function buildExplainer(input: ExplainerInput): RecallExplainer {
+  const importanceBoost = Math.min(input.importance, 5) / 5;
+
+  const daysSince = input.last_recalled_at
+    ? (Date.now() - new Date(input.last_recalled_at).getTime()) / 86_400_000
+    : 90;
+  const recencyBoost = Math.max(0, 1 - daysSince / 90);
+
+  const ftsRank = input.ftsRank ?? null;
+  const semanticScore = input.semanticScore ?? null;
+
+  let totalScore: number;
+  if (semanticScore !== null && ftsRank !== null) {
+    totalScore = (ftsRank * 0.40) + (semanticScore * 0.35) + (importanceBoost * 0.15) + (recencyBoost * 0.10);
+  } else if (ftsRank !== null) {
+    totalScore = (ftsRank * 0.75) + (importanceBoost * 0.15) + (recencyBoost * 0.10);
+  } else if (semanticScore !== null) {
+    totalScore = (semanticScore * 0.75) + (importanceBoost * 0.15) + (recencyBoost * 0.10);
+  } else {
+    totalScore = (importanceBoost * 0.15) + (recencyBoost * 0.10);
+  }
+
+  return {
+    ftsRank,
+    semanticScore,
+    importanceBoost,
+    recencyBoost,
+    totalScore,
+    temperature: computeTemperature(input.recall_count, input.last_recalled_at),
+  };
+}

package/src/scheduler/index.ts

@@ -0,0 +1,97 @@
+/**
+ * scheduler/index.ts — Honker leader-elected cron for the janitor.
+ *
+ * Registers the janitor as a recurring Honker schedule and runs two loops in
+ * the long-lived process: (1) the built-in leader-elected scheduler loop (only
+ * ONE of the agent processes ticks, via Honker's advisory lock), which enqueues
+ * a `janitor-run` job on each boundary; (2) a worker that claims those jobs,
+ * runs the janitor, and `notify()`s the result on the "janitor" channel.
+ *
+ * Dormant + inert when Honker is unavailable — the caller keeps the existing
+ * `startAutoRun()` interval + `POST /api/janitor/run` fallback.
+ */
+import { getHonker } from "../lib/honker.js";
+
+export const JANITOR_SCHEDULE_NAME = "ltm-janitor";
+export const JANITOR_QUEUE = "ltm-janitor-queue";
+export const JANITOR_CHANNEL = "janitor";
+export const DEFAULT_JANITOR_CRON = "@every 6h";
+
+export interface JanitorSchedulerHandle {
+  /** Whether the leader loop + worker are running (false without Honker). */
+  readonly running: boolean;
+  /** Stop both loops and wait for them to settle. */
+  stop(): Promise<void>;
+}
+
+const INERT: JanitorSchedulerHandle = { running: false, stop: async () => {} };
+
+/**
+ * Register the janitor schedule (idempotent by name). Returns false when Honker
+ * is unavailable.
+ */
+export function registerJanitorSchedule(cron: string = DEFAULT_JANITOR_CRON): boolean {
+  const h = getHonker();
+  if (!h) return false;
+  try {
+    h.scheduler().add({
+      name: JANITOR_SCHEDULE_NAME,
+      queue: JANITOR_QUEUE,
+      schedule: cron,
+      payload: { kind: "janitor-run" },
+    });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Start the leader-elected janitor scheduler + its run worker. No-op (inert
+ * handle) when Honker is unavailable.
+ */
+export function startJanitorScheduler(opts?: { cron?: string; owner?: string }): JanitorSchedulerHandle {
+  const h = getHonker();
+  if (!h) return INERT;
+  if (!registerJanitorSchedule(opts?.cron)) return INERT;
+
+  const owner = opts?.owner ?? `ltm-sched-${process.pid}`;
+  const controller = new AbortController();
+  const scheduler = h.scheduler();
+  const queue = h.queue(JANITOR_QUEUE, { maxAttempts: 3 });
+  const waker = queue.claimWaker();
+
+  // Loop 1 — leader-elected ticking (enqueues janitor-run jobs on boundaries).
+  const leader = scheduler.run({ owner, signal: controller.signal }).catch(() => {});
+
+  // Loop 2 — claim janitor-run jobs, run the janitor, notify the result.
+  const worker = (async () => {
+    const { runJanitor } = await import("../janitor/index.js");
+    try {
+      while (!controller.signal.aborted) {
+        const job = await waker.next(owner, { signal: controller.signal });
+        if (!job) return;
+        try {
+          const status = await runJanitor();
+          job.ack();
+          try { h.notify(JANITOR_CHANNEL, { type: "janitor-complete", status }); } catch { /* notify best-effort */ }
+        } catch (err) {
+          if (controller.signal.aborted) return;
+          const msg = err instanceof Error ? (err.stack ?? err.message) : String(err);
+          job.retry(30, msg);
+        }
+      }
+    } finally {
+      waker.close();
+    }
+  })();
+  worker.catch(() => {});
+
+  return {
+    running: true,
+    stop: async () => {
+      controller.abort();
+      await Promise.allSettled([leader, worker]);
+    },
+  };
+}