@loreai/core 0.0.1 → 0.10.0

package/src/config.ts

@@ -0,0 +1,132 @@
+import { z } from "zod";
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+
+export const LoreConfig = z.object({
+  model: z
+    .object({
+      providerID: z.string(),
+      modelID: z.string(),
+    })
+    .optional(),
+  budget: z
+    .object({
+      distilled: z.number().min(0.05).max(0.5).default(0.25),
+      raw: z.number().min(0.1).max(0.7).default(0.4),
+      output: z.number().min(0.1).max(0.5).default(0.25),
+      /** Max fraction of usable context reserved for LTM system-prompt injection. Default: 0.10 (10%). */
+      ltm: z.number().min(0.02).max(0.3).default(0.10),
+    })
+    .default({ distilled: 0.25, raw: 0.4, output: 0.25, ltm: 0.10 }),
+  distillation: z
+    .object({
+      minMessages: z.number().min(3).default(8),
+      maxSegment: z.number().min(5).default(50),
+      metaThreshold: z.number().min(3).default(10),
+    })
+    .default({ minMessages: 8, maxSegment: 50, metaThreshold: 10 }),
+  knowledge: z
+    .object({
+      /** Set to false to disable long-term knowledge storage and system-prompt injection.
+       *  Conversation recall (temporal search, distillation search) and context management
+       *  (gradient transform, distillation) remain fully active. Disabling this turns off
+       *  the curator, knowledge DB writes, AGENTS.md sync, and LTM injection into the
+       *  system prompt. Default: true. */
+      enabled: z.boolean().default(true),
+    })
+    .default({ enabled: true }),
+  curator: z
+    .object({
+      enabled: z.boolean().default(true),
+      onIdle: z.boolean().default(true),
+      afterTurns: z.number().min(1).default(3),
+      /** Max knowledge entries per project before consolidation triggers. Default: 25. */
+      maxEntries: z.number().min(10).default(25),
+    })
+    .default({ enabled: true, onIdle: true, afterTurns: 3, maxEntries: 25 }),
+  pruning: z
+    .object({
+      /** Days to keep distilled temporal messages before pruning. Default: 120. */
+      retention: z.number().min(1).default(120),
+      /** Max total temporal_messages storage in MB before emergency pruning. Default: 1024 (1 GB). */
+      maxStorage: z.number().min(50).default(1024),
+    })
+    .default({ retention: 120, maxStorage: 1024 }),
+  search: z
+    .object({
+      /** BM25 column weights for knowledge FTS5 [title, content, category]. */
+      ftsWeights: z
+        .object({
+          title: z.number().min(0).default(6.0),
+          content: z.number().min(0).default(2.0),
+          category: z.number().min(0).default(3.0),
+        })
+        .default({ title: 6.0, content: 2.0, category: 3.0 }),
+      /** Max results per source in recall tool before fusion. Default: 10. */
+      recallLimit: z.number().min(1).max(50).default(10),
+      /** Enable LLM-based query expansion for the recall tool. Default: false.
+       *  When enabled, the configured model generates 2–3 alternative query phrasings
+       *  before search, improving recall for ambiguous queries. */
+      queryExpansion: z.boolean().default(false),
+      /** Vector embedding search.
+       *  Supports multiple providers: "voyage" (Voyage AI, VOYAGE_API_KEY),
+       *  "openai" (OpenAI, OPENAI_API_KEY).
+       *  Automatically enabled when the configured provider's API key env var is set.
+       *  Set enabled: false to explicitly disable even with the key present. */
+      embeddings: z
+        .object({
+          /** Enable/disable vector embedding search. Default: true.
+           *  Set to false to explicitly disable even when the API key is set. */
+          enabled: z.boolean().default(true),
+          /** Embedding provider. Default: "voyage".
+           *  Each provider reads its own env var for the API key:
+           *  - "voyage": VOYAGE_API_KEY (default model: voyage-code-3, 1024 dims)
+           *  - "openai": OPENAI_API_KEY (default model: text-embedding-3-small, 1536 dims) */
+          provider: z.enum(["voyage", "openai"]).default("voyage"),
+          /** Model ID for the embedding provider. Default depends on provider. */
+          model: z.string().default("voyage-code-3"),
+          /** Embedding dimensions. Default: 1024. */
+          dimensions: z.number().min(256).max(2048).default(1024),
+        })
+        .default({
+          enabled: true,
+          provider: "voyage",
+          model: "voyage-code-3",
+          dimensions: 1024,
+        }),
+    })
+    .default({
+      ftsWeights: { title: 6.0, content: 2.0, category: 3.0 },
+      recallLimit: 10,
+      queryExpansion: false,
+      embeddings: { enabled: true, provider: "voyage" as const, model: "voyage-code-3", dimensions: 1024 },
+    }),
+  crossProject: z.boolean().default(false),
+  agentsFile: z
+    .object({
+      /** Set to false to disable all AGENTS.md export/import behaviour. */
+      enabled: z.boolean().default(true),
+      /** Path to the agents file, relative to the project root. */
+      path: z.string().default("AGENTS.md"),
+    })
+    .default({ enabled: true, path: "AGENTS.md" }),
+});
+
+export type LoreConfig = z.infer<typeof LoreConfig>;
+
+let current: LoreConfig = LoreConfig.parse({});
+
+export function config(): LoreConfig {
+  return current;
+}
+
+export async function load(directory: string): Promise<LoreConfig> {
+  const path = join(directory, ".lore.json");
+  if (existsSync(path)) {
+    const raw = JSON.parse(readFileSync(path, "utf8"));
+    current = LoreConfig.parse(raw);
+    return current;
+  }
+  current = LoreConfig.parse({});
+  return current;
+}

package/src/curator.ts

@@ -0,0 +1,220 @@
+import { config } from "./config";
+import * as temporal from "./temporal";
+import * as ltm from "./ltm";
+import * as log from "./log";
+import { CURATOR_SYSTEM, curatorUser, CONSOLIDATION_SYSTEM, consolidationUser } from "./prompt";
+import type { LLMClient } from "./types";
+
+/**
+ * Maximum length (chars) for a single knowledge entry's content.
+ * ~400 tokens at chars/3. Entries exceeding this are truncated with a notice.
+ * The curator prompt also instructs the model to stay within this limit,
+ * so truncation is a last-resort safety net.
+ */
+const MAX_ENTRY_CONTENT_LENGTH = 1200;
+
+type CuratorOp =
+  | {
+      op: "create";
+      category: string;
+      title: string;
+      content: string;
+      scope: "project" | "global";
+      crossProject?: boolean;
+    }
+  | { op: "update"; id: string; content?: string; confidence?: number }
+  | { op: "delete"; id: string; reason: string };
+
+function parseOps(text: string): CuratorOp[] {
+  const cleaned = text
+    .trim()
+    .replace(/^```json?\s*/i, "")
+    .replace(/\s*```$/i, "");
+  try {
+    const parsed = JSON.parse(cleaned);
+    if (!Array.isArray(parsed)) return [];
+    return parsed.filter(
+      (op: unknown) =>
+        typeof op === "object" &&
+        op !== null &&
+        "op" in op &&
+        typeof (op as Record<string, unknown>).op === "string",
+    ) as CuratorOp[];
+  } catch {
+    return [];
+  }
+}
+
+// Track which messages we've already curated — per session to prevent
+// cross-session leaking (curation on session A advancing the timestamp
+// past session B's messages, causing B's curation to find < 3 recent).
+const lastCuratedAt = new Map<string, number>();
+
+export async function run(input: {
+  llm: LLMClient;
+  projectPath: string;
+  sessionID: string;
+  model?: { providerID: string; modelID: string };
+}): Promise<{ created: number; updated: number; deleted: number }> {
+  const cfg = config();
+  if (!cfg.curator.enabled) return { created: 0, updated: 0, deleted: 0 };
+
+  // Get recent messages since last curation
+  const all = temporal.bySession(input.projectPath, input.sessionID);
+  const sessionCuratedAt = lastCuratedAt.get(input.sessionID) ?? 0;
+  const recent = all.filter((m) => m.created_at > sessionCuratedAt);
+  if (recent.length < 3) return { created: 0, updated: 0, deleted: 0 };
+
+  const text = recent.map((m) => `[${m.role}] ${m.content}`).join("\n\n");
+  const existing = ltm.forProject(input.projectPath, false);
+  const existingForPrompt = existing.map((e) => ({
+    id: e.id,
+    category: e.category,
+    title: e.title,
+    content: e.content,
+  }));
+
+  const userContent = curatorUser({
+    messages: text,
+    existing: existingForPrompt,
+  });
+  const model = input.model ?? cfg.model;
+  const responseText = await input.llm.prompt(
+    CURATOR_SYSTEM,
+    userContent,
+    { model, workerID: "lore-curator" },
+  );
+  if (!responseText) return { created: 0, updated: 0, deleted: 0 };
+
+  const ops = parseOps(responseText);
+  let created = 0;
+  let updated = 0;
+  let deleted = 0;
+
+  const idsToSync: string[] = [];
+
+  for (const op of ops) {
+    if (op.op === "create") {
+      // Truncate oversized content — the model should stay within the prompt's
+      // 500-word limit, but enforce it here as a hard safety net.
+      const content =
+        op.content.length > MAX_ENTRY_CONTENT_LENGTH
+          ? op.content.slice(0, MAX_ENTRY_CONTENT_LENGTH) +
+            " [truncated — entry too long]"
+          : op.content;
+      const id = ltm.create({
+        projectPath: op.scope === "project" ? input.projectPath : undefined,
+        category: op.category,
+        title: op.title,
+        content,
+        session: input.sessionID,
+        scope: op.scope,
+        crossProject: op.crossProject ?? true,
+      });
+      idsToSync.push(id);
+      created++;
+    } else if (op.op === "update") {
+      const entry = ltm.get(op.id);
+      if (entry) {
+        const content =
+          op.content !== undefined && op.content.length > MAX_ENTRY_CONTENT_LENGTH
+            ? op.content.slice(0, MAX_ENTRY_CONTENT_LENGTH) +
+              " [truncated — entry too long]"
+            : op.content;
+        ltm.update(op.id, { content, confidence: op.confidence });
+        if (op.content !== undefined) idsToSync.push(op.id);
+        updated++;
+      }
+    } else if (op.op === "delete") {
+      const entry = ltm.get(op.id);
+      if (entry) {
+        ltm.remove(op.id);
+        deleted++;
+      }
+    }
+  }
+
+  // Sync cross-references for created/updated entries
+  for (const id of idsToSync) {
+    ltm.syncRefs(id);
+  }
+
+  lastCuratedAt.set(input.sessionID, Date.now());
+  return { created, updated, deleted };
+}
+
+export function resetCurationTracker(sessionID?: string) {
+  if (sessionID) {
+    lastCuratedAt.delete(sessionID);
+  } else {
+    lastCuratedAt.clear();
+  }
+}
+
+/**
+ * Consolidation pass: reviews ALL project entries and merges/trims/deletes
+ * to reduce entry count to cfg.curator.maxEntries. Only runs when the current
+ * entry count exceeds the target. Uses the same worker session as curation.
+ *
+ * Only "update" and "delete" ops are applied — consolidation never creates entries.
+ */
+export async function consolidate(input: {
+  llm: LLMClient;
+  projectPath: string;
+  sessionID: string;
+  model?: { providerID: string; modelID: string };
+}): Promise<{ updated: number; deleted: number }> {
+  const cfg = config();
+  if (!cfg.curator.enabled) return { updated: 0, deleted: 0 };
+
+  const entries = ltm.forProject(input.projectPath, false);
+  if (entries.length <= cfg.curator.maxEntries) return { updated: 0, deleted: 0 };
+
+  const entriesForPrompt = entries.map((e) => ({
+    id: e.id,
+    category: e.category,
+    title: e.title,
+    content: e.content,
+  }));
+
+  const userContent = consolidationUser({
+    entries: entriesForPrompt,
+    targetMax: cfg.curator.maxEntries,
+  });
+  const model = input.model ?? cfg.model;
+  const responseText = await input.llm.prompt(
+    CONSOLIDATION_SYSTEM,
+    userContent,
+    { model, workerID: "lore-curator" },
+  );
+  if (!responseText) return { updated: 0, deleted: 0 };
+
+  const ops = parseOps(responseText);
+  let updated = 0;
+  let deleted = 0;
+
+  for (const op of ops) {
+    // Consolidation only applies update and delete — never create.
+    if (op.op === "update") {
+      const entry = ltm.get(op.id);
+      if (entry) {
+        const content =
+          op.content !== undefined && op.content.length > MAX_ENTRY_CONTENT_LENGTH
+            ? op.content.slice(0, MAX_ENTRY_CONTENT_LENGTH) +
+              " [truncated — entry too long]"
+            : op.content;
+        ltm.update(op.id, { content, confidence: op.confidence });
+        updated++;
+      }
+    } else if (op.op === "delete") {
+      const entry = ltm.get(op.id);
+      if (entry) {
+        ltm.remove(op.id);
+        deleted++;
+      }
+    }
+    // "create" ops are silently ignored — consolidation must not add entries.
+  }
+
+  return { updated, deleted };
+}

package/src/db/driver.bun.ts

@@ -0,0 +1,18 @@
+// Bun runtime driver for Lore's SQLite access.
+//
+// Selected automatically via the `#db/driver` subpath import map when running
+// under Bun (OpenCode plugin, `bun test`).
+//
+// The `Database` class is re-exported as-is; `bun:sqlite`'s API already matches
+// everything Lore uses: `.query(sql)` with cached prepared statements, `.run()`,
+// `.all()`, `.get()`, transactions, PRAGMAs, BLOB columns, and FTS5.
+
+import { Database } from "bun:sqlite";
+import { createHash } from "node:crypto";
+
+export { Database };
+
+/** Stable SHA-256 hex digest — replaces the Bun-only `Bun.CryptoHasher`. */
+export function sha256(input: string): string {
+  return createHash("sha256").update(input).digest("hex");
+}

package/src/db/driver.node.ts

@@ -0,0 +1,54 @@
+// Node runtime driver for Lore's SQLite access.
+//
+// Selected via the `#db/driver` subpath import map when running under Node
+// (Pi extension, future ACP server, and CI nodes that aren't Bun). `node:sqlite`
+// has shipped in Node since 22.5 and stabilized (no flag) in Node 24.
+//
+// Bun deliberately does NOT implement `node:sqlite`, so src code that imports
+// from this file must go through `#db/driver`. Never import `node:sqlite`
+// directly outside this file — it will break `bun test` which runs against src.
+
+import { DatabaseSync, type StatementSync } from "node:sqlite";
+import { createHash } from "node:crypto";
+
+/**
+ * Per-database cache of prepared statements keyed by SQL string.
+ *
+ * `bun:sqlite` automatically caches prepared statements per-DB when using
+ * `.query(sql)`; `node:sqlite` has only `.prepare(sql)` which recompiles on
+ * every call. We add a thin `.query()` alias on top of `.prepare()` with
+ * caching so every existing call site (`db().query(...).all(...)`) keeps
+ * working identically.
+ *
+ * WeakMap: cache is tied to the Database instance lifetime, no manual cleanup.
+ */
+const statementCache = new WeakMap<DatabaseSync, Map<string, StatementSync>>();
+
+/**
+ * Drop-in replacement for `bun:sqlite`'s `Database`.
+ *
+ * Adds a `.query()` method that caches the underlying `StatementSync`
+ * per SQL string. All other methods (`.prepare()`, `.exec()`, `.run()`,
+ * `.close()`, PRAGMAs, transactions) come from `DatabaseSync` unchanged.
+ */
+export class Database extends DatabaseSync {
+  /** Cached prepared statement for this SQL. Compiled on first call. */
+  query(sql: string): StatementSync {
+    let map = statementCache.get(this);
+    if (!map) {
+      map = new Map<string, StatementSync>();
+      statementCache.set(this, map);
+    }
+    let stmt = map.get(sql);
+    if (!stmt) {
+      stmt = this.prepare(sql);
+      map.set(sql, stmt);
+    }
+    return stmt;
+  }
+}
+
+/** Stable SHA-256 hex digest — replaces the Bun-only `Bun.CryptoHasher`. */
+export function sha256(input: string): string {
+  return createHash("sha256").update(input).digest("hex");
+}