@loreai/core 0.12.0 → 0.13.1

package/src/index.ts

@@ -15,6 +15,7 @@ export * as distillation from "./distillation";
 export * as curator from "./curator";
 export * as embedding from "./embedding";
 export * as latReader from "./lat-reader";
+export * as patternExtract from "./pattern-extract";
 export * as log from "./log";
 
 export {
@@ -72,6 +73,7 @@ export {
   getLastTransformEstimate,
   toolStripAnnotation,
   onIdleResume,
+  getLastTurnAt,
   consumeCameOutOfIdle,
   // Test-only — exposed at the barrel so host-package tests can simulate idle
   // gaps without sleeping. Not part of the public API.
@@ -93,13 +95,18 @@ export {
   COMPACT_SUMMARY_TEMPLATE,
   buildCompactPrompt,
 } from "./prompt";
-export { shouldImport, importFromFile, exportToFile } from "./agents-file";
+export {
+  shouldImport,
+  importFromFile,
+  exportToFile,
+  exportLoreFile,
+  importLoreFile,
+  shouldImportLoreFile,
+  loreFileExists,
+  LORE_FILE,
+} from "./agents-file";
 export { workerSessionIDs, isWorkerSession } from "./worker";
 export * as workerModel from "./worker-model";
-export {
-  WORKER_JUDGE_SYSTEM,
-  workerJudgeUser,
-} from "./worker-model";
 export {
   ftsQuery,
   ftsQueryOr,
@@ -107,6 +114,7 @@ export {
   reciprocalRankFusion,
   expandQuery,
   extractTopTerms,
+  exactTermMatchRank,
 } from "./search";
 export {
   serialize,

package/src/pattern-extract.ts

@@ -0,0 +1,108 @@
+/**
+ * Lightweight regex-based pattern extraction from distillation observations.
+ *
+ * Scans for decision/preference/choice patterns and returns structured
+ * extractions that can be stored as knowledge entries. No LLM required.
+ *
+ * Patterns target how decisions and preferences are typically expressed
+ * in distilled engineering context:
+ *   - "decided to use X"
+ *   - "chose X over Y"
+ *   - "switched from X to Y"
+ *   - "prefers X for Y"
+ *   - "going with X because Y"
+ *
+ * Extracted entries participate in the normal curator cycle — the curator
+ * can consolidate or remove them based on actual value. The extraction is
+ * a cheap seed, not a permanent fixture.
+ */
+
+export type ExtractedPattern = {
+  category: "decision" | "preference";
+  /** Short descriptive title, e.g. "Chose PostgreSQL over MySQL". */
+  title: string;
+  /** Full matched text for context. */
+  content: string;
+};
+
+type PatternDef = {
+  regex: RegExp;
+  category: "decision" | "preference";
+  titleFn: (match: RegExpMatchArray) => string;
+};
+
+const PATTERNS: PatternDef[] = [
+  // Decision patterns
+  {
+    regex: /decided to (?:use |switch to |go with |adopt )(.+?)(?:\.|,|$)/gi,
+    category: "decision",
+    titleFn: (m) => `Decided to use ${m[1].trim()}`,
+  },
+  {
+    regex: /chose (.+?) over (.+?)(?:\.|,|$)/gi,
+    category: "decision",
+    titleFn: (m) => `Chose ${m[1].trim()} over ${m[2].trim()}`,
+  },
+  {
+    regex: /switched from (.+?) to (.+?)(?:\.|,|$)/gi,
+    category: "decision",
+    titleFn: (m) => `Switched from ${m[1].trim()} to ${m[2].trim()}`,
+  },
+  {
+    regex: /going with (.+?) (?:because|for|due to)(.+?)(?:\.|,|$)/gi,
+    category: "decision",
+    titleFn: (m) => `Going with ${m[1].trim()}`,
+  },
+  {
+    regex: /migrat(?:ed|ing) (?:from .+? )?to (.+?)(?:\.|,|$)/gi,
+    category: "decision",
+    titleFn: (m) => `Migrated to ${m[1].trim()}`,
+  },
+  {
+    regex: /adopted (.+?) (?:for|as|instead)(.+?)(?:\.|,|$)/gi,
+    category: "decision",
+    titleFn: (m) => `Adopted ${m[1].trim()}`,
+  },
+
+  // Preference patterns
+  {
+    regex: /prefers? (.+?) (?:over|to|instead of|rather than) (.+?)(?:\.|,|$)/gi,
+    category: "preference",
+    titleFn: (m) => `Prefers ${m[1].trim()} over ${m[2].trim()}`,
+  },
+  {
+    regex:
+      /(?:user |team |we )(?:always |usually |typically )(?:use|prefer|go with) (.+?)(?:\.|,|$)/gi,
+    category: "preference",
+    titleFn: (m) => `Typically uses ${m[1].trim()}`,
+  },
+];
+
+/**
+ * Extract decision/preference patterns from distillation observations text.
+ *
+ * Returns structured entries suitable for `ltm.create()`. Deduplicates by
+ * lowercased title within a single call.
+ *
+ * @param observations  The distilled observations text to scan.
+ * @returns             Array of extracted patterns (may be empty).
+ */
+export function extractPatterns(observations: string): ExtractedPattern[] {
+  const results: ExtractedPattern[] = [];
+  const seen = new Set<string>();
+
+  for (const { regex, category, titleFn } of PATTERNS) {
+    // Reset lastIndex for global regexes reused across calls
+    regex.lastIndex = 0;
+    let match: RegExpMatchArray | null;
+    while ((match = regex.exec(observations)) !== null) {
+      const title = titleFn(match);
+      const key = title.toLowerCase();
+      if (seen.has(key)) continue;
+      seen.add(key);
+      results.push({ category, title, content: match[0].trim() });
+    }
+  }
+
+  return results;
+}

package/src/recall.ts

@@ -19,7 +19,9 @@ import type { LoreConfig } from "./config";
 import type { LLMClient } from "./types";
 import {
   EMPTY_QUERY,
+  exactTermMatchRank,
   expandQuery,
+  filterTerms,
   ftsQuery,
   ftsQueryOr,
   reciprocalRankFusion,
@@ -36,6 +38,7 @@ type Distillation = {
   generation: number;
   created_at: number;
   session_id: string;
+  c_norm: number | null;
 };
 
 export type ScoredDistillation = Distillation & { rank: number };
@@ -72,6 +75,41 @@ type TaggedResult =
   | { source: "temporal"; item: temporal.ScoredTemporalMessage }
   | { source: "lat-section"; item: latReader.ScoredLatSection };
 
+// ---------------------------------------------------------------------------
+// Tagged result helpers (used by exact-match boost + formatting)
+// ---------------------------------------------------------------------------
+
+/** Extract searchable text from any TaggedResult variant. */
+function getTaggedText(tagged: TaggedResult): string {
+  switch (tagged.source) {
+    case "knowledge":
+    case "cross-knowledge":
+      return `${tagged.item.title} ${tagged.item.content}`;
+    case "distillation":
+      return tagged.item.observations;
+    case "temporal":
+      return tagged.item.content;
+    case "lat-section":
+      return `${tagged.item.heading} ${tagged.item.content}`;
+  }
+}
+
+/** Unified key function for TaggedResult — source-prefixed ID for RRF dedup. */
+function taggedResultKey(r: TaggedResult): string {
+  switch (r.source) {
+    case "knowledge":
+      return `k:${r.item.id}`;
+    case "cross-knowledge":
+      return `xk:${r.item.id}`;
+    case "distillation":
+      return `d:${r.item.id}`;
+    case "temporal":
+      return `t:${r.item.id}`;
+    case "lat-section":
+      return `lat:${r.item.id}`;
+  }
+}
+
 // ---------------------------------------------------------------------------
 // Distillation search
 // ---------------------------------------------------------------------------
@@ -93,8 +131,8 @@ function searchDistillationsLike(input: {
     .join(" AND ");
   const likeParams = terms.map((term) => `%${term}%`);
   const sql = input.sessionID
-    ? `SELECT id, observations, generation, created_at, session_id FROM distillations WHERE project_id = ? AND session_id = ? AND ${conditions} ORDER BY created_at DESC LIMIT ?`
-    : `SELECT id, observations, generation, created_at, session_id FROM distillations WHERE project_id = ? AND ${conditions} ORDER BY created_at DESC LIMIT ?`;
+    ? `SELECT id, observations, generation, created_at, session_id, c_norm FROM distillations WHERE project_id = ? AND session_id = ? AND ${conditions} ORDER BY created_at DESC LIMIT ?`
+    : `SELECT id, observations, generation, created_at, session_id, c_norm FROM distillations WHERE project_id = ? AND ${conditions} ORDER BY created_at DESC LIMIT ?`;
   const allParams = input.sessionID
     ? [input.pid, input.sessionID, ...likeParams, input.limit]
     : [input.pid, ...likeParams, input.limit];
@@ -115,13 +153,13 @@ function searchDistillationsScored(input: {
   if (q === EMPTY_QUERY) return [];
 
   const ftsSQL = input.sessionID
-    ? `SELECT d.id, d.observations, d.generation, d.created_at, d.session_id, rank
+    ? `SELECT d.id, d.observations, d.generation, d.created_at, d.session_id, d.c_norm, rank
        FROM distillation_fts f
        CROSS JOIN distillations d ON d.rowid = f.rowid
        WHERE distillation_fts MATCH ?
        AND d.project_id = ? AND d.session_id = ?
        ORDER BY rank LIMIT ?`
-    : `SELECT d.id, d.observations, d.generation, d.created_at, d.session_id, rank
+    : `SELECT d.id, d.observations, d.generation, d.created_at, d.session_id, d.c_norm, rank
        FROM distillation_fts f
        CROSS JOIN distillations d ON d.rowid = f.rowid
        WHERE distillation_fts MATCH ?
@@ -241,7 +279,7 @@ export async function runRecall(input: RecallInput): Promise<RecallResult> {
   let queries = [query];
   if (searchConfig?.queryExpansion && llm) {
     try {
-      queries = await expandQuery(llm, query);
+      queries = await expandQuery(llm, query, undefined, sessionID);
     } catch (err) {
       log.info("recall: query expansion failed, using original:", err);
     }
@@ -376,7 +414,7 @@ export async function runRecall(input: RecallInput): Promise<RecallResult> {
           .map((hit): TaggedResult | null => {
             const row = db()
               .query(
-                "SELECT id, observations, generation, created_at, session_id FROM distillations WHERE id = ?",
+                "SELECT id, observations, generation, created_at, session_id, c_norm FROM distillations WHERE id = ?",
               )
               .get(hit.id) as Distillation | null;
             if (!row) return null;
@@ -448,6 +486,86 @@ export async function runRecall(input: RecallInput): Promise<RecallResult> {
     }
   }
 
+  // Distillation quality list: rank distillation candidates by a quality score
+  // that combines temporal clustering (c_norm) and age. Segments with low c_norm
+  // (uniformly distributed timestamps) are considered higher quality than bursty
+  // segments (high c_norm). Among high-c_norm segments, recent ones are more
+  // likely relevant. This adds a mild signal — RRF naturally blends it with the
+  // BM25 and vector signals without overriding them.
+  {
+    const distillationCandidates: Array<{
+      tagged: TaggedResult;
+      key: string;
+      qualityScore: number;
+    }> = [];
+
+    for (const list of allRrfLists) {
+      for (const item of list.items) {
+        if (item.source !== "distillation") continue;
+        const key = `d:${item.item.id}`;
+        const d = item.item as ScoredDistillation;
+        const cNorm = d.c_norm ?? 0; // NULL → treat as uniform (best case)
+        // Quality score: lower c_norm is better. For high c_norm, recency
+        // partially compensates. Age is normalized to days (capped at 90).
+        const ageDays = Math.min(
+          (Date.now() - d.created_at) / 86_400_000,
+          90,
+        );
+        // score ∈ [0, ~1]: 0 = best quality (uniform + recent)
+        // c_norm dominates (0–1), age adds a mild 0–0.1 penalty
+        const score = cNorm + (ageDays / 90) * 0.1;
+        distillationCandidates.push({ tagged: item, key, qualityScore: score });
+      }
+    }
+
+    if (distillationCandidates.length > 1) {
+      // De-duplicate by key (same distillation may appear in BM25 + vector lists)
+      const seen = new Set<string>();
+      const unique = distillationCandidates.filter((c) => {
+        if (seen.has(c.key)) return false;
+        seen.add(c.key);
+        return true;
+      });
+
+      // Sort by quality: lowest score first (best quality)
+      unique.sort((a, b) => a.qualityScore - b.qualityScore);
+
+      allRrfLists.push({
+        items: unique.map((c) => c.tagged),
+        key: (r) => `d:${r.item.id}`,
+      });
+    }
+  }
+
+  // Exact-match boost: add an additional RRF list that ranks candidates by
+  // the number of exact query term matches. This boosts proper nouns, file
+  // names, and technical terms that BM25's prefix/stem matching may dilute.
+  // Only runs when there are meaningful terms and existing candidates.
+  if (filterTerms(query).length > 0 && allRrfLists.length > 0) {
+    // Collect unique candidates across all lists
+    const allCandidates = new Map<string, TaggedResult>();
+    for (const list of allRrfLists) {
+      for (const item of list.items) {
+        const key = list.key(item);
+        if (!allCandidates.has(key)) allCandidates.set(key, item);
+      }
+    }
+
+    const candidateEntries = [...allCandidates.entries()];
+    const exactRanked = exactTermMatchRank(
+      candidateEntries,
+      ([, tagged]) => getTaggedText(tagged),
+      query,
+    );
+
+    if (exactRanked.length) {
+      allRrfLists.push({
+        items: exactRanked.map(([, item]) => item),
+        key: taggedResultKey,
+      });
+    }
+  }
+
   const fused = reciprocalRankFusion<TaggedResult>(allRrfLists);
   return formatFusedResults(fused, 20);
 }

package/src/search.ts

@@ -267,6 +267,41 @@ export function reciprocalRankFusion<T>(
   return [...scores.values()].sort((a, b) => b.score - a.score);
 }
 
+// ---------------------------------------------------------------------------
+// Exact term match ranking (Phase 5 — MemPalace-inspired keyword boost)
+// ---------------------------------------------------------------------------
+
+/**
+ * Score candidates by exact query term overlap.
+ *
+ * Returns items sorted by number of exact term matches (descending).
+ * Used as an additional RRF list to boost results that contain query terms
+ * verbatim — important for proper nouns, file names, and technical terms
+ * that BM25's prefix matching + Porter stemming can miss or dilute.
+ *
+ * Terms are filtered through the standard stopword + single-char filter
+ * (same as `ftsQuery`), then matched case-insensitively via `includes()`.
+ */
+export function exactTermMatchRank<T>(
+  items: T[],
+  getText: (item: T) => string,
+  query: string,
+): T[] {
+  const terms = filterTerms(query).map((t) => t.toLowerCase());
+  if (!terms.length) return [];
+
+  const scored = items
+    .map((item) => {
+      const text = getText(item).toLowerCase();
+      const matches = terms.filter((t) => text.includes(t)).length;
+      return { item, matches };
+    })
+    .filter((s) => s.matches > 0)
+    .sort((a, b) => b.matches - a.matches);
+
+  return scored.map((s) => s.item);
+}
+
 // ---------------------------------------------------------------------------
 // LLM query expansion (Phase 4)
 // ---------------------------------------------------------------------------
@@ -290,6 +325,7 @@ export async function expandQuery(
   llm: LLMClient,
   query: string,
   model?: { providerID: string; modelID: string },
+  sessionID?: string,
 ): Promise<string[]> {
   const TIMEOUT_MS = 3000;
 
@@ -299,7 +335,7 @@ export async function expandQuery(
       llm.prompt(
         QUERY_EXPANSION_SYSTEM,
         `Input: "${query}"`,
-        { model, workerID: "lore-query-expand" },
+        { model, workerID: "lore-query-expand", thinking: false, urgent: true, sessionID },
       ),
       new Promise<null>((resolve) => setTimeout(() => resolve(null), TIMEOUT_MS)),
     ]);

package/src/types.ts

@@ -189,7 +189,7 @@ export interface LLMClient {
    *
    * @param system  System prompt text
    * @param user    User message text
-   * @param opts    Optional model selection and worker identification
+   * @param opts    Optional model selection, worker identification, and thinking control
    * @returns The assistant's text response, or null on failure
    */
   prompt(
@@ -203,6 +203,46 @@ export interface LLMClient {
        * (e.g. OpenCode uses this as the session agent name).
        */
       workerID?: string;
+      /**
+       * Disable extended thinking/reasoning for this call.
+       *
+       * Background workers discard thinking tokens — they only extract the
+       * text response. Setting `thinking: false` tells the adapter to avoid
+       * producing (and billing for) thinking tokens when possible.
+       *
+       * Adapter behavior:
+       * - Gateway: no-op (bare API call never triggers thinking)
+       * - Pi: passes `thinkingEnabled: false` to `complete()`
+       * - OpenCode: cannot honor — SDK has no thinking toggle on session.prompt();
+       *   relies on Part A (non-reasoning model selection) instead
+       */
+      thinking?: boolean;
+      /**
+       * When true, the request must be processed immediately and the result
+       * returned before the next user turn. When false or absent, the request
+       * may be deferred to a batch queue for cost savings (50% discount via
+       * Anthropic's Message Batches API).
+       *
+       * Callers that `await` the result for a blocking operation (compaction,
+       * overflow recovery, query expansion) should set `urgent: true`.
+       * Fire-and-forget background work (incremental distillation, idle
+       * curation) should leave it unset or set `false`.
+       *
+       * Only the gateway's BatchLLMClient honors this flag; other adapters
+       * (OpenCode, Pi) ignore it and always process immediately.
+       */
+      urgent?: boolean;
+      /**
+       * Session identifier for per-session auth credential lookup.
+       *
+       * The gateway uses this to resolve the correct API key or OAuth
+       * token for the session that triggered the work, preventing
+       * cross-session key mixups when multiple clients are connected.
+       *
+       * Other adapters (OpenCode, Pi) ignore this field — they resolve
+       * auth through their own mechanisms.
+       */
+      sessionID?: string;
     },
   ): Promise<string | null>;
 }

package/src/worker-model.ts

@@ -25,7 +25,11 @@ export type ModelInfo = {
   providerID: string;
   cost: { input: number }; // per-token cost
   status: string;
-  capabilities: { input: { text: boolean } };
+  capabilities: {
+    input: { text: boolean };
+    /** Whether this model supports extended thinking/reasoning. */
+    reasoning?: boolean;
+  };
 };
 
 /** Result of a worker model validation stored in kv_meta. */
@@ -64,8 +68,17 @@ export function selectWorkerCandidates(
 
   if (eligible.length === 0) return [];
 
-  // Sort by cost ascending (cheapest first)
-  const sorted = [...eligible].sort((a, b) => a.cost.input - b.cost.input);
+  // Sort by cost ascending, then prefer non-reasoning models at equal cost.
+  // Non-reasoning models don't produce thinking tokens, avoiding wasted spend
+  // on tokens that background workers discard.
+  const sorted = [...eligible].sort((a, b) => {
+    const costDiff = a.cost.input - b.cost.input;
+    if (costDiff !== 0) return costDiff;
+    // At equal cost, non-reasoning (0) sorts before reasoning (1)
+    const aReasoning = a.capabilities.reasoning ? 1 : 0;
+    const bReasoning = b.capabilities.reasoning ? 1 : 0;
+    return aReasoning - bReasoning;
+  });
 
   // Cheapest overall
   const cheapest = sorted[0];
@@ -139,6 +152,11 @@ export function storeValidatedWorkerModel(result: WorkerModelResult): void {
     .run(key, value, value);
 }
 
+/** Clear a stored worker model validation (e.g. when the model is deprecated). */
+export function clearValidatedWorkerModel(providerID: string): void {
+  db().query("DELETE FROM kv_meta WHERE key = ?").run(`${KV_PREFIX}${providerID}`);
+}
+
 /**
  * Check whether the stored validation is stale (fingerprint mismatch).
  */
@@ -237,6 +255,122 @@ export function parseJudgeScore(response: string): number | null {
   return parseInt(match[1], 10);
 }
 
+// ---------------------------------------------------------------------------
+// Validation orchestration
+// ---------------------------------------------------------------------------
+
+import { DISTILLATION_SYSTEM, distillationUser } from "./prompt";
+import type { LLMClient } from "./types";
+
+export type ValidationInput = {
+  llm: LLMClient;
+  providerID: string;
+  sessionModelID: string;
+  candidates: ModelInfo[];
+  /** Recent gen-0 distillation to use as reference (observations text). */
+  referenceObservations: string;
+  /** Source messages text for re-running distillation with candidates. */
+  sourceMessagesText: string;
+  /** Date string for the distillation prompt. */
+  date: string;
+};
+
+/**
+ * Run the two-phase quality validation for worker model candidates.
+ * Returns the cheapest passing candidate, or null if none pass.
+ */
+export async function runValidation(
+  input: ValidationInput,
+): Promise<WorkerModelResult | null> {
+  const { llm, candidates, referenceObservations, sourceMessagesText, date } = input;
+
+  const userPrompt = distillationUser({
+    messages: sourceMessagesText,
+    date,
+  });
+
+  for (const candidate of candidates) {
+    // Skip the session model — it produced the reference, no need to test
+    if (candidate.id === input.sessionModelID) continue;
+
+    // Phase 1: run distillation with candidate model
+    let candidateObservations: string | null = null;
+    try {
+      const raw = await llm.prompt(DISTILLATION_SYSTEM, userPrompt, {
+        model: { providerID: candidate.providerID, modelID: candidate.id },
+        workerID: "lore-distill",
+        thinking: false,
+      });
+      if (raw) {
+        // Parse <observations>...</observations> block
+        const match = raw.match(/<observations>([\s\S]*?)<\/observations>/);
+        candidateObservations = match ? match[1].trim() : raw.trim();
+      }
+    } catch (e) {
+      log.warn(`worker model validation: candidate ${candidate.id} failed:`, e);
+      continue;
+    }
+
+    const structural = structuralCheck(candidateObservations, referenceObservations);
+    if (!structural.passed) {
+      log.info(
+        `worker model validation: ${candidate.id} failed structural check: ${structural.reason}`,
+      );
+      continue;
+    }
+
+    // Phase 2: LLM judge (using session model)
+    let judgeScore: number | null = null;
+    try {
+      const judgeResponse = await llm.prompt(
+        WORKER_JUDGE_SYSTEM,
+        workerJudgeUser(referenceObservations, candidateObservations!),
+        { workerID: "lore-distill", thinking: false }, // use session model (no model override)
+      );
+      if (judgeResponse) {
+        judgeScore = parseJudgeScore(judgeResponse);
+      }
+    } catch (e) {
+      log.warn(`worker model validation: judge call failed for ${candidate.id}:`, e);
+    }
+
+    if (judgeScore !== null && judgeScore < 3) {
+      log.info(
+        `worker model validation: ${candidate.id} failed judge (score=${judgeScore})`,
+      );
+      continue;
+    }
+
+    // Candidate passed both phases
+    const fingerprint = computeModelFingerprint(
+      input.providerID,
+      input.sessionModelID,
+      candidates.map((c) => c.id),
+    );
+
+    const result: WorkerModelResult = {
+      modelID: candidate.id,
+      providerID: candidate.providerID,
+      fingerprint,
+      validatedAt: Date.now(),
+      judgeScore,
+    };
+    storeValidatedWorkerModel(result);
+    log.info(
+      `worker model validated: ${candidate.id} (judge=${judgeScore}) for provider ${input.providerID}`,
+    );
+    return result;
+  }
+
+  // No candidate passed — clear any stale stored result so we don't keep
+  // routing worker calls to a potentially-deprecated model.
+  clearValidatedWorkerModel(input.providerID);
+  log.info(
+    `worker model validation: no candidate passed for ${input.providerID} — cleared stale entry`,
+  );
+  return null;
+}
+
 // ---------------------------------------------------------------------------
 // Effective worker model resolution
 // ---------------------------------------------------------------------------
@@ -253,9 +387,12 @@ export function resolveWorkerModel(
   // Explicit override wins
   if (configWorkerModel) return configWorkerModel;
 
-  // Check for validated auto-selection
+  // Check for validated auto-selection.
+  // Don't trust entries older than 24h — model may have been deprecated.
+  // Validation will re-run on next idle cycle and either re-confirm or clear.
   const validated = getValidatedWorkerModel(providerID);
-  if (validated) {
+  const MAX_AGE_MS = 24 * 60 * 60 * 1000;
+  if (validated && Date.now() - validated.validatedAt <= MAX_AGE_MS) {
     return { providerID: validated.providerID, modelID: validated.modelID };
   }