@loreai/core 0.11.1 → 0.13.0

package/src/index.ts

@@ -15,6 +15,7 @@ export * as distillation from "./distillation";
 export * as curator from "./curator";
 export * as embedding from "./embedding";
 export * as latReader from "./lat-reader";
+export * as patternExtract from "./pattern-extract";
 export * as log from "./log";
 
 export {
@@ -72,6 +73,7 @@ export {
   getLastTransformEstimate,
   toolStripAnnotation,
   onIdleResume,
+  getLastTurnAt,
   consumeCameOutOfIdle,
   // Test-only — exposed at the barrel so host-package tests can simulate idle
   // gaps without sleeping. Not part of the public API.
@@ -93,13 +95,18 @@ export {
   COMPACT_SUMMARY_TEMPLATE,
   buildCompactPrompt,
 } from "./prompt";
-export { shouldImport, importFromFile, exportToFile } from "./agents-file";
+export {
+  shouldImport,
+  importFromFile,
+  exportToFile,
+  exportLoreFile,
+  importLoreFile,
+  shouldImportLoreFile,
+  loreFileExists,
+  LORE_FILE,
+} from "./agents-file";
 export { workerSessionIDs, isWorkerSession } from "./worker";
 export * as workerModel from "./worker-model";
-export {
-  WORKER_JUDGE_SYSTEM,
-  workerJudgeUser,
-} from "./worker-model";
 export {
   ftsQuery,
   ftsQueryOr,
@@ -107,6 +114,7 @@ export {
   reciprocalRankFusion,
   expandQuery,
   extractTopTerms,
+  exactTermMatchRank,
 } from "./search";
 export {
   serialize,

package/src/pattern-extract.ts

@@ -0,0 +1,108 @@
+/**
+ * Lightweight regex-based pattern extraction from distillation observations.
+ *
+ * Scans for decision/preference/choice patterns and returns structured
+ * extractions that can be stored as knowledge entries. No LLM required.
+ *
+ * Patterns target how decisions and preferences are typically expressed
+ * in distilled engineering context:
+ *   - "decided to use X"
+ *   - "chose X over Y"
+ *   - "switched from X to Y"
+ *   - "prefers X for Y"
+ *   - "going with X because Y"
+ *
+ * Extracted entries participate in the normal curator cycle — the curator
+ * can consolidate or remove them based on actual value. The extraction is
+ * a cheap seed, not a permanent fixture.
+ */
+
+export type ExtractedPattern = {
+  category: "decision" | "preference";
+  /** Short descriptive title, e.g. "Chose PostgreSQL over MySQL". */
+  title: string;
+  /** Full matched text for context. */
+  content: string;
+};
+
+type PatternDef = {
+  regex: RegExp;
+  category: "decision" | "preference";
+  titleFn: (match: RegExpMatchArray) => string;
+};
+
+const PATTERNS: PatternDef[] = [
+  // Decision patterns
+  {
+    regex: /decided to (?:use |switch to |go with |adopt )(.+?)(?:\.|,|$)/gi,
+    category: "decision",
+    titleFn: (m) => `Decided to use ${m[1].trim()}`,
+  },
+  {
+    regex: /chose (.+?) over (.+?)(?:\.|,|$)/gi,
+    category: "decision",
+    titleFn: (m) => `Chose ${m[1].trim()} over ${m[2].trim()}`,
+  },
+  {
+    regex: /switched from (.+?) to (.+?)(?:\.|,|$)/gi,
+    category: "decision",
+    titleFn: (m) => `Switched from ${m[1].trim()} to ${m[2].trim()}`,
+  },
+  {
+    regex: /going with (.+?) (?:because|for|due to)(.+?)(?:\.|,|$)/gi,
+    category: "decision",
+    titleFn: (m) => `Going with ${m[1].trim()}`,
+  },
+  {
+    regex: /migrat(?:ed|ing) (?:from .+? )?to (.+?)(?:\.|,|$)/gi,
+    category: "decision",
+    titleFn: (m) => `Migrated to ${m[1].trim()}`,
+  },
+  {
+    regex: /adopted (.+?) (?:for|as|instead)(.+?)(?:\.|,|$)/gi,
+    category: "decision",
+    titleFn: (m) => `Adopted ${m[1].trim()}`,
+  },
+
+  // Preference patterns
+  {
+    regex: /prefers? (.+?) (?:over|to|instead of|rather than) (.+?)(?:\.|,|$)/gi,
+    category: "preference",
+    titleFn: (m) => `Prefers ${m[1].trim()} over ${m[2].trim()}`,
+  },
+  {
+    regex:
+      /(?:user |team |we )(?:always |usually |typically )(?:use|prefer|go with) (.+?)(?:\.|,|$)/gi,
+    category: "preference",
+    titleFn: (m) => `Typically uses ${m[1].trim()}`,
+  },
+];
+
+/**
+ * Extract decision/preference patterns from distillation observations text.
+ *
+ * Returns structured entries suitable for `ltm.create()`. Deduplicates by
+ * lowercased title within a single call.
+ *
+ * @param observations  The distilled observations text to scan.
+ * @returns             Array of extracted patterns (may be empty).
+ */
+export function extractPatterns(observations: string): ExtractedPattern[] {
+  const results: ExtractedPattern[] = [];
+  const seen = new Set<string>();
+
+  for (const { regex, category, titleFn } of PATTERNS) {
+    // Reset lastIndex for global regexes reused across calls
+    regex.lastIndex = 0;
+    let match: RegExpMatchArray | null;
+    while ((match = regex.exec(observations)) !== null) {
+      const title = titleFn(match);
+      const key = title.toLowerCase();
+      if (seen.has(key)) continue;
+      seen.add(key);
+      results.push({ category, title, content: match[0].trim() });
+    }
+  }
+
+  return results;
+}

package/src/recall.ts

@@ -19,7 +19,9 @@ import type { LoreConfig } from "./config";
 import type { LLMClient } from "./types";
 import {
   EMPTY_QUERY,
+  exactTermMatchRank,
   expandQuery,
+  filterTerms,
   ftsQuery,
   ftsQueryOr,
   reciprocalRankFusion,
@@ -36,6 +38,7 @@ type Distillation = {
   generation: number;
   created_at: number;
   session_id: string;
+  c_norm: number | null;
 };
 
 export type ScoredDistillation = Distillation & { rank: number };
@@ -72,6 +75,41 @@ type TaggedResult =
   | { source: "temporal"; item: temporal.ScoredTemporalMessage }
   | { source: "lat-section"; item: latReader.ScoredLatSection };
 
+// ---------------------------------------------------------------------------
+// Tagged result helpers (used by exact-match boost + formatting)
+// ---------------------------------------------------------------------------
+
+/** Extract searchable text from any TaggedResult variant. */
+function getTaggedText(tagged: TaggedResult): string {
+  switch (tagged.source) {
+    case "knowledge":
+    case "cross-knowledge":
+      return `${tagged.item.title} ${tagged.item.content}`;
+    case "distillation":
+      return tagged.item.observations;
+    case "temporal":
+      return tagged.item.content;
+    case "lat-section":
+      return `${tagged.item.heading} ${tagged.item.content}`;
+  }
+}
+
+/** Unified key function for TaggedResult — source-prefixed ID for RRF dedup. */
+function taggedResultKey(r: TaggedResult): string {
+  switch (r.source) {
+    case "knowledge":
+      return `k:${r.item.id}`;
+    case "cross-knowledge":
+      return `xk:${r.item.id}`;
+    case "distillation":
+      return `d:${r.item.id}`;
+    case "temporal":
+      return `t:${r.item.id}`;
+    case "lat-section":
+      return `lat:${r.item.id}`;
+  }
+}
+
 // ---------------------------------------------------------------------------
 // Distillation search
 // ---------------------------------------------------------------------------
@@ -93,8 +131,8 @@ function searchDistillationsLike(input: {
     .join(" AND ");
   const likeParams = terms.map((term) => `%${term}%`);
   const sql = input.sessionID
-    ? `SELECT id, observations, generation, created_at, session_id FROM distillations WHERE project_id = ? AND session_id = ? AND ${conditions} ORDER BY created_at DESC LIMIT ?`
-    : `SELECT id, observations, generation, created_at, session_id FROM distillations WHERE project_id = ? AND ${conditions} ORDER BY created_at DESC LIMIT ?`;
+    ? `SELECT id, observations, generation, created_at, session_id, c_norm FROM distillations WHERE project_id = ? AND session_id = ? AND ${conditions} ORDER BY created_at DESC LIMIT ?`
+    : `SELECT id, observations, generation, created_at, session_id, c_norm FROM distillations WHERE project_id = ? AND ${conditions} ORDER BY created_at DESC LIMIT ?`;
   const allParams = input.sessionID
     ? [input.pid, input.sessionID, ...likeParams, input.limit]
     : [input.pid, ...likeParams, input.limit];
@@ -115,13 +153,13 @@ function searchDistillationsScored(input: {
   if (q === EMPTY_QUERY) return [];
 
   const ftsSQL = input.sessionID
-    ? `SELECT d.id, d.observations, d.generation, d.created_at, d.session_id, rank
+    ? `SELECT d.id, d.observations, d.generation, d.created_at, d.session_id, d.c_norm, rank
        FROM distillation_fts f
        CROSS JOIN distillations d ON d.rowid = f.rowid
        WHERE distillation_fts MATCH ?
        AND d.project_id = ? AND d.session_id = ?
        ORDER BY rank LIMIT ?`
-    : `SELECT d.id, d.observations, d.generation, d.created_at, d.session_id, rank
+    : `SELECT d.id, d.observations, d.generation, d.created_at, d.session_id, d.c_norm, rank
        FROM distillation_fts f
        CROSS JOIN distillations d ON d.rowid = f.rowid
        WHERE distillation_fts MATCH ?
@@ -241,7 +279,7 @@ export async function runRecall(input: RecallInput): Promise<RecallResult> {
   let queries = [query];
   if (searchConfig?.queryExpansion && llm) {
     try {
-      queries = await expandQuery(llm, query);
+      queries = await expandQuery(llm, query, undefined, sessionID);
     } catch (err) {
       log.info("recall: query expansion failed, using original:", err);
     }
@@ -322,6 +360,24 @@ export async function runRecall(input: RecallInput): Promise<RecallResult> {
         key: (r) => `t:${r.item.id}`,
       },
     );
+
+    // Recency-biased list for temporal results: same candidates re-ranked
+    // by created_at (newest first). RRF naturally boosts messages that
+    // appear in both the BM25 and recency lists — i.e. results that are
+    // both semantically relevant AND recent. Uses the same `t:` key prefix
+    // so RRF merges rather than duplicates.
+    if (temporalResults.length > 0) {
+      const recencySorted = [...temporalResults].sort(
+        (a, b) => b.created_at - a.created_at,
+      );
+      allRrfLists.push({
+        items: recencySorted.map((item) => ({
+          source: "temporal" as const,
+          item,
+        })),
+        key: (r) => `t:${r.item.id}`,
+      });
+    }
   }
 
   // Vector search on the original query (not expansions — avoid redundant embeds).
@@ -358,7 +414,7 @@ export async function runRecall(input: RecallInput): Promise<RecallResult> {
           .map((hit): TaggedResult | null => {
             const row = db()
               .query(
-                "SELECT id, observations, generation, created_at, session_id FROM distillations WHERE id = ?",
+                "SELECT id, observations, generation, created_at, session_id, c_norm FROM distillations WHERE id = ?",
               )
               .get(hit.id) as Distillation | null;
             if (!row) return null;
@@ -430,6 +486,86 @@ export async function runRecall(input: RecallInput): Promise<RecallResult> {
     }
   }
 
+  // Distillation quality list: rank distillation candidates by a quality score
+  // that combines temporal clustering (c_norm) and age. Segments with low c_norm
+  // (uniformly distributed timestamps) are considered higher quality than bursty
+  // segments (high c_norm). Among high-c_norm segments, recent ones are more
+  // likely relevant. This adds a mild signal — RRF naturally blends it with the
+  // BM25 and vector signals without overriding them.
+  {
+    const distillationCandidates: Array<{
+      tagged: TaggedResult;
+      key: string;
+      qualityScore: number;
+    }> = [];
+
+    for (const list of allRrfLists) {
+      for (const item of list.items) {
+        if (item.source !== "distillation") continue;
+        const key = `d:${item.item.id}`;
+        const d = item.item as ScoredDistillation;
+        const cNorm = d.c_norm ?? 0; // NULL → treat as uniform (best case)
+        // Quality score: lower c_norm is better. For high c_norm, recency
+        // partially compensates. Age is normalized to days (capped at 90).
+        const ageDays = Math.min(
+          (Date.now() - d.created_at) / 86_400_000,
+          90,
+        );
+        // score ∈ [0, ~1]: 0 = best quality (uniform + recent)
+        // c_norm dominates (0–1), age adds a mild 0–0.1 penalty
+        const score = cNorm + (ageDays / 90) * 0.1;
+        distillationCandidates.push({ tagged: item, key, qualityScore: score });
+      }
+    }
+
+    if (distillationCandidates.length > 1) {
+      // De-duplicate by key (same distillation may appear in BM25 + vector lists)
+      const seen = new Set<string>();
+      const unique = distillationCandidates.filter((c) => {
+        if (seen.has(c.key)) return false;
+        seen.add(c.key);
+        return true;
+      });
+
+      // Sort by quality: lowest score first (best quality)
+      unique.sort((a, b) => a.qualityScore - b.qualityScore);
+
+      allRrfLists.push({
+        items: unique.map((c) => c.tagged),
+        key: (r) => `d:${r.item.id}`,
+      });
+    }
+  }
+
+  // Exact-match boost: add an additional RRF list that ranks candidates by
+  // the number of exact query term matches. This boosts proper nouns, file
+  // names, and technical terms that BM25's prefix/stem matching may dilute.
+  // Only runs when there are meaningful terms and existing candidates.
+  if (filterTerms(query).length > 0 && allRrfLists.length > 0) {
+    // Collect unique candidates across all lists
+    const allCandidates = new Map<string, TaggedResult>();
+    for (const list of allRrfLists) {
+      for (const item of list.items) {
+        const key = list.key(item);
+        if (!allCandidates.has(key)) allCandidates.set(key, item);
+      }
+    }
+
+    const candidateEntries = [...allCandidates.entries()];
+    const exactRanked = exactTermMatchRank(
+      candidateEntries,
+      ([, tagged]) => getTaggedText(tagged),
+      query,
+    );
+
+    if (exactRanked.length) {
+      allRrfLists.push({
+        items: exactRanked.map(([, item]) => item),
+        key: taggedResultKey,
+      });
+    }
+  }
+
   const fused = reciprocalRankFusion<TaggedResult>(allRrfLists);
   return formatFusedResults(fused, 20);
 }

package/src/search.ts

@@ -267,6 +267,41 @@ export function reciprocalRankFusion<T>(
   return [...scores.values()].sort((a, b) => b.score - a.score);
 }
 
+// ---------------------------------------------------------------------------
+// Exact term match ranking (Phase 5 — MemPalace-inspired keyword boost)
+// ---------------------------------------------------------------------------
+
+/**
+ * Score candidates by exact query term overlap.
+ *
+ * Returns items sorted by number of exact term matches (descending).
+ * Used as an additional RRF list to boost results that contain query terms
+ * verbatim — important for proper nouns, file names, and technical terms
+ * that BM25's prefix matching + Porter stemming can miss or dilute.
+ *
+ * Terms are filtered through the standard stopword + single-char filter
+ * (same as `ftsQuery`), then matched case-insensitively via `includes()`.
+ */
+export function exactTermMatchRank<T>(
+  items: T[],
+  getText: (item: T) => string,
+  query: string,
+): T[] {
+  const terms = filterTerms(query).map((t) => t.toLowerCase());
+  if (!terms.length) return [];
+
+  const scored = items
+    .map((item) => {
+      const text = getText(item).toLowerCase();
+      const matches = terms.filter((t) => text.includes(t)).length;
+      return { item, matches };
+    })
+    .filter((s) => s.matches > 0)
+    .sort((a, b) => b.matches - a.matches);
+
+  return scored.map((s) => s.item);
+}
+
 // ---------------------------------------------------------------------------
 // LLM query expansion (Phase 4)
 // ---------------------------------------------------------------------------
@@ -290,6 +325,7 @@ export async function expandQuery(
   llm: LLMClient,
   query: string,
   model?: { providerID: string; modelID: string },
+  sessionID?: string,
 ): Promise<string[]> {
   const TIMEOUT_MS = 3000;
 
@@ -299,7 +335,7 @@ export async function expandQuery(
       llm.prompt(
         QUERY_EXPANSION_SYSTEM,
         `Input: "${query}"`,
-        { model, workerID: "lore-query-expand" },
+        { model, workerID: "lore-query-expand", thinking: false, urgent: true, sessionID },
       ),
       new Promise<null>((resolve) => setTimeout(() => resolve(null), TIMEOUT_MS)),
     ]);

package/src/temporal.ts

@@ -280,6 +280,45 @@ export function searchScored(input: {
   }
 }
 
+/**
+ * Normalized variance of relative-existence weights over message timestamps.
+ *
+ * Measures temporal attention imbalance: 0 means timestamps are evenly
+ * distributed (uniform attention), 1 means a single distant timestamp
+ * dominates (attention stuck in the past). Useful as a lightweight
+ * signal for distillation segmentation, recall time-biasing, and
+ * idle-resume awareness.
+ *
+ * Only meaningful for n ≥ 2. Returns 0 for 0 or 1 timestamps.
+ *
+ * Based on the "Temporal Clustering via Relative Existence" heuristic
+ * from D7x7z49/llm-context-idea.
+ */
+export function temporalCnorm(
+  timestamps: number[],
+  now: number = Date.now(),
+): number {
+  const n = timestamps.length;
+  if (n < 2) return 0;
+
+  // Existence durations: how long each piece has existed
+  const durations = timestamps.map((t) => now - t);
+  const totalDuration = durations.reduce((a, b) => a + b, 0);
+  if (totalDuration <= 0) return 0;
+
+  // Relative existence weights (positive, sum to 1)
+  const weights = durations.map((d) => d / totalDuration);
+
+  // Normalized variance: Var(w) / Var_max
+  // Var(w) = (1/n) * Σ(w_i - 1/n)²
+  // Var_max = (n-1) / n²  (when one weight = 1, rest = 0)
+  const uniform = 1 / n;
+  const variance =
+    weights.reduce((sum, w) => sum + (w - uniform) ** 2, 0) / n;
+  const maxVariance = (n - 1) / (n * n);
+  return maxVariance === 0 ? 0 : variance / maxVariance;
+}
+
 export function count(projectPath: string, sessionID?: string): number {
   const pid = ensureProject(projectPath);
   const query = sessionID

package/src/types.ts

@@ -189,7 +189,7 @@ export interface LLMClient {
    *
    * @param system  System prompt text
    * @param user    User message text
-   * @param opts    Optional model selection and worker identification
+   * @param opts    Optional model selection, worker identification, and thinking control
    * @returns The assistant's text response, or null on failure
    */
   prompt(
@@ -203,6 +203,46 @@ export interface LLMClient {
        * (e.g. OpenCode uses this as the session agent name).
        */
       workerID?: string;
+      /**
+       * Disable extended thinking/reasoning for this call.
+       *
+       * Background workers discard thinking tokens — they only extract the
+       * text response. Setting `thinking: false` tells the adapter to avoid
+       * producing (and billing for) thinking tokens when possible.
+       *
+       * Adapter behavior:
+       * - Gateway: no-op (bare API call never triggers thinking)
+       * - Pi: passes `thinkingEnabled: false` to `complete()`
+       * - OpenCode: cannot honor — SDK has no thinking toggle on session.prompt();
+       *   relies on Part A (non-reasoning model selection) instead
+       */
+      thinking?: boolean;
+      /**
+       * When true, the request must be processed immediately and the result
+       * returned before the next user turn. When false or absent, the request
+       * may be deferred to a batch queue for cost savings (50% discount via
+       * Anthropic's Message Batches API).
+       *
+       * Callers that `await` the result for a blocking operation (compaction,
+       * overflow recovery, query expansion) should set `urgent: true`.
+       * Fire-and-forget background work (incremental distillation, idle
+       * curation) should leave it unset or set `false`.
+       *
+       * Only the gateway's BatchLLMClient honors this flag; other adapters
+       * (OpenCode, Pi) ignore it and always process immediately.
+       */
+      urgent?: boolean;
+      /**
+       * Session identifier for per-session auth credential lookup.
+       *
+       * The gateway uses this to resolve the correct API key or OAuth
+       * token for the session that triggered the work, preventing
+       * cross-session key mixups when multiple clients are connected.
+       *
+       * Other adapters (OpenCode, Pi) ignore this field — they resolve
+       * auth through their own mechanisms.
+       */
+      sessionID?: string;
     },
   ): Promise<string | null>;
 }