@loreai/core 0.17.1 → 0.19.0

package/src/pattern-extract.ts

@@ -12,6 +12,13 @@
  *   - "prefers X for Y"
  *   - "going with X because Y"
  *
+ * Also matches process instruction patterns from distilled observations
+ * where the observer normalizes user assertions:
+ *   - "User stated always X"
+ *   - "User said never Y"
+ *   - "User stated make sure to X"
+ *   - "User stated don't forget to X"
+ *
  * Extracted entries participate in the normal curator cycle — the curator
  * can consolidate or remove them based on actual value. The extraction is
  * a cheap seed, not a permanent fixture.
@@ -76,6 +83,33 @@ const PATTERNS: PatternDef[] = [
     category: "preference",
     titleFn: (m) => `Typically uses ${m[1].trim()}`,
   },
+
+  // Process instruction patterns — match distilled observations recording
+  // user assertions about workflow/process rules. The distillation observer
+  // normalizes user instructions into "User stated always X" phrasing.
+  // These require "stated/asserted/said" to avoid overlapping with the
+  // existing "typically uses" pattern above (which already handles
+  // "user always use/prefer/go with X").
+  {
+    regex: /(?:user |team |we )(?:stated |asserted |said )(?:to )?always (.+?)(?:\.|,|$)/gi,
+    category: "preference",
+    titleFn: (m) => `Always ${m[1].trim()}`,
+  },
+  {
+    regex: /(?:user |team |we )(?:stated |asserted |said )(?:to )?never (.+?)(?:\.|,|$)/gi,
+    category: "preference",
+    titleFn: (m) => `Never ${m[1].trim()}`,
+  },
+  {
+    regex: /(?:user |team |we )(?:stated |asserted |said )(?:to )?make sure to (.+?)(?:\.|,|$)/gi,
+    category: "preference",
+    titleFn: (m) => `Make sure to ${m[1].trim()}`,
+  },
+  {
+    regex: /(?:user |team |we )(?:stated |asserted |said )(?:to )?(?:don't|do not) forget (?:to )?(.+?)(?:\.|,|$)/gi,
+    category: "preference",
+    titleFn: (m) => `Always ${m[1].trim()}`,
+  },
 ];
 
 /**
@@ -96,6 +130,13 @@ export function extractPatterns(observations: string): ExtractedPattern[] {
     regex.lastIndex = 0;
     let match: RegExpMatchArray | null;
     while ((match = regex.exec(observations)) !== null) {
+      // Skip false positives: template placeholders (e.g. "X", "Y"),
+      // quoted fragments, or very short captures that are clearly not
+      // real technology/tool names. Plain apostrophes (') are allowed
+      // since they appear in valid names like "Bun's test runner".
+      const captures = match.slice(1);
+      if (captures.some((c) => c && (c.trim().length <= 2 || /["\u201C\u201D`\u2018\u2019]/.test(c)))) continue;
+
       const title = titleFn(match);
       const key = title.toLowerCase();
       if (seen.has(key)) continue;

package/src/prompt.ts

@@ -222,6 +222,10 @@ Focus ONLY on knowledge that helps a coding agent work effectively on THIS codeb
 - Environment/tooling setup details that affect development
 - Important relationships between components that aren't obvious from reading the code
 - User preferences and working style specific to how they use this project
+- Repeated user instructions — when the user says things like "always", "never",
+  "make sure to", "don't forget to", these are high-value preference candidates.
+  If you see instruction-like language, prioritize extracting it as a "preference" entry.
+  These instructions represent how the user wants to work and should persist across sessions.
 
 Do NOT extract:
 - Task-specific details (file currently being edited, current bug being fixed)
@@ -316,7 +320,9 @@ IMPORTANT:
 2. When updating, REPLACE the content with a complete rewrite — never append.
 3. If entries cover the same system from different angles, merge them: update one, delete the rest.
 4. Only create a new entry for genuinely distinct knowledge with no existing home.
-5. Keep all entries under 150 words. If an existing entry is too long, use an update op to trim it.`;
+5. Keep all entries under 150 words. If an existing entry is too long, use an update op to trim it.
+6. Pay special attention to user instructions ("always do X", "never do Y", "make sure to X").
+   These are strong signals for "preference" entries with high confidence.`;
 }
 
 /**

package/src/recall.ts

@@ -475,14 +475,27 @@ export async function searchRecall(
     }
   }
 
+  // Determine vector boost weight: for queries with enough meaningful terms,
+  // boost vector search lists so semantic similarity outweighs keyword noise.
+  const queryTermCount = filterTerms(query).length;
+  const vectorWeight =
+    queryTermCount >= (searchConfig?.vectorBoostMinTerms ?? 3)
+      ? (searchConfig?.vectorBoostWeight ?? 1.5)
+      : 1;
+
   // Collect per-query RRF lists. Original query is always first; if expansion
   // produced extras, we still weight the original twice by adding both original
   // and expanded lists (RRF naturally weights items appearing in more lists).
   const allRrfLists: Array<{
     items: TaggedResult[];
     key: (r: TaggedResult) => string;
+    weight?: number;
   }> = [];
 
+  // Track where primary (first-query) lists end so the MAX_RRF_LISTS cap
+  // trims expanded-query lists first, preserving vector/supplemental lists.
+  let primaryListEnd = 0;
+
   for (const q of queries) {
     const knowledgeResults: ltm.ScoredKnowledgeEntry[] = [];
     if (knowledgeEnabled && scope !== "session") {
@@ -568,7 +581,15 @@ export async function searchRecall(
         key: (r) => `t:${r.item.id}`,
       });
     }
+
+    // Mark the end of the first (original) query's lists. Supplemental lists
+    // (vector, lat.md, cross-project, quality, exact-match) are appended after
+    // the loop and should be preserved over expanded-query lists when capping.
+    if (primaryListEnd === 0) {
+      primaryListEnd = allRrfLists.length;
+    }
   }
+  const perQueryListEnd = allRrfLists.length;
 
   // Vector search on the original query (not expansions — avoid redundant embeds).
   if (embedding.isAvailable() && scope !== "session") {
@@ -593,6 +614,7 @@ export async function searchRecall(
           allRrfLists.push({
             items: vectorTagged,
             key: (r) => `k:${r.item.id}`,
+            weight: vectorWeight,
           });
         }
       }
@@ -618,6 +640,7 @@ export async function searchRecall(
           allRrfLists.push({
             items: distVectorTagged,
             key: (r) => `d:${r.item.id}`,
+            weight: vectorWeight,
           });
         }
       }
@@ -648,6 +671,7 @@ export async function searchRecall(
           allRrfLists.push({
             items: temporalVectorTagged,
             key: (r) => `t:${r.item.id}`,
+            weight: vectorWeight,
           });
         }
       }
@@ -786,6 +810,25 @@ export async function searchRecall(
     }
   }
 
+  // Cap the number of RRF lists to prevent score inflation from marginal items.
+  // With query expansion (3 queries × 4 sources + supplemental lists), the list
+  // count can exceed 15. Each list gives marginal items enough cumulative RRF
+  // score to clear the relevance floor.
+  //
+  // Priority: primary (original query BM25 + recency) and supplemental
+  // (vector, lat.md, cross-project, quality, exact-match) are high-value.
+  // Expanded-query BM25 lists are lowest priority — trim those first.
+  const MAX_RRF_LISTS = 10;
+  if (allRrfLists.length > MAX_RRF_LISTS) {
+    // Layout: [0..primaryListEnd) = primary, [primaryListEnd..perQueryEnd) = expanded, [perQueryEnd..) = supplemental
+    const primary = allRrfLists.slice(0, primaryListEnd);
+    const expanded = allRrfLists.slice(primaryListEnd, perQueryListEnd);
+    const supplemental = allRrfLists.slice(perQueryListEnd);
+    const budget = Math.max(0, MAX_RRF_LISTS - primary.length - supplemental.length);
+    allRrfLists.length = 0;
+    allRrfLists.push(...primary, ...expanded.slice(0, budget), ...supplemental);
+  }
+
   const fused = reciprocalRankFusion<TaggedResult>(allRrfLists);
 
   // Cap output: return at most 3x the per-source limit. With 7+ RRF sources
@@ -885,11 +928,6 @@ export async function runRecall(input: RecallInput): Promise<RecallResult> {
     return recallById(input.id);
   }
 
-  // Short-circuit vague queries — stopwords-only would match everything.
-  if (ftsQuery(input.query) === EMPTY_QUERY) {
-    return "Query too vague — try using specific keywords, file names, or technical terms.";
-  }
-
   const fused = await searchRecall(input);
   const recallCfg = input.searchConfig?.recall;
   return formatFusedResults(fused, {

package/src/search.ts

@@ -302,29 +302,31 @@ export function normalizeRank(
 /**
  * Reciprocal Rank Fusion: merge multiple ranked lists into a single ranked list.
  *
- * RRF score = Σ(1 / (k + rank_i)) for each list where the item appears.
+ * RRF score = Σ(weight / (k + rank_i)) for each list where the item appears.
  * k = 60 is standard (from Cormack et al., 2009; also used by QMD).
  *
  * RRF is rank-based, not score-based — raw score magnitude differences across
  * different FTS5 tables don't matter. Only relative ordering within each list.
  *
- * @param lists  Each list provides items (in ranked order) and a key function
- *               for deduplication. Items at the front of the array are rank 0.
+ * @param lists  Each list provides items (in ranked order), a key function
+ *               for deduplication, and an optional weight (default 1).
+ *               Items at the front of the array are rank 0.
  * @param k      Smoothing constant. Default 60.
  * @returns      Fused list sorted by RRF score descending. When items appear
  *               in multiple lists, the first occurrence's item is kept.
  */
 export function reciprocalRankFusion<T>(
-  lists: Array<{ items: T[]; key: (item: T) => string }>,
+  lists: Array<{ items: T[]; key: (item: T) => string; weight?: number }>,
   k = 60,
 ): Array<{ item: T; score: number }> {
   const scores = new Map<string, { item: T; score: number }>();
 
   for (const list of lists) {
+    const w = list.weight ?? 1;
     for (let rank = 0; rank < list.items.length; rank++) {
       const item = list.items[rank];
       const id = list.key(item);
-      const rrfScore = 1 / (k + rank);
+      const rrfScore = w / (k + rank);
       const existing = scores.get(id);
       if (existing) {
         existing.score += rrfScore;

package/src/session-limiter.ts

@@ -0,0 +1,47 @@
+/**
+ * Per-key concurrency limiter using p-limit.
+ *
+ * Each key (typically a session ID) gets its own p-limit(1) instance,
+ * serializing async operations on the same key while allowing different
+ * keys to run fully in parallel.
+ *
+ * Two independent limiter pools are provided — one for distillation and
+ * one for curation — so they don't block each other.
+ */
+
+import pLimit from "p-limit";
+
+type LimitFunction = ReturnType<typeof pLimit>;
+
+function createLimiterPool() {
+  const limiters = new Map<string, LimitFunction>();
+
+  /** Get or create a p-limit(1) limiter for the given key. */
+  function get(key: string): LimitFunction {
+    let limiter = limiters.get(key);
+    if (!limiter) {
+      limiter = pLimit(1);
+      limiters.set(key, limiter);
+    }
+    return limiter;
+  }
+
+  /** Check if a limiter for `key` is currently busy (active or pending work). */
+  function isBusy(key: string): boolean {
+    const limiter = limiters.get(key);
+    return limiter ? limiter.activeCount + limiter.pendingCount > 0 : false;
+  }
+
+  /** Clear all limiters (for test cleanup). */
+  function clear(): void {
+    limiters.clear();
+  }
+
+  return { get, isBusy, clear };
+}
+
+/** Serializes distillation.run() and metaDistill() per session. */
+export const distillLimiter = createLimiterPool();
+
+/** Serializes curator.run() per session with skip-if-busy semantics. */
+export const curatorLimiter = createLimiterPool();

package/src/temporal.ts

@@ -171,6 +171,8 @@ export function markDistilled(ids: string[]) {
     .run(...ids);
 }
 
+// Only searches undistilled messages — distilled content is already represented
+// in distillation search results and would duplicate/dilute temporal hits.
 // LIKE-based fallback for when FTS5 fails unexpectedly.
 function searchLike(input: {
   pid: string;
@@ -186,8 +188,8 @@ function searchLike(input: {
   const conditions = terms.map(() => "LOWER(content) LIKE ?").join(" AND ");
   const likeParams = terms.map((t) => `%${t}%`);
   const query = input.sessionID
-    ? `SELECT * FROM temporal_messages WHERE project_id = ? AND session_id = ? AND ${conditions} ORDER BY created_at DESC LIMIT ?`
-    : `SELECT * FROM temporal_messages WHERE project_id = ? AND ${conditions} ORDER BY created_at DESC LIMIT ?`;
+    ? `SELECT * FROM temporal_messages WHERE project_id = ? AND session_id = ? AND distilled = 0 AND ${conditions} ORDER BY created_at DESC LIMIT ?`
+    : `SELECT * FROM temporal_messages WHERE project_id = ? AND distilled = 0 AND ${conditions} ORDER BY created_at DESC LIMIT ?`;
   const params = input.sessionID
     ? [input.pid, input.sessionID, ...likeParams, input.limit]
     : [input.pid, ...likeParams, input.limit];
@@ -208,11 +210,11 @@ export function search(input: {
   const ftsSQL = input.sessionID
     ? `SELECT m.* FROM temporal_fts f
        CROSS JOIN temporal_messages m ON m.rowid = f.rowid
-       WHERE f.content MATCH ? AND m.project_id = ? AND m.session_id = ?
+       WHERE f.content MATCH ? AND m.project_id = ? AND m.session_id = ? AND m.distilled = 0
        ORDER BY rank LIMIT ?`
     : `SELECT m.* FROM temporal_fts f
        CROSS JOIN temporal_messages m ON m.rowid = f.rowid
-       WHERE f.content MATCH ? AND m.project_id = ?
+       WHERE f.content MATCH ? AND m.project_id = ? AND m.distilled = 0
        ORDER BY rank LIMIT ?`;
 
   try {
@@ -251,11 +253,11 @@ export function searchScored(input: {
   const ftsSQL = input.sessionID
     ? `SELECT m.*, rank FROM temporal_fts f
        CROSS JOIN temporal_messages m ON m.rowid = f.rowid
-       WHERE f.content MATCH ? AND m.project_id = ? AND m.session_id = ?
+       WHERE f.content MATCH ? AND m.project_id = ? AND m.session_id = ? AND m.distilled = 0
        ORDER BY rank LIMIT ?`
     : `SELECT m.*, rank FROM temporal_fts f
        CROSS JOIN temporal_messages m ON m.rowid = f.rowid
-       WHERE f.content MATCH ? AND m.project_id = ?
+       WHERE f.content MATCH ? AND m.project_id = ? AND m.distilled = 0
        ORDER BY rank LIMIT ?`;
 
   try {
@@ -322,6 +324,16 @@ export function count(projectPath: string, sessionID?: string): number {
   ).count;
 }
 
+/** Quick existence check — true if any temporal messages exist for this session. */
+export function hasMessages(projectPath: string, sessionID: string): boolean {
+  const pid = ensureProject(projectPath);
+  return !!db()
+    .query(
+      "SELECT 1 FROM temporal_messages WHERE project_id = ? AND session_id = ? LIMIT 1",
+    )
+    .get(pid, sessionID);
+}
+
 export function undistilledCount(
   projectPath: string,
   sessionID?: string,

package/src/types.ts

@@ -183,7 +183,7 @@ export type LoreMessageWithParts = {
  * Host adapters implement this:
  * - OpenCode: wraps `client.session.create()` + `client.session.prompt()`
  * - Pi: wraps `complete()` from `@mariozechner/pi-ai`
- * - Standalone: direct `fetch()` to provider APIs
+ * - Gateway: direct `fetch()` to provider APIs
  */
 export interface LLMClient {
   /**