@loreai/core 0.15.0 → 0.17.0

package/dist/types/worker-model.d.ts

@@ -1,16 +1,17 @@
 /**
- * Dynamic worker model selection.
+ * Worker model resolution.
  *
- * Background workers (distillation, curation, query expansion) don't need
- * frontier reasoning. This module discovers cheaper models from the same
- * provider and validates their quality via a two-phase comparison:
- *   Phase 1: structural checks (parsability, observation count, token bounds)
- *   Phase 2: LLM judge (session model rates candidate output vs reference)
+ * Background workers (distillation, curation, query expansion) default to
+ * sonnet-4-6 when the session model is more expensive ($1.50+/M input).
+ * Sonnet-4-6 produces equivalent-quality distillations at lower cost.
+ * An explicit `workerModel` config override takes priority over this default.
  *
- * Results are persisted in kv_meta and re-evaluated when the model landscape
- * changes (new models, session model switch, model deprecation).
+ * Resolution order:
+ *   1. Explicit config override (`workerModel`)
+ *   2. Cost-aware default (sonnet-4 for expensive session models)
+ *   3. Session model fallback (same model as the conversation)
  */
-/** Minimal model info needed for worker selection — provider-agnostic. */
+/** Minimal model info — kept for downstream consumers. */
 export type ModelInfo = {
     id: string;
     providerID: string;
@@ -26,85 +27,19 @@ export type ModelInfo = {
         reasoning?: boolean;
     };
 };
-/** Result of a worker model validation stored in kv_meta. */
-export type WorkerModelResult = {
-    modelID: string;
-    providerID: string;
-    fingerprint: string;
-    validatedAt: number;
-    judgeScore: number | null;
-};
-/**
- * Select worker model candidates from the available models.
- *
- * Returns up to 2 candidates: cheapest overall + one tier below the session
- * model. The session model itself is included (if it's the cheapest, the list
- * has 1 entry and no comparison is needed).
- */
-export declare function selectWorkerCandidates(sessionModel: {
-    id: string;
-    providerID: string;
-    cost: {
-        input: number;
-    };
-}, providerModels: ModelInfo[]): ModelInfo[];
-/**
- * Compute a fingerprint from the model landscape. Changes when:
- * - Models are added or removed from the provider
- * - The session model changes
- */
-export declare function computeModelFingerprint(providerID: string, sessionModelID: string, activeModelIDs: string[]): string;
-export declare function getValidatedWorkerModel(providerID: string): WorkerModelResult | null;
-export declare function storeValidatedWorkerModel(result: WorkerModelResult): void;
-/** Clear a stored worker model validation (e.g. when the model is deprecated). */
-export declare function clearValidatedWorkerModel(providerID: string): void;
-/**
- * Check whether the stored validation is stale (fingerprint mismatch).
- */
-export declare function isValidationStale(stored: WorkerModelResult | null, currentFingerprint: string): boolean;
-export type StructuralCheckResult = {
-    passed: boolean;
-    observationCount: number;
-    tokenCount: number;
-    reason?: string;
-};
-/**
- * Structural quality check: does the candidate distillation output meet
- * minimum quality thresholds relative to the reference?
- */
-export declare function structuralCheck(candidateObservations: string | null, referenceObservations: string): StructuralCheckResult;
-export declare const WORKER_JUDGE_SYSTEM = "You are evaluating distillation quality. You will be given a REFERENCE distillation (produced by a capable model) and a CANDIDATE distillation (produced by a cheaper model) of the same conversation segment.\n\nRate the candidate on a scale of 1-5:\n5 = Captures all key facts and decisions, equivalent to reference\n4 = Captures most facts, minor omissions\n3 = Captures the essential facts, some detail loss acceptable\n2 = Missing important facts or technical details\n1 = Significantly incomplete or inaccurate\n\nRespond with ONLY a single digit (1-5).";
-export declare function workerJudgeUser(reference: string, candidate: string): string;
-/** Parse the judge's score from a response. Returns null on parse failure. */
-export declare function parseJudgeScore(response: string): number | null;
-import type { LLMClient } from "./types";
-export type ValidationInput = {
-    llm: LLMClient;
-    providerID: string;
-    sessionModelID: string;
-    candidates: ModelInfo[];
-    /** Recent gen-0 distillation to use as reference (observations text). */
-    referenceObservations: string;
-    /** Source messages text for re-running distillation with candidates. */
-    sourceMessagesText: string;
-    /** Date string for the distillation prompt. */
-    date: string;
-};
-/**
- * Run the two-phase quality validation for worker model candidates.
- * Returns the cheapest passing candidate, or null if none pass.
- */
-export declare function runValidation(input: ValidationInput): Promise<WorkerModelResult | null>;
 /**
  * Resolve the effective worker model for a given provider.
- * Priority: explicit config > validated auto-selection > session model (fallback).
+ * Priority: explicit config override > cost-aware default > session model.
  */
-export declare function resolveWorkerModel(providerID: string, configWorkerModel?: {
+export declare function resolveWorkerModel(_providerID: string, configWorkerModel?: {
     providerID: string;
     modelID: string;
 }, configModel?: {
     providerID: string;
     modelID: string;
+}, costAwareDefault?: {
+    providerID: string;
+    modelID: string;
 }): {
     providerID: string;
     modelID: string;

package/dist/types/worker-model.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"worker-model.d.ts","sourceRoot":"","sources":["../../src/worker-model.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAUH,0EAA0E;AAC1E,MAAM,MAAM,SAAS,GAAG;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE;QACZ,KAAK,EAAE;YAAE,IAAI,EAAE,OAAO,CAAA;SAAE,CAAC;QACzB,+DAA+D;QAC/D,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB,CAAC;CACH,CAAC;AAEF,6DAA6D;AAC7D,MAAM,MAAM,iBAAiB,GAAG;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B,CAAC;AAQF;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CACpC,YAAY,EAAE;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,EACzE,cAAc,EAAE,SAAS,EAAE,GAC1B,SAAS,EAAE,CA6Cb;AAMD;;;;GAIG;AACH,wBAAgB,uBAAuB,CACrC,UAAU,EAAE,MAAM,EAClB,cAAc,EAAE,MAAM,EACtB,cAAc,EAAE,MAAM,EAAE,GACvB,MAAM,CAKR;AAMD,wBAAgB,uBAAuB,CACrC,UAAU,EAAE,MAAM,GACjB,iBAAiB,GAAG,IAAI,CAU1B;AAED,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,iBAAiB,GAAG,IAAI,CAQzE;AAED,kFAAkF;AAClF,wBAAgB,yBAAyB,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAElE;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,iBAAiB,GAAG,IAAI,EAChC,kBAAkB,EAAE,MAAM,GACzB,OAAO,CAGT;AAMD,MAAM,MAAM,qBAAqB,GAAG;IAClC,MAAM,EAAE,OAAO,CAAC;IAChB,gBAAgB,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF;;;GAGG;AACH,wBAAgB,eAAe,CAC7B,qBAAqB,EAAE,MAAM,GAAG,IAAI,EACpC,qBAAqB,EAAE,MAAM,GAC5B,qBAAqB,CAsCvB;AAMD,eAAO,MAAM,mBAAmB,ijBASQ,CAAC;AAEzC,wBAAgB,eAAe,CAC7B,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,GAChB,MAAM,CAER;AAED,8EAA8E;AAC9E,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAI/D;AAOD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAEzC,MAAM,MAAM,eAAe,GAAG;IAC5B,GAAG,EAAE,SAAS,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,SAAS,EAAE,CAAC;IACxB,yEAAyE;IACzE,qBAAqB,EAAE,MAAM,CAAC;IAC9B,wEAAwE;IACxE,kBAAkB,EAAE,MAAM,CAAC;IAC3B,+CAA+C;IAC/C,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AAEF;;;GAGG;AACH,wBAAsB,aAAa,CACjC,KAAK,EAAE,eAAe,GACrB,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC,CAwFnC;AAMD;;;GAGG;AACH,wBAAgB,kBAAkB,CAChC,UAAU,EAAE,MAAM,EAClB,iBAAiB,CAAC,EAAE;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,EAC3D,WAAW,CAAC,EAAE;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GACpD;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAerD"}
+{"version":3,"file":"worker-model.d.ts","sourceRoot":"","sources":["../../src/worker-model.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAMH,0DAA0D;AAC1D,MAAM,MAAM,SAAS,GAAG;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE;QACZ,KAAK,EAAE;YAAE,IAAI,EAAE,OAAO,CAAA;SAAE,CAAC;QACzB,+DAA+D;QAC/D,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB,CAAC;CACH,CAAC;AAMF;;;GAGG;AACH,wBAAgB,kBAAkB,CAChC,WAAW,EAAE,MAAM,EACnB,iBAAiB,CAAC,EAAE;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,EAC3D,WAAW,CAAC,EAAE;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,EACrD,gBAAgB,CAAC,EAAE;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GACzD;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAUrD"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loreai/core",
-  "version": "0.15.0",
+  "version": "0.17.0",
   "type": "module",
   "license": "FSL-1.1-Apache-2.0",
   "description": "Shared memory engine for Lore — three-tier storage, distillation, gradient context management",
@@ -24,11 +24,14 @@
   },
   "dependencies": {
     "@huggingface/hub": "2.11.0",
-    "fastembed": "^2.1.0",
+    "micromark": "^4.0.0",
     "remark": "^15.0.1",
     "uuidv7": "^1.1.0",
     "zod": "^4.3.6"
   },
+  "optionalDependencies": {
+    "fastembed": "^2.1.0"
+  },
   "devDependencies": {
     "@types/mdast": "^4.0.4"
   },

package/src/agents-file.ts

@@ -8,8 +8,9 @@
  * without duplication.
  */
 
-import { existsSync, readFileSync, writeFileSync, mkdirSync } from "fs";
+import { existsSync, readFileSync, writeFileSync, mkdirSync, statSync } from "fs";
 import { dirname, join } from "path";
+import { db } from "./db";
 import * as ltm from "./ltm";
 import { serialize, inline, h, ul, liph, strong, t, root, unescapeMarkdown } from "./markdown";
 
@@ -50,6 +51,51 @@ const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/
 /** Matches `<!-- lore:UUID -->` tracking markers. */
 const MARKER_RE = /^<!--\s*lore:([0-9a-f-]+)\s*-->$/;
 
+// ---------------------------------------------------------------------------
+// File cache (kv_meta) — skip redundant import/export work
+// ---------------------------------------------------------------------------
+
+type LoreFileCache = {
+  /** File mtime (milliseconds) at last processing. */
+  mtimeMs: number;
+  /** hashSection() of file content at that time. */
+  hash: string;
+};
+
+const CACHE_PREFIX = "lore_file_cache:";
+
+function getCache(fp: string): LoreFileCache | null {
+  const row = db()
+    .query("SELECT value FROM kv_meta WHERE key = ?")
+    .get(CACHE_PREFIX + fp) as { value: string } | null;
+  if (!row) return null;
+  try {
+    return JSON.parse(row.value);
+  } catch {
+    return null;
+  }
+}
+
+function setCache(fp: string, entry: LoreFileCache): void {
+  const key = CACHE_PREFIX + fp;
+  const value = JSON.stringify(entry);
+  db()
+    .query(
+      "INSERT INTO kv_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = ?",
+    )
+    .run(key, value, value);
+}
+
+/**
+ * Clear the cached mtime/hash for a project's `.lore.md`.
+ * Useful in tests or after data wipes to force a full re-check.
+ */
+export function clearLoreFileCache(projectPath: string): void {
+  db()
+    .query("DELETE FROM kv_meta WHERE key = ?")
+    .run(CACHE_PREFIX + join(projectPath, LORE_FILE));
+}
+
 // ---------------------------------------------------------------------------
 // Types
 // ---------------------------------------------------------------------------
@@ -453,24 +499,61 @@ export function loreFileExists(projectPath: string): boolean {
 /**
  * Export current knowledge entries to `.lore.md` in the project root.
  * The entire file is lore-owned — no section markers, no content to preserve.
+ *
+ * Skips the write if the content hash matches the cached hash (DB state
+ * unchanged since last export), avoiding unnecessary filesystem writes
+ * and mtime bumps.
  */
 export function exportLoreFile(projectPath: string): void {
   const sectionBody = buildSection(projectPath);
   const content = LORE_FILE_HEADER + "\n" + sectionBody;
-  writeFileSync(join(projectPath, LORE_FILE), content, "utf8");
+  const contentHash = hashSection(content);
+
+  const fp = join(projectPath, LORE_FILE);
+
+  // Skip write if content hash matches cached hash (DB state unchanged).
+  const cached = getCache(fp);
+  if (cached && cached.hash === contentHash) {
+    return;
+  }
+
+  // Content changed — write and update cache.
+  writeFileSync(fp, content, "utf8");
+  const { mtimeMs } = statSync(fp);
+  setCache(fp, { mtimeMs, hash: contentHash });
 }
 
 /**
  * Returns true if `.lore.md` needs to be imported:
- * - File exists and its content differs from what lore would currently produce
+ * - File exists and its content differs from what lore would currently produce.
+ *
+ * Uses an mtime + content-hash cache to skip the expensive buildSection()
+ * call when the file hasn't been touched since we last processed it.
  */
 export function shouldImportLoreFile(projectPath: string): boolean {
   const fp = join(projectPath, LORE_FILE);
   if (!existsSync(fp)) return false;
 
+  // Fast path: if mtime hasn't changed since last processing, skip entirely.
+  const { mtimeMs } = statSync(fp);
+  const cached = getCache(fp);
+  if (cached && cached.mtimeMs === mtimeMs) {
+    return false;
+  }
+
+  // Slow path: mtime changed (or first check) — read file and compare content.
   const fileContent = readFileSync(fp, "utf8");
+  const fileHash = hashSection(fileContent);
   const expected = LORE_FILE_HEADER + "\n" + buildSection(projectPath);
-  return hashSection(fileContent) !== hashSection(expected);
+  const expectedHash = hashSection(expected);
+
+  if (fileHash === expectedHash) {
+    // File matches DB — update cache so next call fast-paths.
+    setCache(fp, { mtimeMs, hash: fileHash });
+    return false;
+  }
+
+  return true;
 }
 
 /**

package/src/config.ts

@@ -35,8 +35,20 @@ export const LoreConfig = z.object({
        *  cost-aware formula from targetCacheReadCostPerTurn. 0 = disabled
        *  (no cap, use full context). Default: undefined (use cost-aware auto). */
       maxLayer0Tokens: z.number().min(0).optional(),
+      /** Target maximum cost (USD) for a single full cache bust (cold write of
+       *  the entire context). Controls the total-context token cap at layer 1+:
+       *  maxContextTokens = targetBustCost / cacheWriteCostPerToken.
+       *  For opus-4-6 ($6.25/M write): $1.00 → 160K cap.
+       *  For sonnet-4 ($3.75/M write): $1.00 → 267K (effectively uncapped at 200K).
+       *  The cap is further adjusted dynamically per session based on observed
+       *  bust rate (EMA) and break frequency. Default: 1.00. Set to 0 to disable. */
+      targetBustCost: z.number().min(0).default(1.00),
+      /** Direct override for the total-context token cap at layer 1+. When set,
+       *  bypasses the cost-aware formula from targetBustCost. 0 = disabled.
+       *  Default: undefined (use cost-aware auto). */
+      maxContextTokens: z.number().min(0).optional(),
     })
-    .default({ distilled: 0.25, raw: 0.4, output: 0.25, ltm: 0.05, targetCacheReadCostPerTurn: 0.10 }),
+    .default({ distilled: 0.25, raw: 0.4, output: 0.25, ltm: 0.05, targetCacheReadCostPerTurn: 0.10, targetBustCost: 1.00 }),
   /**
    * Cold-cache idle-resume handling.
    *
@@ -62,8 +74,15 @@ export const LoreConfig = z.object({
   distillation: z
     .object({
       minMessages: z.number().min(3).default(5),
-      maxSegment: z.number().min(5).default(30),
-      metaThreshold: z.number().min(3).default(10),
+      /** Minimum total tokens for a segment to be worth distilling.
+       *  Segments below this are deferred (normal mode) or absorbed without
+       *  an LLM call (force/urgent mode). Default: 64. */
+      minSegmentTokens: z.number().min(16).default(64),
+      /** Maximum total tokens per distillation segment. Segments exceeding
+       *  this are split at time-gap or token boundaries. Replaces the former
+       *  message-count-based maxSegment. Default: 8192. */
+      maxSegmentTokens: z.number().min(256).default(16384),
+      metaThreshold: z.number().min(3).default(20),
       /** Max chars per tool output when rendering temporal messages for distillation input.
        *  Outputs longer than this are replaced with a compact annotation preserving line
        *  count, error signals, and file paths. Default: 2000 (matches upstream OpenCode's
@@ -72,8 +91,9 @@ export const LoreConfig = z.object({
     })
     .default({
       minMessages: 5,
-      maxSegment: 30,
-      metaThreshold: 10,
+      minSegmentTokens: 64,
+      maxSegmentTokens: 16384,
+      metaThreshold: 20,
       toolOutputMaxChars: 2_000,
     }),
   knowledge: z
@@ -148,12 +168,55 @@ export const LoreConfig = z.object({
           model: "BGESmallENV15",
           dimensions: 384,
         }),
+      /** Recall output formatting — controls how search results are presented to the agent. */
+      recall: z
+        .object({
+          /** Total character budget for recall output. Controls how much context the
+           *  recall results consume. ~2K tokens at 8000 chars. Default: 8000. */
+          charBudget: z.number().min(2000).max(20000).default(8000),
+          /** Minimum RRF score relative to top result. Results below
+           *  topScore * relevanceFloor are dropped. Default: 0.15.
+           *  Set to 0 to disable score-based cutoff. */
+          relevanceFloor: z.number().min(0).max(1).default(0.15),
+          /** Max results to show in recall output. Default: 15. */
+          maxResults: z.number().min(3).max(30).default(15),
+        })
+        .default({ charBudget: 8000, relevanceFloor: 0.15, maxResults: 15 }),
     })
     .default({
       ftsWeights: { title: 6.0, content: 2.0, category: 3.0 },
       recallLimit: 10,
       queryExpansion: false,
       embeddings: { enabled: true, provider: "local" as const, model: "BGESmallENV15", dimensions: 384 },
+      recall: { charBudget: 8000, relevanceFloor: 0.15, maxResults: 15 },
+    }),
+  cache: z
+    .object({
+      /** TTL for the conversation cache breakpoint.
+       *  - "5m" — standard Anthropic ephemeral (5 min eviction, 1.25× write cost)
+       *  - "1h" — extended 1-hour TTL (2× write cost, requires extended cache tier)
+       *  - "auto" — auto-upgrade to 1h when frequent cold-cache turns are detected.
+       *    Monitors rolling window of recent turns; upgrades when >40% are cold-cache,
+       *    downgrades when <20%. Auto-syncs idleResumeMinutes to 60 when 1h is active.
+       *  Default: "auto". */
+      conversationTTL: z.enum(["5m", "1h", "auto"]).default("auto"),
+      /** Speculative cache warming — sends max_tokens:0 keepalive requests to
+       *  refresh the Anthropic prompt cache before it expires. Uses survival
+       *  analysis on inter-turn gaps to predict whether the user will return. */
+      warming: z
+        .object({
+          /** Enable cache warming. Default: true. */
+          enabled: z.boolean().default(true),
+          /** Override the survival probability threshold below which warming is
+           *  skipped. Default: auto-derived from cache read/write cost ratio
+           *  (~0.08 for 5m TTL, ~0.05 for 1h TTL). */
+          minReturnProbability: z.number().min(0).max(1).optional(),
+        })
+        .default({ enabled: true }),
+    })
+    .default({
+      conversationTTL: "auto",
+      warming: { enabled: true },
     }),
   crossProject: z.boolean().default(false),
   agentsFile: z

package/src/curator.ts

@@ -82,7 +82,7 @@ export async function run(input: {
   const responseText = await input.llm.prompt(
     CURATOR_SYSTEM,
     userContent,
-    { model, workerID: "lore-curator", thinking: false, sessionID: input.sessionID },
+    { model, workerID: "lore-curator", thinking: false, sessionID: input.sessionID, maxTokens: 2048 },
   );
   if (!responseText) return { created: 0, updated: 0, deleted: 0 };
 
@@ -185,7 +185,7 @@ export async function consolidate(input: {
   const responseText = await input.llm.prompt(
     CONSOLIDATION_SYSTEM,
     userContent,
-    { model, workerID: "lore-curator", thinking: false, sessionID: input.sessionID },
+    { model, workerID: "lore-curator", thinking: false, sessionID: input.sessionID, maxTokens: 4096 },
   );
   if (!responseText) return { updated: 0, deleted: 0 };