@loreai/core 0.11.1 → 0.12.0

package/dist/bun/db.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../../src/db.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAyVtC,wBAAgB,EAAE,IAAI,QAAQ,CAkC7B;AAqCD,wBAAgB,KAAK,SAKpB;AAGD,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,CAYjE;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAK1D;AAED,2DAA2D;AAC3D,wBAAgB,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAKrD;AAED;;;GAGG;AACH,wBAAgB,UAAU,IAAI,OAAO,CAKpC;AAMD;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAK3D;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAYxE"}
+{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../../src/db.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAyVtC,wBAAgB,EAAE,IAAI,QAAQ,CAkC7B;AAmGD,wBAAgB,KAAK,SAKpB;AAGD,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,CAYjE;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAK1D;AAED,2DAA2D;AAC3D,wBAAgB,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAKrD;AAED;;;GAGG;AACH,wBAAgB,UAAU,IAAI,OAAO,CAKpC;AAMD;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAK3D;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAYxE"}

package/dist/bun/distillation.d.ts

@@ -3,6 +3,32 @@ import { workerSessionIDs } from "./worker";
 import type { LLMClient } from "./types";
 export { workerSessionIDs };
 type TemporalMessage = temporal.TemporalMessage;
+/**
+ * Compression health ratio: k / √N.
+ *
+ * k = distilled token count, N = source token count.
+ * Values < 1.0 signal likely lossy compression (below the square-root
+ * boundary). Values > 1.0 signal relatively faithful compression.
+ *
+ * Based on the "LLM Context Square Root Theory" heuristic from
+ * D7x7z49/llm-context-idea. The specific threshold is unvalidated —
+ * use as a diagnostic signal, not a hard gate.
+ */
+export declare function compressionRatio(distilledTokens: number, sourceTokens: number): number;
+/**
+ * Segment detection: group related messages into distillation-sized chunks.
+ *
+ * When the message count exceeds `maxSegment`, prefers splitting at the
+ * largest inter-message time gap (if it's ≥ 3× the median gap) to respect
+ * natural conversation boundaries. Falls back to count-based splitting at
+ * `maxSegment` when timestamps are uniform.
+ *
+ * Trailing segments with < 3 messages are merged into the previous segment
+ * to avoid tiny distillation inputs with too little context.
+ *
+ * Exported for testing; `run()` is the production caller.
+ */
+export declare function detectSegments(messages: TemporalMessage[], maxSegment: number): TemporalMessage[][];
 /**
  * Truncate tool outputs within a `TemporalMessage.content` string (produced
  * by `temporal.partsToText`). Plain text and `[reasoning]` chunks pass

package/dist/bun/distillation.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"distillation.d.ts","sourceRoot":"","sources":["../../src/distillation.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,QAAQ,MAAM,YAAY,CAAC;AAWvC,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAC5C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAGzC,OAAO,EAAE,gBAAgB,EAAE,CAAC;AAE5B,KAAK,eAAe,GAAG,QAAQ,CAAC,eAAe,CAAC;AAmDhD;;;;;;;;;;;;GAYG;AACH,wBAAgB,4BAA4B,CAC1C,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,GACf,MAAM,CAwBR;AAgBD;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,eAAe,EAAE,EAC3B,kBAAkB,CAAC,EAAE,MAAM,GAC1B,MAAM,CAUR;AAED,KAAK,kBAAkB,GAAG;IACxB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAwBF;;;;;;;;;;;;GAYG;AACH,wBAAgB,sBAAsB,CACpC,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,GAChB,MAAM,GAAG,SAAS,CAEpB;AAwBD,+EAA+E;AAC/E,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CAQpD;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF;;;;;;;;;;;;;GAaG;AACH,wBAAgB,cAAc,CAC5B,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,eAAe,UAAQ,GACtB,YAAY,EAAE,CAqBhB;AA0ID,wBAAsB,GAAG,CAAC,KAAK,EAAE;IAC/B,GAAG,EAAE,SAAS,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IAChD,qEAAqE;IACrE,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB,GAAG,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC,CA4DjD;AAsDD;;;;;;;GAOG;AACH,wBAAsB,WAAW,CAAC,KAAK,EAAE;IACvC,GAAG,EAAE,SAAS,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;CACjD,GAAG,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC,CA+ErC"}
+{"version":3,"file":"distillation.d.ts","sourceRoot":"","sources":["../../src/distillation.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,QAAQ,MAAM,YAAY,CAAC;AAWvC,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAC5C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAGzC,OAAO,EAAE,gBAAgB,EAAE,CAAC;AAE5B,KAAK,eAAe,GAAG,QAAQ,CAAC,eAAe,CAAC;AAEhD;;;;;;;;;;GAUG;AACH,wBAAgB,gBAAgB,CAC9B,eAAe,EAAE,MAAM,EACvB,YAAY,EAAE,MAAM,GACnB,MAAM,CAGR;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,eAAe,EAAE,EAC3B,UAAU,EAAE,MAAM,GACjB,eAAe,EAAE,EAAE,CAGrB;AAwGD;;;;;;;;;;;;GAYG;AACH,wBAAgB,4BAA4B,CAC1C,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,GACf,MAAM,CAwBR;AAgBD;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,eAAe,EAAE,EAC3B,kBAAkB,CAAC,EAAE,MAAM,GAC1B,MAAM,CAUR;AAED,KAAK,kBAAkB,GAAG;IACxB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAwBF;;;;;;;;;;;;GAYG;AACH,wBAAgB,sBAAsB,CACpC,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,GAChB,MAAM,GAAG,SAAS,CAEpB;AAwBD,+EAA+E;AAC/E,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CAQpD;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF;;;;;;;;;;;;;GAaG;AACH,wBAAgB,cAAc,CAC5B,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,eAAe,UAAQ,GACtB,YAAY,EAAE,CAqBhB;AA0ID,wBAAsB,GAAG,CAAC,KAAK,EAAE;IAC/B,GAAG,EAAE,SAAS,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IAChD,qEAAqE;IACrE,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB,GAAG,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC,CA4DjD;AAmED;;;;;;;GAOG;AACH,wBAAsB,WAAW,CAAC,KAAK,EAAE;IACvC,GAAG,EAAE,SAAS,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;CACjD,GAAG,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC,CA+ErC"}

package/dist/bun/index.js

@@ -131,6 +131,7 @@ __export(temporal_exports, {
   search: () => search2,
   searchScored: () => searchScored,
   store: () => store,
+  temporalCnorm: () => temporalCnorm,
   undistilled: () => undistilled,
   undistilledCount: () => undistilledCount
 });
@@ -508,16 +509,47 @@ function migrate(database) {
     "SELECT name FROM sqlite_master WHERE type='table' AND name='schema_version'"
   ).get();
   const current2 = row ? database.query("SELECT version FROM schema_version").get()?.version ?? 0 : 0;
-  if (current2 >= MIGRATIONS.length) return;
+  if (current2 >= MIGRATIONS.length) {
+    recoverMissingObjects(database);
+    return;
+  }
   for (let i = current2; i < MIGRATIONS.length; i++) {
     if (i === VACUUM_MIGRATION_INDEX) {
       database.exec("PRAGMA auto_vacuum = INCREMENTAL");
       database.exec("VACUUM");
     } else {
-      database.exec(MIGRATIONS[i]);
+      try {
+        database.exec(MIGRATIONS[i]);
+      } catch (e) {
+        if (e instanceof Error && /duplicate column name/i.test(e.message)) {
+          const stripped = stripAppliedAlters(MIGRATIONS[i], database);
+          if (stripped.trim()) database.exec(stripped);
+        } else {
+          throw e;
+        }
+      }
     }
   }
   database.exec(`UPDATE schema_version SET version = ${MIGRATIONS.length}`);
+  recoverMissingObjects(database);
+}
+function stripAppliedAlters(migration, database) {
+  return migration.replace(
+    /ALTER\s+TABLE\s+(\w+)\s+ADD\s+COLUMN\s+(\w+)\b[^;]*;/gi,
+    (match, table, column) => {
+      const cols = database.query(`PRAGMA table_info(${table})`).all();
+      if (cols.some((c) => c.name === column)) return "";
+      return match;
+    }
+  );
+}
+function recoverMissingObjects(database) {
+  database.exec(`
+    CREATE TABLE IF NOT EXISTS kv_meta (
+      key TEXT PRIMARY KEY,
+      value TEXT NOT NULL
+    );
+  `);
 }
 function close() {
   if (instance) {
@@ -11423,6 +11455,18 @@ function searchScored(input) {
     return [];
   }
 }
+function temporalCnorm(timestamps, now = Date.now()) {
+  const n = timestamps.length;
+  if (n < 2) return 0;
+  const durations = timestamps.map((t2) => now - t2);
+  const totalDuration = durations.reduce((a, b) => a + b, 0);
+  if (totalDuration <= 0) return 0;
+  const weights = durations.map((d) => d / totalDuration);
+  const uniform = 1 / n;
+  const variance = weights.reduce((sum, w) => sum + (w - uniform) ** 2, 0) / n;
+  const maxVariance = (n - 1) / (n * n);
+  return maxVariance === 0 ? 0 : variance / maxVariance;
+}
 function count(projectPath, sessionID) {
   const pid = ensureProject(projectPath);
   const query = sessionID ? "SELECT COUNT(*) as count FROM temporal_messages WHERE project_id = ? AND session_id = ?" : "SELECT COUNT(*) as count FROM temporal_messages WHERE project_id = ?";
@@ -26740,6 +26784,8 @@ function check2(projectPath) {
 // src/distillation.ts
 var distillation_exports = {};
 __export(distillation_exports, {
+  compressionRatio: () => compressionRatio,
+  detectSegments: () => detectSegments,
   latestMetaObservations: () => latestMetaObservations,
   loadForSession: () => loadForSession,
   messagesToText: () => messagesToText,
@@ -27597,25 +27643,50 @@ function isWorkerSession(sessionID) {
 }
 
 // src/distillation.ts
+function compressionRatio(distilledTokens, sourceTokens) {
+  if (sourceTokens <= 0) return 0;
+  return distilledTokens / Math.sqrt(sourceTokens);
+}
 function detectSegments(messages, maxSegment) {
   if (messages.length <= maxSegment) return [messages];
-  const segments = [];
-  let current2 = [];
-  for (const msg of messages) {
-    current2.push(msg);
-    if (current2.length >= maxSegment) {
-      segments.push(current2);
-      current2 = [];
-    }
-  }
-  if (current2.length > 0) {
-    if (current2.length < 3 && segments.length > 0) {
-      segments[segments.length - 1].push(...current2);
-    } else {
-      segments.push(current2);
+  return splitSegments(messages, maxSegment);
+}
+var MIN_SEGMENT = 3;
+var GAP_THRESHOLD_MULTIPLIER = 3;
+function splitSegments(messages, maxSegment) {
+  if (messages.length <= maxSegment) return [messages];
+  const splitIdx = findSplitIndex(messages, maxSegment);
+  const left = messages.slice(0, splitIdx);
+  const right = messages.slice(splitIdx);
+  const result = splitSegments(left, maxSegment);
+  if (right.length < MIN_SEGMENT) {
+    result[result.length - 1].push(...right);
+  } else {
+    result.push(...splitSegments(right, maxSegment));
+  }
+  return result;
+}
+function findSplitIndex(messages, maxSegment) {
+  const gaps = [];
+  for (let i = 1; i < messages.length; i++) {
+    gaps.push({
+      index: i,
+      gap: messages[i].created_at - messages[i - 1].created_at
+    });
+  }
+  if (gaps.length === 0) return maxSegment;
+  const sortedGaps = gaps.map((g) => g.gap).sort((a, b) => a - b);
+  const medianGap = sortedGaps[Math.floor(sortedGaps.length / 2)];
+  let bestGap = { index: -1, gap: 0 };
+  for (const g of gaps) {
+    if (g.gap > bestGap.gap && g.index >= MIN_SEGMENT && messages.length - g.index >= MIN_SEGMENT) {
+      bestGap = g;
     }
   }
-  return segments;
+  if (bestGap.index > 0 && bestGap.gap >= medianGap * GAP_THRESHOLD_MULTIPLIER) {
+    return bestGap.index;
+  }
+  return maxSegment;
 }
 function formatTime(ms) {
   const d = new Date(ms);
@@ -27853,6 +27924,13 @@ async function distillSegment(input) {
     generation: 0
   });
   markDistilled(input.messages.map((m) => m.id));
+  const distilledTokens = Math.ceil(result.observations.length / 3);
+  const sourceTokens = input.messages.reduce((sum, m) => sum + m.tokens, 0);
+  const rComp = compressionRatio(distilledTokens, sourceTokens);
+  const cNorm = temporalCnorm(input.messages.map((m) => m.created_at));
+  info(
+    `distill segment: ${input.messages.length} msgs, ${sourceTokens}\u2192${distilledTokens} tokens, R=${rComp.toFixed(2)}, C_norm=${cNorm.toFixed(3)}`
+  );
   if (isAvailable()) {
     embedDistillation(distillId, result.observations);
   }
@@ -28217,6 +28295,18 @@ async function runRecall(input) {
         key: (r) => `t:${r.item.id}`
       }
     );
+    if (temporalResults.length > 0) {
+      const recencySorted = [...temporalResults].sort(
+        (a, b) => b.created_at - a.created_at
+      );
+      allRrfLists.push({
+        items: recencySorted.map((item) => ({
+          source: "temporal",
+          item
+        })),
+        key: (r) => `t:${r.item.id}`
+      });
+    }
   }
   if (isAvailable() && scope !== "session") {
     try {