@ulrichc1/sparn 1.2.2 → 1.4.0

package/dist/daemon/index.js

@@ -1,5 +1,5 @@
 // src/daemon/index.ts
-import { appendFileSync as appendFileSync2, existsSync as existsSync2, unlinkSync } from "fs";
+import { appendFileSync as appendFileSync2, existsSync as existsSync3, readFileSync as readFileSync2, unlinkSync, writeFileSync as writeFileSync2 } from "fs";
 
 // src/core/kv-memory.ts
 import { copyFileSync, existsSync } from "fs";
@@ -23,6 +23,7 @@ async function createKVMemory(dbPath) {
     const integrityCheck = db.pragma("quick_check", { simple: true });
     if (integrityCheck !== "ok") {
       console.error("\u26A0 Database corruption detected!");
+      db.close();
       if (existsSync(dbPath)) {
         const backupPath = createBackup(dbPath);
         if (backupPath) {
@@ -30,7 +31,6 @@ async function createKVMemory(dbPath) {
         }
       }
       console.log("Attempting database recovery...");
-      db.close();
       db = new Database(dbPath);
     }
   } catch (error) {
@@ -42,6 +42,7 @@ async function createKVMemory(dbPath) {
     db = new Database(dbPath);
   }
   db.pragma("journal_mode = WAL");
+  db.pragma("foreign_keys = ON");
   db.exec(`
     CREATE TABLE IF NOT EXISTS entries_index (
       id TEXT PRIMARY KEY NOT NULL,
@@ -81,6 +82,36 @@ async function createKVMemory(dbPath) {
     CREATE INDEX IF NOT EXISTS idx_entries_timestamp ON entries_index(timestamp DESC);
     CREATE INDEX IF NOT EXISTS idx_stats_timestamp ON optimization_stats(timestamp DESC);
   `);
+  db.exec(`
+    CREATE VIRTUAL TABLE IF NOT EXISTS entries_fts USING fts5(id, content, tokenize='porter');
+  `);
+  db.exec(`
+    CREATE TRIGGER IF NOT EXISTS entries_fts_insert
+    AFTER INSERT ON entries_value
+    BEGIN
+      INSERT OR REPLACE INTO entries_fts(id, content) VALUES (NEW.id, NEW.content);
+    END;
+  `);
+  db.exec(`
+    CREATE TRIGGER IF NOT EXISTS entries_fts_delete
+    AFTER DELETE ON entries_value
+    BEGIN
+      DELETE FROM entries_fts WHERE id = OLD.id;
+    END;
+  `);
+  db.exec(`
+    CREATE TRIGGER IF NOT EXISTS entries_fts_update
+    AFTER UPDATE ON entries_value
+    BEGIN
+      DELETE FROM entries_fts WHERE id = OLD.id;
+      INSERT INTO entries_fts(id, content) VALUES (NEW.id, NEW.content);
+    END;
+  `);
+  db.exec(`
+    INSERT OR IGNORE INTO entries_fts(id, content)
+    SELECT id, content FROM entries_value
+    WHERE id NOT IN (SELECT id FROM entries_fts);
+  `);
   const putIndexStmt = db.prepare(`
     INSERT OR REPLACE INTO entries_index
     (id, hash, timestamp, score, ttl, state, accessCount, isBTSP)
@@ -169,14 +200,20 @@ async function createKVMemory(dbPath) {
         sql += " AND i.isBTSP = ?";
         params.push(filters.isBTSP ? 1 : 0);
       }
+      if (filters.tags && filters.tags.length > 0) {
+        for (const tag of filters.tags) {
+          sql += " AND v.tags LIKE ?";
+          params.push(`%"${tag}"%`);
+        }
+      }
       sql += " ORDER BY i.score DESC";
       if (filters.limit) {
         sql += " LIMIT ?";
         params.push(filters.limit);
-      }
-      if (filters.offset) {
-        sql += " OFFSET ?";
-        params.push(filters.offset);
+        if (filters.offset) {
+          sql += " OFFSET ?";
+          params.push(filters.offset);
+        }
       }
       const stmt = db.prepare(sql);
       const rows = stmt.all(...params);
@@ -211,7 +248,22 @@ async function createKVMemory(dbPath) {
     },
     async compact() {
       const before = db.prepare("SELECT COUNT(*) as count FROM entries_index").get();
-      db.exec("DELETE FROM entries_index WHERE ttl <= 0");
+      const now = Date.now();
+      db.prepare("DELETE FROM entries_index WHERE isBTSP = 0 AND (timestamp + ttl * 1000) < ?").run(
+        now
+      );
+      db.exec("DELETE FROM entries_index WHERE isBTSP = 0 AND ttl <= 0");
+      const candidates = db.prepare("SELECT id, timestamp, ttl FROM entries_index WHERE isBTSP = 0").all();
+      for (const row of candidates) {
+        const ageSeconds = Math.max(0, (now - row.timestamp) / 1e3);
+        const ttlSeconds = row.ttl;
+        if (ttlSeconds <= 0) continue;
+        const decay = 1 - Math.exp(-ageSeconds / ttlSeconds);
+        if (decay >= 0.95) {
+          db.prepare("DELETE FROM entries_index WHERE id = ?").run(row.id);
+        }
+      }
+      db.exec("DELETE FROM entries_value WHERE id NOT IN (SELECT id FROM entries_index)");
       db.exec("VACUUM");
       const after = db.prepare("SELECT COUNT(*) as count FROM entries_index").get();
       return before.count - after.count;
@@ -231,6 +283,9 @@ async function createKVMemory(dbPath) {
         stats.entries_pruned,
         stats.duration_ms
       );
+      db.prepare(
+        "DELETE FROM optimization_stats WHERE id NOT IN (SELECT id FROM optimization_stats ORDER BY timestamp DESC LIMIT 1000)"
+      ).run();
     },
     async getOptimizationStats() {
       const stmt = db.prepare(`
@@ -243,16 +298,114 @@ async function createKVMemory(dbPath) {
     },
     async clearOptimizationStats() {
       db.exec("DELETE FROM optimization_stats");
+    },
+    async searchFTS(query, limit = 10) {
+      if (!query || query.trim().length === 0) return [];
+      const sanitized = query.replace(/[{}()[\]"':*^~]/g, " ").trim();
+      if (sanitized.length === 0) return [];
+      const stmt = db.prepare(`
+        SELECT
+          f.id, f.content, rank,
+          i.hash, i.timestamp, i.score, i.ttl, i.state, i.accessCount, i.isBTSP,
+          v.tags, v.metadata
+        FROM entries_fts f
+        JOIN entries_index i ON f.id = i.id
+        JOIN entries_value v ON f.id = v.id
+        WHERE entries_fts MATCH ?
+        ORDER BY rank
+        LIMIT ?
+      `);
+      try {
+        const rows = stmt.all(sanitized, limit);
+        return rows.map((r) => ({
+          entry: {
+            id: r.id,
+            content: r.content,
+            hash: r.hash,
+            timestamp: r.timestamp,
+            score: r.score,
+            ttl: r.ttl,
+            state: r.state,
+            accessCount: r.accessCount,
+            tags: r.tags ? JSON.parse(r.tags) : [],
+            metadata: r.metadata ? JSON.parse(r.metadata) : {},
+            isBTSP: r.isBTSP === 1
+          },
+          rank: r.rank
+        }));
+      } catch {
+        return [];
+      }
     }
   };
 }
 
+// src/utils/tokenizer.ts
+import { encode } from "gpt-tokenizer";
+var usePrecise = false;
+function setPreciseTokenCounting(enabled) {
+  usePrecise = enabled;
+}
+function estimateTokens(text) {
+  if (!text || text.length === 0) {
+    return 0;
+  }
+  if (usePrecise) {
+    return encode(text).length;
+  }
+  const words = text.split(/\s+/).filter((w) => w.length > 0);
+  const wordCount = words.length;
+  const charCount = text.length;
+  const charEstimate = Math.ceil(charCount / 4);
+  const wordEstimate = Math.ceil(wordCount * 0.75);
+  return Math.max(wordEstimate, charEstimate);
+}
+
 // src/daemon/consolidation-scheduler.ts
 import { appendFileSync } from "fs";
 
+// src/utils/tfidf.ts
+function tokenize(text) {
+  return text.toLowerCase().split(/\s+/).filter((word) => word.length > 0);
+}
+function calculateTF(term, tokens) {
+  const count = tokens.filter((t) => t === term).length;
+  return Math.sqrt(count);
+}
+function createTFIDFIndex(entries) {
+  const documentFrequency = /* @__PURE__ */ new Map();
+  for (const entry of entries) {
+    const tokens = tokenize(entry.content);
+    const uniqueTerms = new Set(tokens);
+    for (const term of uniqueTerms) {
+      documentFrequency.set(term, (documentFrequency.get(term) || 0) + 1);
+    }
+  }
+  return {
+    documentFrequency,
+    totalDocuments: entries.length
+  };
+}
+function scoreTFIDF(entry, index) {
+  const tokens = tokenize(entry.content);
+  if (tokens.length === 0) return 0;
+  const uniqueTerms = new Set(tokens);
+  let totalScore = 0;
+  for (const term of uniqueTerms) {
+    const tf = calculateTF(term, tokens);
+    const docsWithTerm = index.documentFrequency.get(term) || 0;
+    if (docsWithTerm === 0) continue;
+    const idf = Math.log(index.totalDocuments / docsWithTerm);
+    totalScore += tf * idf;
+  }
+  return totalScore / tokens.length;
+}
+
 // src/core/engram-scorer.ts
 function createEngramScorer(config2) {
   const { defaultTTL } = config2;
+  const recencyWindowMs = (config2.recencyBoostMinutes ?? 30) * 60 * 1e3;
+  const recencyMultiplier = config2.recencyBoostMultiplier ?? 1.3;
   function calculateDecay(ageInSeconds, ttlInSeconds) {
     if (ttlInSeconds === 0) return 1;
     if (ageInSeconds <= 0) return 0;
@@ -272,6 +425,13 @@ function createEngramScorer(config2) {
     if (entry.isBTSP) {
       score = Math.max(score, 0.9);
     }
+    if (!entry.isBTSP && recencyWindowMs > 0) {
+      const ageMs = currentTime - entry.timestamp;
+      if (ageMs >= 0 && ageMs < recencyWindowMs) {
+        const boostFactor = 1 + (recencyMultiplier - 1) * (1 - ageMs / recencyWindowMs);
+        score = score * boostFactor;
+      }
+    }
     return Math.max(0, Math.min(1, score));
   }
   function refreshTTL(entry) {
@@ -380,13 +540,15 @@ function createSleepCompressor() {
   function cosineSimilarity(text1, text2) {
     const words1 = tokenize(text1);
     const words2 = tokenize(text2);
-    const vocab = /* @__PURE__ */ new Set([...words1, ...words2]);
     const vec1 = {};
     const vec2 = {};
-    for (const word of vocab) {
-      vec1[word] = words1.filter((w) => w === word).length;
-      vec2[word] = words2.filter((w) => w === word).length;
+    for (const word of words1) {
+      vec1[word] = (vec1[word] ?? 0) + 1;
+    }
+    for (const word of words2) {
+      vec2[word] = (vec2[word] ?? 0) + 1;
     }
+    const vocab = /* @__PURE__ */ new Set([...words1, ...words2]);
     let dotProduct = 0;
     let mag1 = 0;
     let mag2 = 0;
@@ -402,9 +564,6 @@ function createSleepCompressor() {
     if (mag1 === 0 || mag2 === 0) return 0;
     return dotProduct / (mag1 * mag2);
   }
-  function tokenize(text) {
-    return text.toLowerCase().split(/\s+/).filter((word) => word.length > 0);
-  }
   return {
     consolidate,
     findDuplicates,
@@ -476,11 +635,10 @@ function createMetricsCollector() {
       ...metric
     };
   }
-  function calculatePercentile(values, percentile) {
-    if (values.length === 0) return 0;
-    const sorted = [...values].sort((a, b) => a - b);
-    const index = Math.ceil(percentile / 100 * sorted.length) - 1;
-    return sorted[index] || 0;
+  function calculatePercentile(sortedValues, percentile) {
+    if (sortedValues.length === 0) return 0;
+    const index = Math.ceil(percentile / 100 * sortedValues.length) - 1;
+    return sortedValues[index] || 0;
   }
   function getSnapshot() {
     const totalRuns = optimizations.length;
@@ -491,7 +649,7 @@ function createMetricsCollector() {
     );
     const totalTokensBefore = optimizations.reduce((sum, m) => sum + m.tokensBefore, 0);
     const averageReduction = totalTokensBefore > 0 ? totalTokensSaved / totalTokensBefore : 0;
-    const durations = optimizations.map((m) => m.duration);
+    const sortedDurations = optimizations.map((m) => m.duration).sort((a, b) => a - b);
     const totalCacheQueries = cacheHits + cacheMisses;
     const hitRate = totalCacheQueries > 0 ? cacheHits / totalCacheQueries : 0;
     return {
@@ -501,9 +659,9 @@ function createMetricsCollector() {
         totalDuration,
         totalTokensSaved,
         averageReduction,
-        p50Latency: calculatePercentile(durations, 50),
-        p95Latency: calculatePercentile(durations, 95),
-        p99Latency: calculatePercentile(durations, 99)
+        p50Latency: calculatePercentile(sortedDurations, 50),
+        p95Latency: calculatePercentile(sortedDurations, 95),
+        p99Latency: calculatePercentile(sortedDurations, 99)
       },
       cache: {
         hitRate,
@@ -559,6 +717,7 @@ function createConsolidationScheduler(options) {
   let lastRun = null;
   let lastResult = null;
   let nextRun = null;
+  let isRunning = false;
   function log2(message) {
     const timestamp = (/* @__PURE__ */ new Date()).toISOString();
     const logMessage = `[${timestamp}] [Consolidation] ${message}
@@ -572,6 +731,11 @@ function createConsolidationScheduler(options) {
     }
   }
   async function runConsolidation() {
+    if (isRunning) {
+      log2("Consolidation already in progress, skipping");
+      return;
+    }
+    isRunning = true;
     const startTime = Date.now();
     log2("Starting scheduled consolidation");
     try {
@@ -624,6 +788,7 @@ function createConsolidationScheduler(options) {
     if (intervalHours !== null && intervalHours > 0) {
       nextRun = Date.now() + intervalHours * 60 * 60 * 1e3;
     }
+    isRunning = false;
   }
   function start() {
     if (intervalHours === null || intervalHours <= 0) {
@@ -669,7 +834,15 @@ function createConsolidationScheduler(options) {
 }
 
 // src/daemon/session-watcher.ts
-import { readdirSync, statSync as statSync2, watch } from "fs";
+import {
+  existsSync as existsSync2,
+  mkdirSync,
+  readdirSync,
+  readFileSync,
+  statSync as statSync2,
+  watch,
+  writeFileSync
+} from "fs";
 import { homedir } from "os";
 import { dirname, join } from "path";
 
@@ -684,6 +857,11 @@ function hashContent(content) {
 
 // src/utils/context-parser.ts
 function parseClaudeCodeContext(context) {
+  const firstNonEmpty = context.split("\n").find((line) => line.trim().length > 0);
+  if (firstNonEmpty?.trim().startsWith("{")) {
+    const jsonlEntries = parseJSONLContext(context);
+    if (jsonlEntries.length > 0) return jsonlEntries;
+  }
   const entries = [];
   const now = Date.now();
   const lines = context.split("\n");
@@ -736,7 +914,7 @@ function createEntry(content, type, baseTime) {
     hash: hashContent(content),
     timestamp: baseTime,
     score: initialScore,
-    state: initialScore > 0.7 ? "active" : initialScore > 0.3 ? "ready" : "silent",
+    state: initialScore >= 0.7 ? "active" : initialScore >= 0.3 ? "ready" : "silent",
     ttl: 24 * 3600,
     // 24 hours default
     accessCount: 0,
@@ -745,52 +923,63 @@ function createEntry(content, type, baseTime) {
     isBTSP: false
   };
 }
-
-// src/utils/tokenizer.ts
-function estimateTokens(text) {
-  if (!text || text.length === 0) {
-    return 0;
+function parseJSONLLine(line) {
+  const trimmed = line.trim();
+  if (trimmed.length === 0) return null;
+  try {
+    return JSON.parse(trimmed);
+  } catch {
+    return null;
   }
-  const words = text.split(/\s+/).filter((w) => w.length > 0);
-  const wordCount = words.length;
-  const charCount = text.length;
-  const charEstimate = Math.ceil(charCount / 4);
-  const wordEstimate = Math.ceil(wordCount * 0.75);
-  return Math.max(wordEstimate, charEstimate);
+}
+function extractContent(content) {
+  if (typeof content === "string") return content;
+  if (Array.isArray(content)) {
+    return content.map((block) => {
+      if (block.type === "text" && block.text) return block.text;
+      if (block.type === "tool_use" && block.name) return `[tool_use: ${block.name}]`;
+      if (block.type === "tool_result") {
+        if (typeof block.content === "string") return block.content;
+        if (Array.isArray(block.content)) {
+          return block.content.filter((c) => c.type === "text" && c.text).map((c) => c.text).join("\n");
+        }
+      }
+      return "";
+    }).filter((s) => s.length > 0).join("\n");
+  }
+  return "";
+}
+function classifyJSONLMessage(msg) {
+  if (Array.isArray(msg.content)) {
+    const hasToolUse = msg.content.some((b) => b.type === "tool_use");
+    const hasToolResult = msg.content.some((b) => b.type === "tool_result");
+    if (hasToolUse) return "tool";
+    if (hasToolResult) return "result";
+  }
+  if (msg.type === "tool_use" || msg.tool_use) return "tool";
+  if (msg.type === "tool_result" || msg.tool_result) return "result";
+  if (msg.role === "user" || msg.role === "assistant") return "conversation";
+  return "other";
+}
+function parseJSONLContext(context) {
+  const entries = [];
+  const now = Date.now();
+  const lines = context.split("\n");
+  for (const line of lines) {
+    const msg = parseJSONLLine(line);
+    if (!msg) continue;
+    const content = extractContent(msg.content);
+    if (!content || content.trim().length === 0) continue;
+    const blockType = classifyJSONLMessage(msg);
+    entries.push(createEntry(content, blockType, now));
+  }
+  return entries;
 }
 
 // src/core/budget-pruner.ts
 function createBudgetPruner(config2) {
   const { tokenBudget, decay } = config2;
   const engramScorer = createEngramScorer(decay);
-  function tokenize(text) {
-    return text.toLowerCase().split(/\s+/).filter((word) => word.length > 0);
-  }
-  function calculateTF(term, tokens) {
-    const count = tokens.filter((t) => t === term).length;
-    return Math.sqrt(count);
-  }
-  function calculateIDF(term, allEntries) {
-    const totalDocs = allEntries.length;
-    const docsWithTerm = allEntries.filter((entry) => {
-      const tokens = tokenize(entry.content);
-      return tokens.includes(term);
-    }).length;
-    if (docsWithTerm === 0) return 0;
-    return Math.log(totalDocs / docsWithTerm);
-  }
-  function calculateTFIDF(entry, allEntries) {
-    const tokens = tokenize(entry.content);
-    if (tokens.length === 0) return 0;
-    const uniqueTerms = [...new Set(tokens)];
-    let totalScore = 0;
-    for (const term of uniqueTerms) {
-      const tf = calculateTF(term, tokens);
-      const idf = calculateIDF(term, allEntries);
-      totalScore += tf * idf;
-    }
-    return totalScore / tokens.length;
-  }
   function getStateMultiplier(entry) {
     if (entry.isBTSP) return 2;
     switch (entry.state) {
@@ -804,8 +993,8 @@ function createBudgetPruner(config2) {
         return 1;
     }
   }
-  function priorityScore(entry, allEntries) {
-    const tfidf = calculateTFIDF(entry, allEntries);
+  function priorityScore(entry, allEntries, index) {
+    const tfidf = index ? scoreTFIDF(entry, index) : scoreTFIDF(entry, createTFIDFIndex(allEntries));
     const currentScore = engramScorer.calculateScore(entry);
     const engramDecay = 1 - currentScore;
     const stateMultiplier = getStateMultiplier(entry);
@@ -824,15 +1013,33 @@ function createBudgetPruner(config2) {
     const originalTokens = entries.reduce((sum, e) => sum + estimateTokens(e.content), 0);
     const btspEntries = entries.filter((e) => e.isBTSP);
     const regularEntries = entries.filter((e) => !e.isBTSP);
-    const btspTokens = btspEntries.reduce((sum, e) => sum + estimateTokens(e.content), 0);
+    let includedBtsp = [];
+    let btspTokens = 0;
+    const sortedBtsp = [...btspEntries].sort((a, b) => b.timestamp - a.timestamp);
+    for (const entry of sortedBtsp) {
+      const tokens = estimateTokens(entry.content);
+      if (btspTokens + tokens <= budget * 0.8) {
+        includedBtsp.push(entry);
+        btspTokens += tokens;
+      }
+    }
+    if (includedBtsp.length === 0 && sortedBtsp.length > 0) {
+      const firstBtsp = sortedBtsp[0];
+      if (firstBtsp) {
+        includedBtsp = [firstBtsp];
+        btspTokens = estimateTokens(firstBtsp.content);
+      }
+    }
+    const excludedBtsp = btspEntries.filter((e) => !includedBtsp.includes(e));
+    const tfidfIndex = createTFIDFIndex(entries);
     const scored = regularEntries.map((entry) => ({
       entry,
-      score: priorityScore(entry, entries),
+      score: priorityScore(entry, entries, tfidfIndex),
       tokens: estimateTokens(entry.content)
     }));
     scored.sort((a, b) => b.score - a.score);
-    const kept = [...btspEntries];
-    const removed = [];
+    const kept = [...includedBtsp];
+    const removed = [...excludedBtsp];
     let currentTokens = btspTokens;
     for (const item of scored) {
       if (currentTokens + item.tokens <= budget) {
@@ -868,9 +1075,6 @@ function createIncrementalOptimizer(config2) {
     updateCount: 0,
     lastFullOptimization: Date.now()
   };
-  function tokenize(text) {
-    return text.toLowerCase().split(/\s+/).filter((word) => word.length > 0);
-  }
   function updateDocumentFrequency(entries, remove = false) {
     for (const entry of entries) {
       const tokens = tokenize(entry.content);
@@ -893,7 +1097,19 @@ function createIncrementalOptimizer(config2) {
     if (!cached) return null;
     return cached.entry;
   }
+  const MAX_CACHE_SIZE = 1e4;
   function cacheEntry(entry, score) {
+    if (state.entryCache.size >= MAX_CACHE_SIZE) {
+      const entries = Array.from(state.entryCache.entries());
+      entries.sort((a, b) => a[1].timestamp - b[1].timestamp);
+      const toRemove = Math.floor(MAX_CACHE_SIZE * 0.2);
+      for (let i = 0; i < toRemove && i < entries.length; i++) {
+        const entry2 = entries[i];
+        if (entry2) {
+          state.entryCache.delete(entry2[0]);
+        }
+      }
+    }
     state.entryCache.set(entry.hash, {
       entry,
       score,
@@ -1020,13 +1236,40 @@ function createIncrementalOptimizer(config2) {
       lastFullOptimization: state.lastFullOptimization
     };
   }
+  function serializeState() {
+    const s = getState();
+    return JSON.stringify({
+      entryCache: Array.from(s.entryCache.entries()),
+      documentFrequency: Array.from(s.documentFrequency.entries()),
+      totalDocuments: s.totalDocuments,
+      updateCount: s.updateCount,
+      lastFullOptimization: s.lastFullOptimization
+    });
+  }
+  function deserializeState(json) {
+    try {
+      const parsed = JSON.parse(json);
+      restoreState({
+        entryCache: new Map(parsed.entryCache),
+        documentFrequency: new Map(parsed.documentFrequency),
+        totalDocuments: parsed.totalDocuments,
+        updateCount: parsed.updateCount,
+        lastFullOptimization: parsed.lastFullOptimization
+      });
+      return true;
+    } catch {
+      return false;
+    }
+  }
   return {
     optimizeIncremental,
     optimizeFull,
     getState,
     restoreState,
     reset,
-    getStats
+    getStats,
+    serializeState,
+    deserializeState
   };
 }
 
@@ -1051,9 +1294,22 @@ function createContextPipeline(config2) {
     currentEntries = result.kept;
     budgetUtilization = result.budgetUtilization;
     if (currentEntries.length > windowSize) {
-      const sorted = [...currentEntries].sort((a, b) => b.timestamp - a.timestamp);
-      const toKeep = sorted.slice(0, windowSize);
-      const toRemove = sorted.slice(windowSize);
+      const timestamps = currentEntries.map((e) => e.timestamp);
+      const minTs = Math.min(...timestamps);
+      const maxTs = Math.max(...timestamps);
+      const tsRange = maxTs - minTs || 1;
+      const scored = currentEntries.map((entry) => {
+        if (entry.isBTSP) return { entry, hybridScore: 2 };
+        const ageNormalized = (entry.timestamp - minTs) / tsRange;
+        const hybridScore = ageNormalized * 0.4 + entry.score * 0.6;
+        return { entry, hybridScore };
+      });
+      scored.sort((a, b) => {
+        if (b.hybridScore !== a.hybridScore) return b.hybridScore - a.hybridScore;
+        return b.entry.timestamp - a.entry.timestamp;
+      });
+      const toKeep = scored.slice(0, windowSize).map((s) => s.entry);
+      const toRemove = scored.slice(windowSize);
       currentEntries = toKeep;
       evictedEntries += toRemove.length;
     }
@@ -1089,7 +1345,15 @@ function createContextPipeline(config2) {
     budgetUtilization = 0;
     optimizer.reset();
   }
+  function serializeOptimizerState() {
+    return optimizer.serializeState();
+  }
+  function deserializeOptimizerState(json) {
+    return optimizer.deserializeState(json);
+  }
   return {
+    serializeOptimizerState,
+    deserializeOptimizerState,
     ingest,
     getContext,
     getEntries,
@@ -1099,7 +1363,7 @@ function createContextPipeline(config2) {
 }
 
 // src/daemon/file-tracker.ts
-import { readFileSync, statSync } from "fs";
+import { closeSync, openSync, readSync, statSync } from "fs";
 function createFileTracker() {
   const positions = /* @__PURE__ */ new Map();
   function readNewLines(filePath) {
@@ -1125,9 +1389,14 @@ function createFileTracker() {
         }
         return [];
       }
-      const buffer = Buffer.alloc(currentSize - pos.position);
-      const fd = readFileSync(filePath);
-      fd.copy(buffer, 0, pos.position, currentSize);
+      const bytesToRead = currentSize - pos.position;
+      const buffer = Buffer.alloc(bytesToRead);
+      const fd = openSync(filePath, "r");
+      try {
+        readSync(fd, buffer, 0, bytesToRead, pos.position);
+      } finally {
+        closeSync(fd);
+      }
       const newContent = (pos.partialLine + buffer.toString("utf-8")).split("\n");
       const partialLine = newContent.pop() || "";
       pos.position = currentSize;
@@ -1186,6 +1455,7 @@ function createSessionWatcher(config2) {
         fullOptimizationInterval: 50
         // Full re-optimization every 50 incremental updates
       });
+      loadState(sessionId, pipeline);
       pipelines.set(sessionId, pipeline);
     }
     return pipeline;
@@ -1236,7 +1506,7 @@ function createSessionWatcher(config2) {
         } else if (entry.endsWith(".jsonl")) {
           const matches = realtime.watchPatterns.some((pattern) => {
             const regex = new RegExp(
-              pattern.replace(/\*\*/g, ".*").replace(/\*/g, "[^/\\\\]*").replace(/\./g, "\\.")
+              pattern.replace(/\./g, "\\.").replace(/\*\*/g, ".*").replace(/(?<!\.)(\*)/g, "[^/\\\\]*")
             );
             return regex.test(fullPath);
           });
@@ -1265,34 +1535,69 @@ function createSessionWatcher(config2) {
   async function start() {
     const projectsDir = getProjectsDir();
     const jsonlFiles = findJsonlFiles(projectsDir);
-    const watchedDirs = /* @__PURE__ */ new Set();
-    for (const file of jsonlFiles) {
-      const dir = dirname(file);
-      if (!watchedDirs.has(dir)) {
-        const watcher2 = watch(dir, { recursive: false }, (_eventType, filename) => {
-          if (filename?.endsWith(".jsonl")) {
-            const fullPath = join(dir, filename);
-            handleFileChange(fullPath);
-          }
-        });
-        watchers.push(watcher2);
-        watchedDirs.add(dir);
+    try {
+      const projectsWatcher = watch(projectsDir, { recursive: true }, (_eventType, filename) => {
+        if (filename?.endsWith(".jsonl")) {
+          const fullPath = join(projectsDir, filename);
+          handleFileChange(fullPath);
+        }
+      });
+      watchers.push(projectsWatcher);
+    } catch {
+      const watchedDirs = /* @__PURE__ */ new Set();
+      for (const file of jsonlFiles) {
+        const dir = dirname(file);
+        if (!watchedDirs.has(dir)) {
+          const watcher2 = watch(dir, { recursive: false }, (_eventType, filename) => {
+            if (filename?.endsWith(".jsonl")) {
+              const fullPath = join(dir, filename);
+              handleFileChange(fullPath);
+            }
+          });
+          watchers.push(watcher2);
+          watchedDirs.add(dir);
+        }
       }
     }
-    const projectsWatcher = watch(projectsDir, { recursive: true }, (_eventType, filename) => {
-      if (filename?.endsWith(".jsonl")) {
-        const fullPath = join(projectsDir, filename);
-        handleFileChange(fullPath);
-      }
-    });
-    watchers.push(projectsWatcher);
     getMetrics().updateDaemon({
       startTime: Date.now(),
       sessionsWatched: jsonlFiles.length,
       memoryUsage: process.memoryUsage().heapUsed
     });
   }
+  function getStatePath() {
+    return join(homedir(), ".sparn", "optimizer-state.json");
+  }
+  function saveState() {
+    try {
+      const stateMap = {};
+      for (const [sessionId, pipeline] of pipelines.entries()) {
+        stateMap[sessionId] = pipeline.serializeOptimizerState();
+      }
+      const statePath = getStatePath();
+      const dir = dirname(statePath);
+      if (!existsSync2(dir)) {
+        mkdirSync(dir, { recursive: true });
+      }
+      writeFileSync(statePath, JSON.stringify(stateMap), "utf-8");
+    } catch {
+    }
+  }
+  function loadState(sessionId, pipeline) {
+    try {
+      const statePath = getStatePath();
+      if (!existsSync2(statePath)) return;
+      const raw = readFileSync(statePath, "utf-8");
+      const stateMap = JSON.parse(raw);
+      const sessionState = stateMap[sessionId];
+      if (sessionState) {
+        pipeline.deserializeOptimizerState(sessionState);
+      }
+    } catch {
+    }
+  }
   function stop() {
+    saveState();
     for (const watcher2 of watchers) {
       watcher2.close();
     }
@@ -1340,11 +1645,22 @@ function createSessionWatcher(config2) {
 var configJson = process.env["SPARN_CONFIG"];
 var pidFile = process.env["SPARN_PID_FILE"];
 var logFile = process.env["SPARN_LOG_FILE"];
+var configFilePath = process.env["SPARN_CONFIG_FILE"];
+if ((!configJson || !pidFile || !logFile) && configFilePath && existsSync3(configFilePath)) {
+  const fileConfig = JSON.parse(readFileSync2(configFilePath, "utf-8"));
+  configJson = configJson || JSON.stringify(fileConfig.config);
+  pidFile = pidFile || fileConfig.pidFile;
+  logFile = logFile || fileConfig.logFile;
+}
 if (!configJson || !pidFile || !logFile) {
   console.error("Daemon: Missing required environment variables");
   process.exit(1);
 }
 var config = JSON.parse(configJson);
+if (config.realtime?.preciseTokenCounting) {
+  setPreciseTokenCounting(true);
+}
+writeFileSync2(pidFile, String(process.pid), "utf-8");
 function log(message) {
   const timestamp = (/* @__PURE__ */ new Date()).toISOString();
   const logMessage = `[${timestamp}] ${message}
@@ -1358,7 +1674,7 @@ function log(message) {
 }
 function cleanup() {
   log("Daemon shutting down");
-  if (pidFile && existsSync2(pidFile)) {
+  if (pidFile && existsSync3(pidFile)) {
     try {
       unlinkSync(pidFile);
     } catch {
@@ -1423,5 +1739,9 @@ watcher.start().then(async () => {
   cleanup();
 });
 setInterval(() => {
-}, 6e4);
+}, 3e4);
+try {
+  process.stdin.resume();
+} catch {
+}
 //# sourceMappingURL=index.js.map