metame-cli 1.6.0 → 1.6.2

package/index.js

@@ -282,15 +282,16 @@ function ensureLaunchdPlist({ daemonScript, daemonLog }) {
   <string>${LAUNCHD_LABEL}</string>
   <key>ProgramArguments</key>
   <array>
-    <string>/usr/bin/caffeinate</string>
-    <string>-i</string>
     <string>${nodePath}</string>
     <string>${daemonScript}</string>
   </array>
   <key>RunAtLoad</key>
   <true/>
   <key>KeepAlive</key>
-  <true/>
+  <dict>
+    <key>SuccessfulExit</key>
+    <false/>
+  </dict>
   <key>ThrottleInterval</key>
   <integer>30</integer>
   <key>StandardOutPath</key>
@@ -2714,10 +2715,8 @@ try {
       } catch { /* PID file stale, daemon not running */ }
     }
     if (!daemonRunning) {
-      const _isMac = process.platform === 'darwin';
-      const dCmd = _isMac ? 'caffeinate' : process.execPath;
-      const dArgs = _isMac ? ['-i', process.execPath, _daemonScript] : [_daemonScript];
-      const bg = spawn(dCmd, dArgs, {
+      const dArgs = [_daemonScript];
+      const bg = spawn(process.execPath, dArgs, {
         detached: true,
         stdio: 'ignore',
         windowsHide: true,

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "metame-cli",
-    "version": "1.6.0",
+    "version": "1.6.2",
     "description": "The Cognitive Profile Layer for Claude Code. Knows how you think, not just what you said.",
     "main": "index.js",
     "bin": {

package/scripts/core/chunker.js

@@ -0,0 +1,100 @@
+'use strict';
+
+/**
+ * core/chunker.js — Recursive delimiter-aware text chunker
+ *
+ * Pure function, no I/O, no dependencies.
+ *
+ * Exports:
+ *   chunkText(text, opts?) → string[]
+ */
+
+// Delimiter hierarchy: paragraphs → lines → sentences → words
+const DELIMITERS = [
+  /\n\n+/,           // paragraphs
+  /\n/,              // lines
+  /(?<=[.!?。！？])\s+/, // sentences
+  /\s+/,             // words
+];
+
+/**
+ * Split text into chunks of approximately targetWords size.
+ *
+ * Algorithm:
+ * 1. Split by highest-level delimiter that produces >1 segment.
+ * 2. Greedily merge consecutive segments until adding the next would exceed targetWords.
+ * 3. If a single segment exceeds targetWords, recurse with the next finer delimiter.
+ * 4. Fragments smaller than targetWords * 0.3 are merged into the previous chunk.
+ *
+ * @param {string} text
+ * @param {{ targetWords?: number }} [opts]
+ * @returns {string[]}
+ */
+function chunkText(text, { targetWords = 300 } = {}) {
+  if (!text || typeof text !== 'string') return [];
+  const trimmed = text.trim();
+  if (!trimmed) return [];
+
+  const wordCount = trimmed.split(/\s+/).length;
+  if (wordCount <= targetWords * 1.5) return [trimmed];
+
+  return _splitRecursive(trimmed, targetWords, 0);
+}
+
+/**
+ * @param {string} text
+ * @param {number} target
+ * @param {number} delimIdx — current position in DELIMITERS hierarchy
+ * @returns {string[]}
+ */
+function _splitRecursive(text, target, delimIdx) {
+  if (delimIdx >= DELIMITERS.length) return [text];
+
+  const segments = text.split(DELIMITERS[delimIdx]).filter(s => s.trim());
+  if (segments.length <= 1) {
+    return _splitRecursive(text, target, delimIdx + 1);
+  }
+
+  // Greedy merge
+  const chunks = [];
+  let current = '';
+
+  for (const seg of segments) {
+    const segWords = seg.split(/\s+/).length;
+    const curWords = current ? current.split(/\s+/).length : 0;
+
+    if (current && curWords + segWords > target) {
+      chunks.push(current.trim());
+      current = seg;
+    } else {
+      current = current ? current + '\n\n' + seg : seg;
+    }
+  }
+  if (current.trim()) chunks.push(current.trim());
+
+  // Recurse on oversized chunks
+  const result = [];
+  for (const chunk of chunks) {
+    const cw = chunk.split(/\s+/).length;
+    if (cw > target * 1.5) {
+      result.push(..._splitRecursive(chunk, target, delimIdx + 1));
+    } else {
+      result.push(chunk);
+    }
+  }
+
+  // Merge tiny trailing fragments into previous chunk
+  const merged = [];
+  for (const chunk of result) {
+    const cw = chunk.split(/\s+/).length;
+    if (merged.length > 0 && cw < target * 0.3) {
+      merged[merged.length - 1] += '\n\n' + chunk;
+    } else {
+      merged.push(chunk);
+    }
+  }
+
+  return merged;
+}
+
+module.exports = { chunkText };

package/scripts/core/embedding.js

@@ -0,0 +1,225 @@
+'use strict';
+
+/**
+ * core/embedding.js — Embedding adapter with two backends:
+ *   1. OpenAI text-embedding-3-small (512-dim) — requires OPENAI_API_KEY
+ *   2. Ollama bge-m3 (1024-dim, local) — fallback when OpenAI key absent
+ *
+ * Exports:
+ *   getEmbedding(text)          → Float32Array | null
+ *   batchEmbed(texts[])         → (Float32Array | null)[]
+ *   embeddingToBuffer(f32)      → Buffer       (for SQLite BLOB write)
+ *   bufferToEmbedding(blob)     → Float32Array  (for SQLite BLOB read)
+ *   isEmbeddingAvailable()      → boolean
+ *
+ * Backend selection (automatic):
+ *   OPENAI_API_KEY set  → OpenAI (512-dim)
+ *   ollama installed    → bge-m3 via localhost:11434 (1024-dim)
+ *   neither             → all functions return null gracefully
+ */
+
+const { existsSync } = require('node:fs');
+
+// ── OpenAI backend ────────────────────────────────────────────────────────────
+const OPENAI_MODEL      = 'text-embedding-3-small';
+const OPENAI_DIMENSIONS = 512;
+const OPENAI_API_URL    = 'https://api.openai.com/v1/embeddings';
+
+// ── Ollama backend ────────────────────────────────────────────────────────────
+const OLLAMA_MODEL      = 'bge-m3';
+const OLLAMA_DIMENSIONS = 1024;
+const OLLAMA_API_URL    = process.env.OLLAMA_HOST
+  ? `${process.env.OLLAMA_HOST}/api/embed`
+  : 'http://localhost:11434/api/embed';
+const OLLAMA_BIN_PATHS  = [
+  '/usr/local/bin/ollama',
+  '/opt/homebrew/bin/ollama',
+  '/usr/bin/ollama',
+];
+
+// ── Shared constants ──────────────────────────────────────────────────────────
+const MAX_INPUT_CHARS = 8000;
+const MAX_RETRIES     = 3;
+const BASE_DELAY_MS   = 2000;
+const BATCH_SIZE      = 100;
+
+// Keep legacy export name for callers that read it directly
+const MODEL      = OPENAI_MODEL;
+const DIMENSIONS = OPENAI_DIMENSIONS;
+
+// ── Backend detection ─────────────────────────────────────────────────────────
+
+function getApiKey() {
+  return process.env.OPENAI_API_KEY || '';
+}
+
+function isOllamaInstalled() {
+  return OLLAMA_BIN_PATHS.some(p => existsSync(p));
+}
+
+function getBackend() {
+  if (getApiKey()) return 'openai';
+  if (isOllamaInstalled()) return 'ollama';
+  return null;
+}
+
+function isEmbeddingAvailable() {
+  return getBackend() !== null;
+}
+
+/**
+ * L2-normalize a Float32Array in-place and return it.
+ * @param {Float32Array} vec
+ * @returns {Float32Array}
+ */
+function l2Normalize(vec) {
+  let norm = 0;
+  for (let i = 0; i < vec.length; i++) norm += vec[i] * vec[i];
+  norm = Math.sqrt(norm);
+  if (norm > 0) {
+    for (let i = 0; i < vec.length; i++) vec[i] /= norm;
+  }
+  return vec;
+}
+
+// ── OpenAI API call ───────────────────────────────────────────────────────────
+
+async function callOpenAI(inputs) {
+  const apiKey = getApiKey();
+  let lastError;
+  for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
+    try {
+      const resp = await fetch(OPENAI_API_URL, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${apiKey}` },
+        body: JSON.stringify({ model: OPENAI_MODEL, input: inputs, dimensions: OPENAI_DIMENSIONS }),
+      });
+      if (resp.status === 429) {
+        const retryAfter = parseInt(resp.headers.get('retry-after') || '0', 10);
+        await new Promise(r => setTimeout(r, Math.max(retryAfter * 1000, BASE_DELAY_MS * Math.pow(2, attempt))));
+        continue;
+      }
+      if (!resp.ok) {
+        const body = await resp.text().catch(() => '');
+        throw new Error(`OpenAI API ${resp.status}: ${body.slice(0, 200)}`);
+      }
+      const data = await resp.json();
+      return data.data.map(item => l2Normalize(new Float32Array(item.embedding)));
+    } catch (err) {
+      lastError = err;
+      if (attempt < MAX_RETRIES - 1) await new Promise(r => setTimeout(r, BASE_DELAY_MS * Math.pow(2, attempt)));
+    }
+  }
+  throw lastError;
+}
+
+// ── Ollama API call (bge-m3, one text at a time — ollama /api/embed) ──────────
+
+async function callOllama(inputs) {
+  const results = [];
+  for (const text of inputs) {
+    let lastError;
+    for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
+      try {
+        const resp = await fetch(OLLAMA_API_URL, {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify({ model: OLLAMA_MODEL, input: text }),
+        });
+        if (!resp.ok) {
+          const body = await resp.text().catch(() => '');
+          throw new Error(`Ollama API ${resp.status}: ${body.slice(0, 200)}`);
+        }
+        const data = await resp.json();
+        const vec = data.embeddings?.[0];
+        if (!vec || !Array.isArray(vec)) throw new Error('Ollama: unexpected response shape');
+        results.push(l2Normalize(new Float32Array(vec)));
+        lastError = null;
+        break;
+      } catch (err) {
+        lastError = err;
+        if (attempt < MAX_RETRIES - 1) await new Promise(r => setTimeout(r, BASE_DELAY_MS * Math.pow(2, attempt)));
+      }
+    }
+    if (lastError) results.push(null);
+  }
+  return results;
+}
+
+// ── Unified router ────────────────────────────────────────────────────────────
+
+async function callApi(inputs) {
+  const backend = getBackend();
+  if (backend === 'openai') return callOpenAI(inputs);
+  if (backend === 'ollama') return callOllama(inputs);
+  return inputs.map(() => null);
+}
+
+/**
+ * Get embedding for a single text.
+ * @param {string} text
+ * @returns {Promise<Float32Array|null>}
+ */
+async function getEmbedding(text) {
+  if (!isEmbeddingAvailable()) return null;
+  if (!text || typeof text !== 'string') return null;
+  const truncated = text.slice(0, MAX_INPUT_CHARS);
+  const results = await callApi([truncated]);
+  return results[0] || null;
+}
+
+/**
+ * Get embeddings for multiple texts in batches.
+ * @param {string[]} texts
+ * @returns {Promise<(Float32Array|null)[]>}
+ */
+async function batchEmbed(texts) {
+  if (!isEmbeddingAvailable()) return texts.map(() => null);
+  if (!texts || texts.length === 0) return [];
+
+  const results = [];
+  for (let i = 0; i < texts.length; i += BATCH_SIZE) {
+    const batch = texts.slice(i, i + BATCH_SIZE).map(t =>
+      (typeof t === 'string' ? t : '').slice(0, MAX_INPUT_CHARS),
+    );
+    const embeddings = await callApi(batch);
+    results.push(...embeddings);
+  }
+  return results;
+}
+
+/**
+ * Convert Float32Array to Buffer for SQLite BLOB storage.
+ * @param {Float32Array} f32
+ * @returns {Buffer}
+ */
+function embeddingToBuffer(f32) {
+  if (!f32) return null;
+  return Buffer.from(f32.buffer, f32.byteOffset, f32.byteLength);
+}
+
+/**
+ * Read Buffer from SQLite BLOB back to Float32Array.
+ * Dimension-agnostic: infers dim from blob length (supports both 512-dim OpenAI
+ * and 1024-dim bge-m3 embeddings stored in the same DB).
+ * @param {Buffer|Uint8Array} blob
+ * @returns {Float32Array|null}
+ */
+function bufferToEmbedding(blob) {
+  if (!blob || blob.length === 0 || blob.length % 4 !== 0) return null;
+  // Copy into aligned ArrayBuffer to avoid RangeError on unaligned byteOffset
+  const aligned = new ArrayBuffer(blob.length);
+  new Uint8Array(aligned).set(new Uint8Array(blob.buffer, blob.byteOffset, blob.length));
+  return new Float32Array(aligned);
+}
+
+module.exports = {
+  getEmbedding,
+  batchEmbed,
+  embeddingToBuffer,
+  bufferToEmbedding,
+  isEmbeddingAvailable,
+  l2Normalize,
+  MODEL,
+  DIMENSIONS,
+};

package/scripts/core/hybrid-search.js

@@ -0,0 +1,296 @@
+'use strict';
+
+/**
+ * core/hybrid-search.js — Hybrid wiki search (FTS5 + Vector + RRF fusion)
+ *
+ * Exports:
+ *   hybridSearchWiki(db, query, opts?) → { wikiPages: object[], facts: object[] }
+ *
+ * When vector embeddings are available:
+ *   1. FTS5 search → page candidates with rank
+ *   2. Vector cosine search on content_chunks → chunk candidates
+ *   3. Chunk → page aggregation (max score per slug, keep best chunk as excerpt)
+ *   4. RRF fusion of FTS page ranks + vector page ranks
+ *   5. Normalize scores to 0-1
+ *
+ * Degradation: no embeddings in DB → pure FTS5 (same as searchWikiAndFacts)
+ */
+
+const { sanitizeFts5 } = require('./wiki-slug');
+const { bufferToEmbedding, getEmbedding, isEmbeddingAvailable } = require('./embedding');
+
+const RRF_K = 60;
+const STALE_THRESHOLD = 0.3;
+const MAX_FTS_RESULTS = 10;
+const MAX_VECTOR_RESULTS = 20;
+
+/**
+ * Dot product of two Float32Arrays (assumes L2-normalized → equals cosine similarity).
+ * @param {Float32Array} a
+ * @param {Float32Array} b
+ * @returns {number}
+ */
+function dotProduct(a, b) {
+  let sum = 0;
+  for (let i = 0; i < a.length; i++) sum += a[i] * b[i];
+  return sum;
+}
+
+/**
+ * Top-K selection via bounded insertion (avoids full sort).
+ * @param {{ score: number }[]} items
+ * @param {number} k
+ * @returns {{ score: number }[]}
+ */
+function topK(items, k) {
+  if (items.length <= k) return items.slice().sort((a, b) => b.score - a.score);
+  const heap = items.slice(0, k).sort((a, b) => a.score - b.score);
+  for (let i = k; i < items.length; i++) {
+    if (items[i].score > heap[0].score) {
+      heap[0] = items[i];
+      heap.sort((a, b) => a.score - b.score);
+    }
+  }
+  return heap.sort((a, b) => b.score - a.score);
+}
+
+/**
+ * FTS5 search for wiki pages.
+ * @param {object} db
+ * @param {string} safeQuery — already sanitized
+ * @returns {{ slug: string, title: string, staleness: number, excerpt: string, ftsRank: number }[]}
+ */
+function ftsSearch(db, safeQuery) {
+  try {
+    return db.prepare(`
+      SELECT wp.slug, wp.title, wp.staleness, wp.last_built_at,
+             snippet(wiki_pages_fts, 2, '<b>', '</b>', '...', 20) as excerpt,
+             rank as ftsRank
+      FROM wiki_pages_fts
+      JOIN wiki_pages wp ON wiki_pages_fts.rowid = wp.rowid
+      WHERE wiki_pages_fts MATCH ?
+      ORDER BY rank
+      LIMIT ?
+    `).all(safeQuery, MAX_FTS_RESULTS);
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * Vector cosine search on content_chunks.
+ * Brute-force scan with top-K heap. Only scans rows with embedding IS NOT NULL.
+ *
+ * @param {object} db
+ * @param {Float32Array} queryEmbedding
+ * @returns {{ page_slug: string, chunk_text: string, score: number }[]}
+ */
+function vectorSearch(db, queryEmbedding) {
+  let rows;
+  try {
+    rows = db.prepare(`
+      SELECT page_slug, chunk_text, embedding
+      FROM content_chunks
+      WHERE embedding IS NOT NULL
+    `).all();
+  } catch {
+    return [];
+  }
+
+  const scored = [];
+  for (const row of rows) {
+    const emb = bufferToEmbedding(row.embedding);
+    if (!emb) continue;
+    const score = dotProduct(queryEmbedding, emb);
+    scored.push({ page_slug: row.page_slug, chunk_text: row.chunk_text, score });
+  }
+
+  return topK(scored, MAX_VECTOR_RESULTS);
+}
+
+/**
+ * Check if any content_chunks have stored embeddings.
+ * Avoids wasting OpenAI API calls when no embeddings exist yet.
+ */
+function hasStoredEmbeddings(db) {
+  try {
+    return !!db.prepare('SELECT 1 FROM content_chunks WHERE embedding IS NOT NULL LIMIT 1').get();
+  } catch { return false; }
+}
+
+/**
+ * Aggregate chunk-level vector results to page-level.
+ * Per slug: keep max score and best chunk text as excerpt.
+ * @param {{ page_slug: string, chunk_text: string, score: number }[]} chunks
+ * @returns {Map<string, { score: number, excerpt: string }>}
+ */
+function aggregateChunksToPages(chunks) {
+  const pages = new Map();
+  for (const c of chunks) {
+    const existing = pages.get(c.page_slug);
+    if (!existing || c.score > existing.score) {
+      pages.set(c.page_slug, { score: c.score, excerpt: c.chunk_text.slice(0, 200) });
+    }
+  }
+  return pages;
+}
+
+/**
+ * RRF fusion of two ranked lists.
+ * @param {Map<string, { ftsRank?: number, vectorRank?: number, title?: string, excerpt?: string, staleness?: number }>} merged
+ * @returns {{ slug: string, score: number, title: string, excerpt: string, staleness: number, stale: boolean, source: string }[]}
+ */
+function rrfFuse(merged) {
+  const results = [];
+  for (const [slug, info] of merged) {
+    let score = 0;
+    let source = '';
+    if (typeof info.ftsRank === 'number') {
+      score += 1 / (RRF_K + info.ftsRank);
+      source = 'fts';
+    }
+    if (typeof info.vectorRank === 'number') {
+      score += 1 / (RRF_K + info.vectorRank);
+      source = source ? 'hybrid' : 'vector';
+    }
+    const staleness = info.staleness || 0;
+    results.push({
+      slug,
+      score,
+      title: info.title || slug,
+      excerpt: info.excerpt || '',
+      staleness,
+      stale: staleness >= STALE_THRESHOLD,
+      source,
+    });
+  }
+  results.sort((a, b) => b.score - a.score);
+  return results;
+}
+
+/**
+ * Normalize scores to 0-1 range.
+ * @param {{ score: number }[]} results
+ */
+function normalizeScores(results) {
+  if (results.length === 0) return;
+  const max = results[0].score;
+  const min = results[results.length - 1].score;
+  for (const r of results) {
+    r.score = max === min ? 1.0 : (r.score - min) / (max - min);
+  }
+}
+
+/**
+ * Main entry: hybrid wiki search.
+ *
+ * @param {object} db
+ * @param {string} query
+ * @param {{ ftsOnly?: boolean, trackSearch?: boolean }} [opts]
+ * @returns {{ wikiPages: object[], facts: object[] }}
+ */
+async function hybridSearchWiki(db, query, { ftsOnly = false, trackSearch = true } = {}) {
+  const safeQuery = sanitizeFts5(query);
+  if (!safeQuery) return { wikiPages: [], facts: [] };
+
+  // 1. FTS5 search (always)
+  const ftsResults = ftsSearch(db, safeQuery);
+
+  // 2. Vector search (if available and not forced FTS-only)
+  let vectorPages = new Map();
+  const hasEmbeddings = !ftsOnly && isEmbeddingAvailable();
+
+  if (hasEmbeddings && hasStoredEmbeddings(db)) {
+    try {
+      const queryEmb = await getEmbedding(query);
+      if (queryEmb) {
+        const chunks = vectorSearch(db, queryEmb);
+        vectorPages = aggregateChunksToPages(chunks);
+      }
+    } catch {
+      // Vector search failed — degrade gracefully
+    }
+  }
+
+  // 3. Merge FTS + vector results into unified map
+  const merged = new Map();
+
+  for (let i = 0; i < ftsResults.length; i++) {
+    const r = ftsResults[i];
+    merged.set(r.slug, {
+      ftsRank: i + 1,
+      title: r.title,
+      excerpt: r.excerpt,
+      staleness: r.staleness,
+    });
+  }
+
+  for (const [slug, vInfo] of vectorPages) {
+    const existing = merged.get(slug);
+    const rank = [...vectorPages.keys()].indexOf(slug) + 1;
+    if (existing) {
+      existing.vectorRank = rank;
+      // Prefer vector excerpt if FTS didn't have a good one
+      if (vInfo.excerpt && (!existing.excerpt || existing.excerpt.length < 20)) {
+        existing.excerpt = vInfo.excerpt;
+      }
+    } else {
+      // Vector-only result — need to fetch page metadata
+      let title = slug;
+      let staleness = 0;
+      try {
+        const page = db.prepare('SELECT title, staleness FROM wiki_pages WHERE slug = ?').get(slug);
+        if (page) { title = page.title; staleness = page.staleness || 0; }
+      } catch { }
+      merged.set(slug, {
+        vectorRank: rank,
+        title,
+        excerpt: vInfo.excerpt,
+        staleness,
+      });
+    }
+  }
+
+  // 4. RRF fusion + normalize
+  const wikiPages = rrfFuse(merged);
+  normalizeScores(wikiPages);
+
+  // 5. Facts search (same as searchWikiAndFacts — FTS5 only)
+  let facts = [];
+  try {
+    facts = db.prepare(`
+      SELECT mi.id, mi.title, mi.content, mi.kind, mi.confidence,
+             snippet(memory_items_fts, 1, '<b>', '</b>', '...', 20) as excerpt,
+             rank as score
+      FROM memory_items_fts
+      JOIN memory_items mi ON memory_items_fts.rowid = mi.rowid
+      WHERE memory_items_fts MATCH ?
+        AND mi.state = 'active'
+      ORDER BY rank
+      LIMIT 10
+    `).all(safeQuery);
+  } catch {
+    facts = [];
+  }
+
+  // 6. Track search counts on matched facts
+  if (trackSearch && facts.length > 0) {
+    const ids = facts.map(r => r.id).filter(Boolean);
+    if (ids.length > 0) {
+      const ph = ids.map(() => '?').join(', ');
+      try {
+        db.prepare(`
+          UPDATE memory_items SET search_count = search_count + 1, last_searched_at = datetime('now')
+          WHERE id IN (${ph})
+        `).run(...ids);
+      } catch { }
+    }
+  }
+
+  return { wikiPages: wikiPages.slice(0, 5), facts };
+}
+
+module.exports = {
+  hybridSearchWiki,
+  _internal: { dotProduct, topK, ftsSearch, vectorSearch, aggregateChunksToPages, rrfFuse, normalizeScores, hasStoredEmbeddings },
+};