@hasna/search 0.0.9 → 0.0.11

package/dist/mcp/index.js

@@ -11644,7 +11644,7 @@ var require_lib2 = __commonJS((exports, module) => {
 var require_package = __commonJS((exports, module) => {
   module.exports = {
     name: "@hasna/search",
-    version: "0.0.9",
+    version: "0.0.11",
     description: "Unified search \u2014 local file index (find files by name/path/content/regex in ms, trigram FTS) + 12 web providers (Google, SerpAPI, Exa, Perplexity, Twitter, Reddit, YouTube, Brave, Bing, Hacker News, GitHub, arXiv) + YouTube transcription. CLI + MCP + REST API + Dashboard.",
     type: "module",
     main: "dist/index.js",
@@ -24742,12 +24742,19 @@ var DEFAULT_CONFIG = {
   defaultLimit: 10,
   defaultProviders: [],
   defaultProfile: null,
+  router: {
+    enabled: false,
+    model: "gpt-oss-120b",
+    maxProviders: 3,
+    timeoutMs: 1200
+  },
   transcriber: {
     baseUrl: "http://localhost:19600",
     fallbackCli: "microservice-transcriber"
   },
   dedup: true,
   maxConcurrent: 5,
+  providerTimeoutMs: 15000,
   indexStaleMinutes: 5,
   indexAutoRefresh: true,
   recordLocalResults: false
@@ -25482,6 +25489,31 @@ var migrations = [
         );
       `);
     }
+  },
+  {
+    version: 2,
+    description: "Local file index filter indexes",
+    up: (db) => {
+      db.exec(`
+        CREATE INDEX IF NOT EXISTS idx_files_root_ext ON files(root_id, ext);
+        CREATE INDEX IF NOT EXISTS idx_files_root_dir ON files(root_id, dir);
+      `);
+    }
+  },
+  {
+    version: 3,
+    description: "Local content short-token filter grams",
+    up: (db) => {
+      db.exec(`
+        CREATE TABLE IF NOT EXISTS file_content_grams (
+          file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
+          gram TEXT NOT NULL,
+          PRIMARY KEY (file_id, gram)
+        );
+        CREATE INDEX IF NOT EXISTS idx_file_content_grams_gram_file
+          ON file_content_grams(gram, file_id);
+      `);
+    }
   }
 ];
 function runIndexMigrations(db) {
@@ -25553,7 +25585,18 @@ function getConfig() {
   try {
     const raw = readFileSync(path, "utf-8");
     const parsed = JSON.parse(raw);
-    return { ...DEFAULT_CONFIG, ...parsed };
+    return {
+      ...DEFAULT_CONFIG,
+      ...parsed,
+      router: {
+        ...DEFAULT_CONFIG.router,
+        ...parsed.router ?? {}
+      },
+      transcriber: {
+        ...DEFAULT_CONFIG.transcriber,
+        ...parsed.transcriber ?? {}
+      }
+    };
   } catch {
     return { ...DEFAULT_CONFIG };
   }
@@ -25971,6 +26014,7 @@ function removeRoot(idOrPath, db) {
   d.exec("BEGIN");
   try {
     d.prepare("DELETE FROM file_content_fts WHERE rowid IN (SELECT id FROM files WHERE root_id = ? AND content_indexed = 1)").run(root.id);
+    d.prepare("DELETE FROM file_content_grams WHERE file_id IN (SELECT id FROM files WHERE root_id = ? AND content_indexed = 1)").run(root.id);
     d.prepare("DELETE FROM index_roots WHERE id = ?").run(root.id);
     d.exec("COMMIT");
   } catch (err) {
@@ -25982,6 +26026,21 @@ function removeRoot(idOrPath, db) {
 function shouldIndexContent(root, file) {
   return root.contentIndexing && file.size > 0 && file.size <= root.maxFileSize && !hasBinaryExtension(file.ext) && !isContentExcluded(file.name);
 }
+function contentShortGrams(body) {
+  const grams = new Set;
+  const words = body.toLowerCase().matchAll(/[a-z0-9_$]+/g);
+  for (const match of words) {
+    const word = match[0];
+    for (let i = 0;i < word.length; i++) {
+      grams.add(word[i]);
+      if (i + 1 < word.length)
+        grams.add(word.slice(i, i + 2));
+    }
+    if (grams.size >= 2048)
+      break;
+  }
+  return [...grams];
+}
 function indexRoot(idOrPath, opts = {}, db) {
   const d = db ?? getIndexDb();
   const root = getRoot(idOrPath, d);
@@ -26000,6 +26059,8 @@ function indexRoot(idOrPath, opts = {}, db) {
     const deleteFile = d.prepare("DELETE FROM files WHERE id = ?");
     const insertContent = d.prepare("INSERT INTO file_content_fts (rowid, body) VALUES (?, ?)");
     const deleteContent = d.prepare("DELETE FROM file_content_fts WHERE rowid = ?");
+    const insertContentGram = d.prepare("INSERT OR IGNORE INTO file_content_grams (file_id, gram) VALUES (?, ?)");
+    const deleteContentGrams = d.prepare("DELETE FROM file_content_grams WHERE file_id = ?");
     const stats = {
       rootId: root.id,
       added: 0,
@@ -26010,38 +26071,57 @@ function indexRoot(idOrPath, opts = {}, db) {
       skippedDirs: skippedDirs.length,
       durationMs: 0
     };
+    const seen = new Set;
+    const changes = [];
+    for (const file of scanned) {
+      seen.add(file.relPath);
+      const prev = existing.get(file.relPath);
+      const changed = !prev || prev.size !== file.size || prev.mtime_ms !== file.mtimeMs;
+      if (prev && !changed && !opts.force)
+        continue;
+      const wantContent = shouldIndexContent(root, file);
+      const absPath = `${root.path}/${file.relPath}`;
+      let isBinary = wantContent ? isBinaryFile(absPath) : hasBinaryExtension(file.ext);
+      let body = null;
+      if (wantContent && !isBinary) {
+        try {
+          body = readFileSync3(absPath, "utf-8");
+        } catch {
+          isBinary = true;
+        }
+      }
+      changes.push({
+        file,
+        prev,
+        isBinary,
+        body,
+        grams: body !== null ? contentShortGrams(body) : [],
+        contentIndexed: body !== null ? 1 : 0
+      });
+    }
     d.exec("BEGIN");
     try {
-      const seen = new Set;
-      for (const file of scanned) {
-        seen.add(file.relPath);
-        const prev = existing.get(file.relPath);
-        const changed = !prev || prev.size !== file.size || prev.mtime_ms !== file.mtimeMs;
-        if (prev && !changed && !opts.force)
-          continue;
-        const wantContent = shouldIndexContent(root, file);
-        const absPath = `${root.path}/${file.relPath}`;
-        let isBinary = wantContent ? isBinaryFile(absPath) : hasBinaryExtension(file.ext);
-        let body = null;
-        if (wantContent && !isBinary) {
-          try {
-            body = readFileSync3(absPath, "utf-8");
-          } catch {
-            isBinary = true;
-          }
-        }
-        const contentIndexed = body !== null ? 1 : 0;
+      for (const { file, prev, isBinary, body, grams, contentIndexed } of changes) {
         if (prev) {
-          if (prev.content_indexed)
+          if (prev.content_indexed) {
             deleteContent.run(prev.id);
+            deleteContentGrams.run(prev.id);
+          }
           updateFile.run(file.size, file.mtimeMs, isBinary ? 1 : 0, contentIndexed, now, prev.id);
-          if (body !== null)
+          if (body !== null) {
             insertContent.run(prev.id, body);
+            for (const gram of grams)
+              insertContentGram.run(prev.id, gram);
+          }
           stats.updated++;
         } else {
           const inserted = insertFile.run(root.id, file.relPath, file.name, file.ext, file.dir, file.size, file.mtimeMs, isBinary ? 1 : 0, contentIndexed, now);
-          if (body !== null)
-            insertContent.run(Number(inserted.lastInsertRowid), body);
+          if (body !== null) {
+            const fileId = Number(inserted.lastInsertRowid);
+            insertContent.run(fileId, body);
+            for (const gram of grams)
+              insertContentGram.run(fileId, gram);
+          }
           stats.added++;
         }
         if (contentIndexed)
@@ -26050,8 +26130,10 @@ function indexRoot(idOrPath, opts = {}, db) {
       for (const [relPath, row] of existing) {
         if (seen.has(relPath))
           continue;
-        if (row.content_indexed)
+        if (row.content_indexed) {
           deleteContent.run(row.id);
+          deleteContentGrams.run(row.id);
+        }
         deleteFile.run(row.id);
         stats.deleted++;
       }
@@ -26073,6 +26155,9 @@ function indexAllRoots(opts = {}, db) {
   return listRoots(db).map((root) => indexRoot(root.id, opts, db));
 }
 var refreshing = new Set;
+var lastDefaultAutoRefreshCheckAt = 0;
+var AUTO_REFRESH_CHECK_THROTTLE_MS = 1000;
+var defaultRefreshScheduled = false;
 function refreshStaleRoots(staleMinutes, db) {
   const cutoff = Date.now() - staleMinutes * 60000;
   const stats = [];
@@ -26096,8 +26181,43 @@ function autoRefreshStaleRoots(db) {
   const config2 = getConfig();
   if (!config2.indexAutoRefresh)
     return [];
+  if (!db) {
+    const now = Date.now();
+    if (now - lastDefaultAutoRefreshCheckAt < AUTO_REFRESH_CHECK_THROTTLE_MS)
+      return [];
+    lastDefaultAutoRefreshCheckAt = now;
+  }
   return refreshStaleRoots(config2.indexStaleMinutes, db);
 }
+function scheduleAutoRefreshStaleRoots(db) {
+  if (db)
+    return autoRefreshStaleRoots(db);
+  const config2 = getConfig();
+  if (!config2.indexAutoRefresh || defaultRefreshScheduled)
+    return [];
+  defaultRefreshScheduled = true;
+  const timer = setTimeout(() => {
+    try {
+      autoRefreshStaleRoots();
+    } catch {} finally {
+      defaultRefreshScheduled = false;
+    }
+  }, 0);
+  timer.unref?.();
+  return [];
+}
+function startBackgroundRefresh() {
+  const minutes = Math.max(1, getConfig().indexStaleMinutes);
+  const timer = setInterval(() => {
+    try {
+      autoRefreshStaleRoots();
+    } catch (err) {
+      console.error("Index refresh failed:", err);
+    }
+  }, minutes * 60000);
+  timer.unref?.();
+  return timer;
+}
 
 // src/lib/local/query.ts
 import { existsSync as existsSync2, readFileSync as readFileSync4 } from "fs";
@@ -26324,6 +26444,9 @@ function compileSearchRegex(pattern, caseSensitive = false) {
 // src/lib/local/query.ts
 var MAX_LINE_LENGTH = 200;
 var MAX_MATCHES_PER_FILE = 5;
+var MAX_PATH_CANDIDATES = 20000;
+var MAX_CONTENT_CANDIDATES = 50000;
+var MAX_REGEX_CANDIDATES = 50000;
 function tokenize(query) {
   return query.replace(/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F]/g, "").split(/\s+/).filter(Boolean);
 }
@@ -26354,11 +26477,40 @@ function filterClauses(opts, db) {
   }
   if (opts.dir) {
     clauses.push("f.dir LIKE ? ESCAPE '\\'");
-    const dir = opts.dir.replace(/^\/|\/$/g, "").replace(/[\\%_]/g, "\\$&");
+    const dir = escapeLike(opts.dir.replace(/^\/|\/$/g, ""));
     params.push(`%${dir}%`);
   }
   return { sql: clauses.length > 0 ? ` AND ${clauses.join(" AND ")}` : "", params };
 }
+function escapeLike(value) {
+  return value.replace(/[\\%_]/g, "\\$&");
+}
+function shortTokenClauses(tokens) {
+  if (tokens.length === 0)
+    return { sql: "", params: [] };
+  return {
+    sql: ` AND ${tokens.map(() => "f.rel_path LIKE ? ESCAPE '\\'").join(" AND ")}`,
+    params: tokens.map((token) => `%${escapeLike(token)}%`)
+  };
+}
+function contentGramClauses(tokens) {
+  const gramTokens = tokens.filter((token) => /^[a-z0-9_$]{1,2}$/.test(token));
+  if (gramTokens.length === 0)
+    return { sql: "", params: [] };
+  return {
+    sql: gramTokens.map((_token, index) => ` AND (
+            NOT EXISTS (
+              SELECT 1 FROM file_content_grams cg_any_${index}
+              WHERE cg_any_${index}.file_id = f.id
+            )
+            OR EXISTS (
+              SELECT 1 FROM file_content_grams cg_${index}
+              WHERE cg_${index}.file_id = f.id AND cg_${index}.gram = ?
+            )
+          )`).join(""),
+    params: gramTokens
+  };
+}
 function rowToHit(row, score) {
   return {
     rootId: row.root_id,
@@ -26428,6 +26580,8 @@ function searchFilePaths(query, opts = {}, db) {
     return [];
   const ftsQuery = buildFtsQuery(query);
   const filters = filterClauses(opts, d);
+  const shortTokens = tokens.filter((t) => t.length < 3).map((t) => t.toLowerCase());
+  const shortFilters = shortTokenClauses(shortTokens);
   const candidateLimit = Math.max(200, limit * 10);
   let rows;
   if (ftsQuery) {
@@ -26435,16 +26589,16 @@ function searchFilePaths(query, opts = {}, db) {
          FROM files_fts fts
          JOIN files f ON f.id = fts.rowid
          JOIN index_roots r ON r.id = f.root_id
-         WHERE files_fts MATCH ?${filters.sql}
+         WHERE files_fts MATCH ?${filters.sql}${shortFilters.sql}
          ORDER BY bm25(files_fts, 10.0, 1.0)
-         LIMIT ?`).all(ftsQuery, ...filters.params, candidateLimit);
-    const namePattern = `${query.trim().replace(/[\\%_]/g, "\\$&")}%`;
+         LIMIT ?`).all(ftsQuery, ...filters.params, ...shortFilters.params, Math.min(candidateLimit, MAX_PATH_CANDIDATES));
+    const namePattern = `${escapeLike(query.trim())}%`;
     const nameRows = d.prepare(`SELECT ${CANDIDATE_COLUMNS}
          FROM files f
          JOIN index_roots r ON r.id = f.root_id
-         WHERE f.name LIKE ? ESCAPE '\\'${filters.sql}
+         WHERE f.name LIKE ? ESCAPE '\\'${filters.sql}${shortFilters.sql}
          ORDER BY length(f.name)
-         LIMIT 100`).all(namePattern, ...filters.params);
+         LIMIT 100`).all(namePattern, ...filters.params, ...shortFilters.params);
     const seen = new Set(rows.map((row) => row.id));
     for (const row of nameRows) {
       if (!seen.has(row.id))
@@ -26452,14 +26606,14 @@ function searchFilePaths(query, opts = {}, db) {
     }
   } else {
     const likeClauses = tokens.map(() => "f.rel_path LIKE ? ESCAPE '\\'").join(" AND ");
-    const likeParams = tokens.map((t) => `%${t.replace(/[\\%_]/g, "\\$&")}%`);
+    const likeParams = tokens.map((t) => `%${escapeLike(t)}%`);
     rows = d.prepare(`SELECT ${CANDIDATE_COLUMNS}
          FROM files f
          JOIN index_roots r ON r.id = f.root_id
          WHERE ${likeClauses}${filters.sql}
-         LIMIT ?`).all(...likeParams, ...filters.params, candidateLimit);
+         ORDER BY length(f.name), length(f.rel_path), f.rel_path
+         LIMIT ?`).all(...likeParams, ...filters.params, Math.min(candidateLimit, MAX_PATH_CANDIDATES));
   }
-  const shortTokens = tokens.filter((t) => t.length < 3).map((t) => t.toLowerCase());
   const filtered = shortTokens.length > 0 ? rows.filter((row) => shortTokens.every((t) => row.rel_path.toLowerCase().includes(t))) : rows;
   return filtered.map((row) => rowToHit(row, scoreFileName(query, tokens, row))).sort((a, b) => b.score - a.score).filter((hit) => existsSync2(hit.absPath)).slice(0, limit);
 }
@@ -26497,24 +26651,31 @@ function searchFilePathsRegex(pattern, opts = {}, db) {
     throw new Error("Regex pattern needs at least one required literal of 3+ characters (e.g. 'handle.*Click', not '\\w+').");
   }
   const filters = filterClauses(opts, d);
-  const rows = d.prepare(`SELECT ${CANDIDATE_COLUMNS}
-       FROM files_fts fts
-       JOIN files f ON f.id = fts.rowid
-       JOIN index_roots r ON r.id = f.root_id
-       WHERE files_fts MATCH ?${filters.sql}
-       ORDER BY fts.rank
-       LIMIT 5000`).all(ftsQuery, ...filters.params);
   const hits = [];
-  for (const row of rows) {
-    if (!regex.test(row.rel_path) && !regex.test(row.name))
-      continue;
-    const depth = row.rel_path.split("/").length - 1;
-    const score = Math.max(0.05, 0.6 - depth * 0.02);
-    const hit = rowToHit(row, score);
-    if (!existsSync2(hit.absPath))
-      continue;
-    hits.push(hit);
-    if (hits.length >= limit)
+  const pageSize = Math.max(500, limit * 20);
+  for (let offset = 0;hits.length < limit && offset < MAX_REGEX_CANDIDATES; offset += pageSize) {
+    const rows = d.prepare(`SELECT ${CANDIDATE_COLUMNS}
+         FROM files_fts fts
+         JOIN files f ON f.id = fts.rowid
+         JOIN index_roots r ON r.id = f.root_id
+         WHERE files_fts MATCH ?${filters.sql}
+         ORDER BY fts.rank
+         LIMIT ? OFFSET ?`).all(ftsQuery, ...filters.params, pageSize, offset);
+    if (rows.length === 0)
+      break;
+    for (const row of rows) {
+      if (!regex.test(row.rel_path) && !regex.test(row.name))
+        continue;
+      const depth = row.rel_path.split("/").length - 1;
+      const score = Math.max(0.05, 0.6 - depth * 0.02);
+      const hit = rowToHit(row, score);
+      if (!existsSync2(hit.absPath))
+        continue;
+      hits.push(hit);
+      if (hits.length >= limit)
+        break;
+    }
+    if (rows.length < pageSize)
       break;
   }
   return hits;
@@ -26528,40 +26689,48 @@ function searchFileContentRegex(pattern, opts = {}, db) {
     throw new Error("Regex pattern needs at least one required literal of 3+ characters (e.g. 'export.*function', not '\\d+').");
   }
   const filters = filterClauses(opts, d);
-  const rows = d.prepare(`SELECT ${CANDIDATE_COLUMNS}
-       FROM file_content_fts fts
-       JOIN files f ON f.id = fts.rowid
-       JOIN index_roots r ON r.id = f.root_id
-       WHERE file_content_fts MATCH ?${filters.sql}
-       ORDER BY fts.rank
-       LIMIT ?`).all(ftsQuery, ...filters.params, Math.max(200, limit * 10));
   const hits = [];
-  for (let i = 0;i < rows.length && hits.length < limit; i++) {
-    const row = rows[i];
-    const absPath = `${row.root_path}/${row.rel_path}`;
-    let content;
-    try {
-      content = readFileSync4(absPath, "utf-8");
-    } catch {
-      continue;
-    }
-    const lines = content.split(`
+  const pageSize = Math.max(200, limit * 10);
+  for (let offset = 0;hits.length < limit && offset < MAX_REGEX_CANDIDATES; offset += pageSize) {
+    const rows = d.prepare(`SELECT ${CANDIDATE_COLUMNS}
+         FROM file_content_fts fts
+         JOIN files f ON f.id = fts.rowid
+         JOIN index_roots r ON r.id = f.root_id
+         WHERE file_content_fts MATCH ?${filters.sql}
+         ORDER BY fts.rank
+         LIMIT ? OFFSET ?`).all(ftsQuery, ...filters.params, pageSize, offset);
+    if (rows.length === 0)
+      break;
+    for (let i = 0;i < rows.length && hits.length < limit; i++) {
+      const row = rows[i];
+      const absPath = `${row.root_path}/${row.rel_path}`;
+      let content;
+      try {
+        content = readFileSync4(absPath, "utf-8");
+      } catch {
+        continue;
+      }
+      const lines = content.split(`
 `);
-    const matches = [];
-    for (let n = 0;n < lines.length && matches.length < MAX_MATCHES_PER_FILE; n++) {
-      if (regex.test(lines[n])) {
-        matches.push({ line: n + 1, text: lines[n].trim().slice(0, MAX_LINE_LENGTH) });
+      const matches = [];
+      for (let n = 0;n < lines.length && matches.length < MAX_MATCHES_PER_FILE; n++) {
+        if (regex.test(lines[n])) {
+          matches.push({ line: n + 1, text: lines[n].trim().slice(0, MAX_LINE_LENGTH) });
+        }
       }
+      if (matches.length === 0)
+        continue;
+      const rankIndex = offset + i;
+      const score = Math.max(0.25, 0.65 - rankIndex * 0.05);
+      hits.push({
+        ...rowToHit(row, score),
+        line: matches[0].line,
+        lineText: matches[0].text,
+        matches
+      });
     }
-    if (matches.length === 0)
-      continue;
-    const score = Math.max(0.25, 0.65 - i * 0.05);
-    hits.push({
-      ...rowToHit(row, score),
-      line: matches[0].line,
-      lineText: matches[0].text,
-      matches
-    });
+    if (rows.length < pageSize)
+      break;
   }
   return hits;
 }
@@ -26572,42 +26741,51 @@ function searchFileContent(query, opts = {}, db) {
   if (!ftsQuery)
     return [];
   const filters = filterClauses(opts, d);
-  const rows = d.prepare(`SELECT ${CANDIDATE_COLUMNS}
-       FROM file_content_fts fts
-       JOIN files f ON f.id = fts.rowid
-       JOIN index_roots r ON r.id = f.root_id
-       WHERE file_content_fts MATCH ?${filters.sql}
-       ORDER BY fts.rank
-       LIMIT ?`).all(ftsQuery, ...filters.params, Math.max(50, limit * 3));
   const tokens = tokenize(query);
   const shortTokens = tokens.filter((t) => t.length < 3).map((t) => t.toLowerCase());
+  const gramFilters = contentGramClauses(shortTokens);
   const scored = [];
-  for (let i = 0;i < rows.length && scored.length < limit * 2; i++) {
-    const row = rows[i];
-    const absPath = `${row.root_path}/${row.rel_path}`;
-    let content;
-    try {
-      content = readFileSync4(absPath, "utf-8");
-    } catch {
-      continue;
-    }
-    if (shortTokens.length > 0) {
-      const lower = content.toLowerCase();
-      if (!shortTokens.every((t) => lower.includes(t)))
+  const pageSize = Math.max(50, limit * 3);
+  for (let offset = 0;scored.length < limit * 2 && offset < MAX_CONTENT_CANDIDATES; offset += pageSize) {
+    const rows = d.prepare(`SELECT ${CANDIDATE_COLUMNS}
+         FROM file_content_fts fts
+         JOIN files f ON f.id = fts.rowid
+         JOIN index_roots r ON r.id = f.root_id
+         WHERE file_content_fts MATCH ?${filters.sql}${gramFilters.sql}
+         ORDER BY fts.rank
+         LIMIT ? OFFSET ?`).all(ftsQuery, ...filters.params, ...gramFilters.params, pageSize, offset);
+    if (rows.length === 0)
+      break;
+    for (let i = 0;i < rows.length && scored.length < limit * 2; i++) {
+      const row = rows[i];
+      const absPath = `${row.root_path}/${row.rel_path}`;
+      let content;
+      try {
+        content = readFileSync4(absPath, "utf-8");
+      } catch {
+        continue;
+      }
+      if (shortTokens.length > 0) {
+        const lower = content.toLowerCase();
+        if (!shortTokens.every((t) => lower.includes(t)))
+          continue;
+      }
+      const { matches, tier } = findLineMatches(content, query, tokens);
+      if (matches.length === 0)
         continue;
+      const rankIndex = offset + i;
+      const base = Math.max(0.25, 0.55 - rankIndex * 0.04);
+      const tierBoost = tier === "phrase" ? 0.1 : tier === "all" ? 0.05 : 0;
+      const score = Math.min(CONTENT_MAX_SCORE, base + tierBoost);
+      scored.push({
+        ...rowToHit(row, score),
+        line: matches[0].line,
+        lineText: matches[0].text,
+        matches
+      });
     }
-    const { matches, tier } = findLineMatches(content, query, tokens);
-    if (matches.length === 0)
-      continue;
-    const base = Math.max(0.25, 0.55 - i * 0.04);
-    const tierBoost = tier === "phrase" ? 0.1 : tier === "all" ? 0.05 : 0;
-    const score = Math.min(CONTENT_MAX_SCORE, base + tierBoost);
-    scored.push({
-      ...rowToHit(row, score),
-      line: matches[0].line,
-      lineText: matches[0].text,
-      matches
-    });
+    if (rows.length < pageSize)
+      break;
   }
   return scored.sort((a, b) => b.score - a.score).slice(0, limit);
 }
@@ -26620,7 +26798,7 @@ class FilesProvider {
     return hasReadyRoot();
   }
   async search(query, options) {
-    autoRefreshStaleRoots();
+    scheduleAutoRefreshStaleRoots();
     const hits = searchFilePaths(query, { limit: options?.limit ?? 10 });
     return hits.map((hit) => ({
       title: hit.name,
@@ -26648,7 +26826,7 @@ class ContentProvider {
     return hasReadyRoot();
   }
   async search(query, options) {
-    autoRefreshStaleRoots();
+    scheduleAutoRefreshStaleRoots();
     const hits = searchFileContent(query, { limit: options?.limit ?? 10 });
     return hits.map((hit) => ({
       title: hit.name,
@@ -26775,7 +26953,7 @@ function rowToSearch(row) {
 }
 function createSearch(data, db) {
   const d = db ?? getDb();
-  const id = generateId();
+  const id = data.id ?? generateId();
   const now = new Date().toISOString();
   d.prepare(`INSERT INTO searches (id, query, providers, profile_id, result_count, duration, created_at)
      VALUES (?, ?, ?, ?, ?, ?, ?)`).run(id, data.query, JSON.stringify(data.providers), data.profileId ?? null, data.resultCount ?? 0, data.duration ?? 0, now);
@@ -26857,7 +27035,7 @@ function createResults(results, db) {
   d.exec("BEGIN");
   try {
     for (const data of results) {
-      const id = generateId();
+      const id = data.id ?? generateId();
       stmt.run(id, data.searchId, data.title, data.url, data.snippet, data.source, data.provider, data.rank, data.score ?? null, data.publishedAt ?? null, data.thumbnail ?? null, JSON.stringify(data.metadata ?? {}), now);
       created.push({
         id,
@@ -27014,13 +27192,300 @@ function isProviderConfigured(provider) {
   return !!Bun.env[provider.apiKeyEnv];
 }
 
+// src/lib/router.ts
+var PROVIDER_DESCRIPTIONS = {
+  files: "Local file names and paths. Best for known filenames, path fragments, extensions, and repo navigation.",
+  content: "Local indexed file contents. Best for code symbols, exact phrases, docs, snippets, and grep-style discovery.",
+  google: "General web search through SerpAPI. Best for broad web coverage and current public pages.",
+  serpapi: "SerpAPI multi-engine web search. Best for general web queries when Google-style results are desired.",
+  exa: "Neural/semantic web search. Best for research, conceptual queries, docs, and high-relevance pages.",
+  perplexity: "Answer-oriented web research with citations. Best for synthesized factual questions and research summaries.",
+  brave: "General independent web search. Best for current web, news-like, product, and navigational queries.",
+  bing: "General web search. Best for current web and Microsoft/Bing-indexed pages.",
+  twitter: "X/Twitter search. Best for tweets, social reactions, breaking discourse, and people posting updates.",
+  reddit: "Reddit search. Best for opinions, product experiences, troubleshooting threads, and community recommendations.",
+  youtube: "YouTube search. Best for videos, tutorials, talks, demos, and channels.",
+  hackernews: "Hacker News search. Best for startup, programming, launch, and technical discussion threads.",
+  github: "GitHub code and repository search. Best for open-source repos, code examples, packages, and implementation details.",
+  arxiv: "arXiv academic search. Best for papers, preprints, ML/AI/math/physics research, and scholarly topics."
+};
+function clampMaxProviders(value) {
+  if (value === undefined || !Number.isFinite(value))
+    return 3;
+  return Math.max(1, Math.min(5, Math.floor(value)));
+}
+function clampConfidence(value) {
+  return typeof value === "number" && Number.isFinite(value) ? Math.max(0, Math.min(1, value)) : 0.5;
+}
+function normalizeCandidates(candidates) {
+  const allowed = new Set(PROVIDER_NAMES);
+  const seen = new Set;
+  const normalized = [];
+  for (const candidate of candidates) {
+    if (!allowed.has(candidate) || seen.has(candidate))
+      continue;
+    seen.add(candidate);
+    normalized.push(candidate);
+  }
+  return normalized;
+}
+function addScore(scores, candidateSet, provider, amount) {
+  if (!candidateSet.has(provider))
+    return;
+  scores.set(provider, (scores.get(provider) ?? 0) + amount);
+}
+function hasAny(query, patterns) {
+  return patterns.some((pattern) => pattern.test(query));
+}
+function routeSearchProvidersHeuristic(query, candidates, options = {}) {
+  const normalized = normalizeCandidates(candidates);
+  const maxProviders = clampMaxProviders(options.maxProviders);
+  if (normalized.length === 0) {
+    return {
+      strategy: "heuristic",
+      selectedProviders: [],
+      candidates: [],
+      reason: "No configured providers were available to route.",
+      confidence: 0
+    };
+  }
+  const candidateSet = new Set(normalized);
+  const scores = new Map;
+  const reasons = [];
+  const q = query.trim().toLowerCase();
+  for (const candidate of normalized)
+    scores.set(candidate, 0.05);
+  if (hasAny(q, [
+    /\b(file|filename|path|folder|directory|repo|workspace)\b/,
+    /(^|[/\s])[\w.-]+\.(ts|tsx|js|jsx|py|rs|go|md|json|yaml|yml|css|html)\b/
+  ])) {
+    addScore(scores, candidateSet, "files", 5);
+    addScore(scores, candidateSet, "content", 3);
+    reasons.push("query looks local-file oriented");
+  }
+  if (hasAny(q, [
+    /\b(function|class|interface|type|const|import|export|error|stack|symbol|grep|regex)\b/,
+    /[A-Za-z_$][\w$]*\([^)]*\)/,
+    /[A-Za-z_$][\w$]*::[A-Za-z_$]/
+  ])) {
+    addScore(scores, candidateSet, "content", 5);
+    addScore(scores, candidateSet, "files", 2);
+    addScore(scores, candidateSet, "github", 1.5);
+    reasons.push("query contains code/content lookup signals");
+  }
+  if (hasAny(q, [/\b(paper|papers|arxiv|preprint|doi|citation|survey|benchmark|research)\b/])) {
+    addScore(scores, candidateSet, "arxiv", 5);
+    addScore(scores, candidateSet, "exa", 3);
+    addScore(scores, candidateSet, "perplexity", 2);
+    reasons.push("query asks for scholarly or research material");
+  }
+  if (hasAny(q, [/\b(github|repo|repository|source code|open source|package|library|sdk|api example)\b/])) {
+    addScore(scores, candidateSet, "github", 5);
+    addScore(scores, candidateSet, "exa", 2);
+    reasons.push("query asks for code or repository material");
+  }
+  if (hasAny(q, [/\b(video|youtube|tutorial|demo|talk|lecture|channel)\b/])) {
+    addScore(scores, candidateSet, "youtube", 5);
+    reasons.push("query asks for video material");
+  }
+  if (hasAny(q, [/\b(reddit|subreddit|opinion|experience|reviews?|worth it|recommendations?)\b/])) {
+    addScore(scores, candidateSet, "reddit", 5);
+    addScore(scores, candidateSet, "hackernews", 1.5);
+    reasons.push("query asks for community discussion");
+  }
+  if (hasAny(q, [/\b(hacker news|hn|show hn|launch|startup)\b/])) {
+    addScore(scores, candidateSet, "hackernews", 5);
+    reasons.push("query asks for Hacker News style discussion");
+  }
+  if (hasAny(q, [/\b(twitter|tweet|tweets|x\.com|social reaction|trending)\b/])) {
+    addScore(scores, candidateSet, "twitter", 5);
+    reasons.push("query asks for social posts");
+  }
+  if (hasAny(q, [/\b(latest|today|yesterday|news|current|2025|2026|price|release|launched)\b/])) {
+    addScore(scores, candidateSet, "brave", 3);
+    addScore(scores, candidateSet, "bing", 2.5);
+    addScore(scores, candidateSet, "google", 2.5);
+    addScore(scores, candidateSet, "serpapi", 2);
+    reasons.push("query appears time-sensitive");
+  }
+  if (reasons.length === 0) {
+    addScore(scores, candidateSet, "exa", 2.5);
+    addScore(scores, candidateSet, "perplexity", 2);
+    addScore(scores, candidateSet, "brave", 1.5);
+    addScore(scores, candidateSet, "google", 1.5);
+    addScore(scores, candidateSet, "hackernews", 0.75);
+    reasons.push("general query fallback");
+  }
+  const selectedProviders = [...scores.entries()].sort((a, b) => b[1] - a[1] || normalized.indexOf(a[0]) - normalized.indexOf(b[0])).slice(0, Math.min(maxProviders, normalized.length)).map(([provider]) => provider);
+  const topScore = scores.get(selectedProviders[0]) ?? 0;
+  const confidence = Math.max(0.35, Math.min(0.9, topScore / 6));
+  return {
+    strategy: "heuristic",
+    selectedProviders,
+    candidates: normalized,
+    reason: reasons.join("; "),
+    confidence
+  };
+}
+function routerSchema(candidates, maxProviders) {
+  return {
+    type: "object",
+    properties: {
+      selectedProviders: {
+        type: "array",
+        items: { type: "string", enum: candidates },
+        minItems: 1,
+        maxItems: maxProviders
+      },
+      reason: { type: "string" },
+      confidence: { type: "number", minimum: 0, maximum: 1 }
+    },
+    required: ["selectedProviders", "reason", "confidence"],
+    additionalProperties: false
+  };
+}
+function parseCerebrasRouting(raw, candidates, maxProviders) {
+  let parsed;
+  try {
+    parsed = JSON.parse(raw);
+  } catch {
+    return null;
+  }
+  if (!Array.isArray(parsed.selectedProviders))
+    return null;
+  const candidateSet = new Set(candidates);
+  const selectedProviders = parsed.selectedProviders.filter((provider) => typeof provider === "string" && candidateSet.has(provider)).slice(0, maxProviders);
+  if (selectedProviders.length === 0)
+    return null;
+  return {
+    selectedProviders,
+    reason: typeof parsed.reason === "string" ? parsed.reason : "Cerebras router selected providers.",
+    confidence: clampConfidence(parsed.confidence)
+  };
+}
+async function routeWithCerebras(query, candidates, options) {
+  const apiKey = Bun.env.CEREBRAS_API_KEY;
+  if (!apiKey) {
+    return {
+      ...routeSearchProvidersHeuristic(query, candidates, options),
+      error: "CEREBRAS_API_KEY is not configured; used heuristic routing."
+    };
+  }
+  const providerGuide = candidates.map((name) => ({
+    name,
+    description: PROVIDER_DESCRIPTIONS[name]
+  }));
+  const res = await fetch("https://api.cerebras.ai/v1/chat/completions", {
+    method: "POST",
+    signal: AbortSignal.timeout(options.timeoutMs),
+    headers: {
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${apiKey}`
+    },
+    body: JSON.stringify({
+      model: options.model,
+      temperature: 0,
+      messages: [
+        {
+          role: "system",
+          content: "You route a search query to the smallest useful set of available search providers. Select only listed providers. Prefer local providers for local files/code in the indexed workspace. Prefer scholarly, code, video, social, or web providers when the query clearly asks for those domains."
+        },
+        {
+          role: "user",
+          content: JSON.stringify({
+            query,
+            maxProviders: options.maxProviders,
+            providers: providerGuide
+          })
+        }
+      ],
+      response_format: {
+        type: "json_schema",
+        json_schema: {
+          name: "search_router",
+          strict: true,
+          schema: routerSchema(candidates, options.maxProviders)
+        }
+      }
+    })
+  });
+  if (!res.ok) {
+    throw new Error(`Cerebras router error: ${res.status} ${res.statusText}`);
+  }
+  const data = await res.json();
+  const content = data.choices?.[0]?.message?.content;
+  if (!content)
+    throw new Error("Cerebras router returned no content");
+  const parsed = parseCerebrasRouting(content, candidates, options.maxProviders);
+  if (!parsed)
+    throw new Error("Cerebras router returned invalid provider selection");
+  return {
+    strategy: "cerebras",
+    candidates,
+    ...parsed
+  };
+}
+async function routeSearchProviders(query, candidates, options = {}) {
+  const normalized = normalizeCandidates(candidates);
+  const maxProviders = Math.min(clampMaxProviders(options.maxProviders), Math.max(1, normalized.length));
+  const timeoutMs = options.timeoutMs && Number.isFinite(options.timeoutMs) ? Math.max(250, Math.floor(options.timeoutMs)) : 1200;
+  const model = options.model ?? Bun.env.CEREBRAS_MODEL ?? "gpt-oss-120b";
+  if (normalized.length === 0) {
+    return routeSearchProvidersHeuristic(query, normalized, { maxProviders });
+  }
+  try {
+    return await routeWithCerebras(query, normalized, { maxProviders, timeoutMs, model });
+  } catch (err) {
+    return {
+      ...routeSearchProvidersHeuristic(query, normalized, { maxProviders }),
+      error: err instanceof Error ? err.message : String(err)
+    };
+  }
+}
+
 // src/lib/search.ts
+async function withTimeout(promise2, timeoutMs, label) {
+  if (!Number.isFinite(timeoutMs) || timeoutMs <= 0)
+    return promise2;
+  let timer;
+  try {
+    return await Promise.race([
+      promise2,
+      new Promise((_resolve, reject) => {
+        timer = setTimeout(() => reject(new Error(`${label} timed out after ${timeoutMs}ms`)), timeoutMs);
+        timer.unref?.();
+      })
+    ]);
+  } finally {
+    if (timer)
+      clearTimeout(timer);
+  }
+}
+async function allSettledLimited(items, concurrency, task) {
+  const results = new Array(items.length);
+  let next = 0;
+  async function worker() {
+    while (next < items.length) {
+      const index = next++;
+      const item = items[index];
+      try {
+        results[index] = { status: "fulfilled", value: await task(item) };
+      } catch (reason) {
+        results[index] = { status: "rejected", reason };
+      }
+    }
+  }
+  const workerCount = Math.min(Math.max(1, Math.floor(concurrency)), items.length);
+  await Promise.all(Array.from({ length: workerCount }, () => worker()));
+  return results;
+}
 async function unifiedSearch(query, opts = {}) {
   const config2 = getConfig();
   const startTime = Date.now();
   const db = opts.db;
   let providerNames = opts.providers ?? [];
-  if (opts.profile) {
+  const smartProfile = opts.profile === "smart";
+  if (opts.profile && !smartProfile) {
     const profile = getProfileByName(opts.profile, db);
     if (profile) {
       providerNames = profile.providers;
@@ -27036,7 +27501,7 @@ async function unifiedSearch(query, opts = {}) {
   }
   const errors4 = [];
   const explicitRequest = (opts.providers?.length ?? 0) > 0 || Boolean(opts.profile);
-  const activeProviders = providerNames.filter((name) => {
+  let activeProviders = providerNames.filter((name) => {
     try {
       if (getProvider(name).isConfigured())
         return true;
@@ -27047,20 +27512,36 @@ async function unifiedSearch(query, opts = {}) {
         });
       }
       return false;
-    } catch {
+    } catch (err) {
+      if (explicitRequest) {
+        errors4.push({
+          provider: name,
+          error: err instanceof Error ? err.message : "unknown provider"
+        });
+      }
       return false;
     }
   });
+  const routingRequested = opts.smart === true || smartProfile || !explicitRequest && config2.router.enabled;
+  let routing;
+  if (routingRequested && activeProviders.length > 0) {
+    routing = await routeSearchProviders(query, activeProviders, {
+      maxProviders: config2.router.maxProviders,
+      timeoutMs: config2.router.timeoutMs,
+      model: config2.router.model
+    });
+    activeProviders = routing.selectedProviders;
+  }
   const searchOptions = {
     limit: config2.defaultLimit,
     ...opts.options
   };
-  const results = await Promise.allSettled(activeProviders.map(async (name) => {
+  const results = await allSettledLimited(activeProviders, config2.maxConcurrent, async (name) => {
     const provider = getProvider(name);
-    const rawResults = await provider.search(query, searchOptions);
+    const rawResults = await withTimeout(provider.search(query, searchOptions), config2.providerTimeoutMs, provider.displayName);
     updateProviderLastUsed(name, db);
     return { name, results: rawResults };
-  }));
+  });
   const allResults = [];
   const searchId = generateId();
   for (const result of results) {
@@ -27114,11 +27595,13 @@ async function unifiedSearch(query, opts = {}) {
         createdAt: new Date().toISOString()
       },
       results: finalResults,
-      errors: errors4
+      errors: errors4,
+      ...routing && { routing }
     };
   }
   const persistable = config2.recordLocalResults ? finalResults : finalResults.filter((r) => !LOCAL_PROVIDER_NAMES.has(r.source));
   const search = createSearch({
+    id: searchId,
     query,
     providers: activeProviders,
     resultCount: persistable.length,
@@ -27127,6 +27610,7 @@ async function unifiedSearch(query, opts = {}) {
   if (persistable.length > 0) {
     createResults(persistable.map((r) => ({
       searchId: search.id,
+      id: r.id,
       title: r.title,
       url: r.url,
       snippet: r.snippet,
@@ -27143,7 +27627,8 @@ async function unifiedSearch(query, opts = {}) {
   return {
     search: { ...search, resultCount: finalResults.length, duration: duration3 },
     results: finalResults,
-    errors: errors4
+    errors: errors4,
+    ...routing && { routing }
   };
 }
 async function searchSingleProvider(provider, query, options, db) {
@@ -27377,7 +27862,7 @@ function findLocal(query, opts = {}, db) {
     return { query, kind, indexed: false, roots: roots.length, total: 0, results: [] };
   }
   if (opts.refresh !== false)
-    autoRefreshStaleRoots(db);
+    scheduleAutoRefreshStaleRoots(db);
   const queryOpts = {
     root: opts.root,
     ext: opts.ext,
@@ -27988,13 +28473,15 @@ function buildServer() {
     providers: exports_external.array(SearchProviderNameSchema).optional().describe("Providers to search (default: all enabled)"),
     profile: exports_external.string().optional().describe("Search profile name (e.g. research, social, code)"),
     limit: exports_external.number().int().min(1).max(100).optional().describe("Max results per provider"),
-    dedup: exports_external.boolean().optional().describe("Deduplicate results by URL (default: true)")
-  }, async ({ query, providers, profile, limit, dedup }) => {
+    dedup: exports_external.boolean().optional().describe("Deduplicate results by URL (default: true)"),
+    smart: exports_external.boolean().optional().describe("Route to the best configured providers before searching")
+  }, async ({ query, providers, profile, limit, dedup, smart }) => {
     const response = await unifiedSearch(query, {
       providers,
       profile,
       options: limit ? { limit } : undefined,
-      dedup
+      dedup,
+      smart
     });
     return {
       content: [
@@ -28013,7 +28500,8 @@ function buildServer() {
               source: r.source,
               score: r.score
             })),
-            errors: response.errors
+            errors: response.errors,
+            routing: response.routing
           }, null, 2)
         }
       ]
@@ -28268,12 +28756,14 @@ function buildServer() {
     default_limit: exports_external.number().int().optional(),
     dedup: exports_external.boolean().optional(),
     max_concurrent: exports_external.number().int().optional(),
+    provider_timeout_ms: exports_external.number().int().optional(),
     default_profile: exports_external.string().nullable().optional()
   }, async (updates) => {
     const config2 = setConfig({
       ...updates.default_limit !== undefined && { defaultLimit: updates.default_limit },
       ...updates.dedup !== undefined && { dedup: updates.dedup },
       ...updates.max_concurrent !== undefined && { maxConcurrent: updates.max_concurrent },
+      ...updates.provider_timeout_ms !== undefined && { providerTimeoutMs: updates.provider_timeout_ms },
       ...updates.default_profile !== undefined && { defaultProfile: updates.default_profile }
     });
     return {
@@ -29537,6 +30027,7 @@ if (handleCliFlags(argv)) {
   process.exit(0);
 }
 async function main() {
+  startBackgroundRefresh();
   if (isHttpMode(argv)) {
     startMcpHttpServer({ port: resolveMcpHttpPort(argv) });
     return;