limbo-ai 1.20.3 → 1.21.0

package/cli.js

@@ -159,12 +159,11 @@ function composeContent() {
       - ${VAULT_DIR}:/data/vault
       - limbo-zeroclaw-state:/home/limbo/.zeroclaw
     secrets:
-      - source: llm_api_key
-        mode: 0444
-      - source: telegram_bot_token
-        mode: 0444
-      - source: gateway_token
-        mode: 0444
+      - llm_api_key
+      - telegram_bot_token
+      - gateway_token
+      - groq_api_key
+      - brave_api_key
     env_file:
       - ${LIMBO_DIR}/.env
     environment:
@@ -186,6 +185,10 @@ secrets:
     file: ${SECRETS_DIR}/telegram_bot_token
   gateway_token:
     file: ${SECRETS_DIR}/gateway_token
+  groq_api_key:
+    file: ${SECRETS_DIR}/groq_api_key
+  brave_api_key:
+    file: ${SECRETS_DIR}/brave_api_key
 
 volumes:
   limbo-data:
@@ -218,12 +221,11 @@ function composeContentHardened() {
       - ${VAULT_DIR}:/data/vault
       - limbo-zeroclaw-state:/home/limbo/.zeroclaw
     secrets:
-      - source: llm_api_key
-        mode: 0444
-      - source: telegram_bot_token
-        mode: 0444
-      - source: gateway_token
-        mode: 0444
+      - llm_api_key
+      - telegram_bot_token
+      - gateway_token
+      - groq_api_key
+      - brave_api_key
     env_file:
       - ${LIMBO_DIR}/.env
     environment:
@@ -276,6 +278,10 @@ secrets:
     file: ${SECRETS_DIR}/telegram_bot_token
   gateway_token:
     file: ${SECRETS_DIR}/gateway_token
+  groq_api_key:
+    file: ${SECRETS_DIR}/groq_api_key
+  brave_api_key:
+    file: ${SECRETS_DIR}/brave_api_key
 
 volumes:
   limbo-data:
@@ -705,6 +711,8 @@ function normalizeConfig(cfg, existingEnv = {}) {
     TELEGRAM_BOT_TOKEN: cfg.telegramToken || (cfg.keepExisting ? existingEnv.TELEGRAM_BOT_TOKEN || '' : ''),
     TELEGRAM_AUTO_PAIR_FIRST_DM: cfg.telegramAutoPair || existingEnv.TELEGRAM_AUTO_PAIR_FIRST_DM || 'false',
     GATEWAY_TOKEN: gatewayToken,
+    VOICE_ENABLED: cfg.voiceEnabled || existingEnv.VOICE_ENABLED || 'false',
+    WEB_SEARCH_ENABLED: cfg.webSearchEnabled || existingEnv.WEB_SEARCH_ENABLED || 'false',
   };
 
   return base;
@@ -713,7 +721,10 @@ function normalizeConfig(cfg, existingEnv = {}) {
 function writeSecretFile(name, value) {
   fs.mkdirSync(SECRETS_DIR, { recursive: true, mode: 0o700 });
   const filePath = path.join(SECRETS_DIR, name);
-  fs.writeFileSync(filePath, value || '', { mode: 0o600 });
+  // Use 0644 so any container user can read the mounted file.
+  // Docker Compose file-based secrets ignore uid/gid/mode settings,
+  // so the host file permissions are what the container sees.
+  fs.writeFileSync(filePath, value || '', { mode: 0o644 });
 }
 
 function writeSecrets(cfg, existingEnv = {}) {
@@ -721,6 +732,8 @@ function writeSecrets(cfg, existingEnv = {}) {
   writeSecretFile('llm_api_key', normalized.LLM_API_KEY);
   writeSecretFile('telegram_bot_token', normalized.TELEGRAM_BOT_TOKEN);
   writeSecretFile('gateway_token', normalized.GATEWAY_TOKEN);
+  writeSecretFile('groq_api_key', cfg.groqApiKey || readSecretFile('groq_api_key'));
+  writeSecretFile('brave_api_key', cfg.braveApiKey || readSecretFile('brave_api_key'));
 }
 
 const SECRET_KEYS = new Set([
@@ -925,9 +938,9 @@ function ensureComposeFile(hardened = false) {
   fs.mkdirSync(path.join(VAULT_DIR, 'maps'), { recursive: true });
   fs.mkdirSync(SECRETS_DIR, { recursive: true, mode: 0o700 });
   // Ensure secret files exist (Docker Compose secrets require the files to be present)
-  for (const name of ['llm_api_key', 'telegram_bot_token', 'gateway_token']) {
+  for (const name of ['llm_api_key', 'telegram_bot_token', 'gateway_token', 'groq_api_key', 'brave_api_key']) {
     const fp = path.join(SECRETS_DIR, name);
-    if (!fs.existsSync(fp)) fs.writeFileSync(fp, '', { mode: 0o600 });
+    if (!fs.existsSync(fp)) fs.writeFileSync(fp, '', { mode: 0o644 });
   }
   if (hardened) {
     // Copy squid config files for egress filtering
@@ -1686,6 +1699,98 @@ function cmdStatus() {
   run('docker compose ps');
 }
 
+function cmdConfig() {
+  const args = process.argv.slice(3);
+  const feature = args[0];
+
+  if (!feature || !['voice', 'web-search'].includes(feature)) {
+    console.log(`
+${c.bold}Usage:${c.reset}
+  limbo config voice --enable --api-key <key>
+  limbo config voice --disable
+  limbo config voice --status
+  limbo config web-search --enable --api-key <key>
+  limbo config web-search --disable
+  limbo config web-search --status
+`);
+    return;
+  }
+
+  if (!fs.existsSync(ENV_FILE)) {
+    die('Limbo is not configured. Run "limbo start" first.');
+  }
+
+  const existingEnv = {};
+  const envContent = fs.readFileSync(ENV_FILE, 'utf8');
+  for (const line of envContent.split('\n')) {
+    const match = line.match(/^([A-Z_]+)=(.*)$/);
+    if (match) existingEnv[match[1]] = match[2].replace(/^"|"$/g, '');
+  }
+
+  const isVoice = feature === 'voice';
+  const envKey = isVoice ? 'VOICE_ENABLED' : 'WEB_SEARCH_ENABLED';
+  const secretName = isVoice ? 'groq_api_key' : 'brave_api_key';
+  const featureLabel = isVoice ? 'Voice transcription' : 'Web search';
+
+  const hasEnable = args.includes('--enable');
+  const hasDisable = args.includes('--disable');
+  const hasStatus = args.includes('--status');
+  const apiKeyIdx = args.indexOf('--api-key');
+  const apiKey = apiKeyIdx !== -1 ? args[apiKeyIdx + 1] : null;
+
+  if (hasStatus) {
+    const enabled = existingEnv[envKey] === 'true';
+    const key = readSecretFile(secretName);
+    console.log(`${featureLabel}: ${enabled ? `${c.green}enabled${c.reset}` : `${c.dim}disabled${c.reset}`}`);
+    if (key) {
+      const masked = key.length > 8 ? key.substring(0, 4) + '...' + key.slice(-4) : '***';
+      console.log(`API Key: ${masked}`);
+    }
+    return;
+  }
+
+  if (!hasEnable && !hasDisable) {
+    die('Specify --enable, --disable, or --status');
+  }
+
+  if (hasEnable) {
+    if (apiKey) {
+      if (isVoice && !apiKey.startsWith('gsk_')) {
+        warn('Groq API keys typically start with gsk_');
+      }
+      if (!isVoice && !apiKey.startsWith('BSA')) {
+        warn('Brave API keys typically start with BSA');
+      }
+      writeSecretFile(secretName, apiKey);
+      ok(`${featureLabel} API key saved.`);
+    } else {
+      const existing = readSecretFile(secretName);
+      if (!existing) {
+        die(`No API key found. Use --api-key <key> to set one.`);
+      }
+    }
+    existingEnv[envKey] = 'true';
+  } else {
+    existingEnv[envKey] = 'false';
+  }
+
+  const newContent = Object.entries(existingEnv)
+    .map(([key, value]) => `${key}=${value}`)
+    .join('\n') + '\n';
+  fs.writeFileSync(ENV_FILE, newContent, { mode: 0o600 });
+  ok(`${featureLabel} ${hasEnable ? 'enabled' : 'disabled'}.`);
+
+  if (fs.existsSync(COMPOSE_FILE)) {
+    log('Restarting container...');
+    try {
+      execSync(`docker compose -f "${COMPOSE_FILE}" restart limbo`, { stdio: 'inherit' });
+      ok('Container restarted.');
+    } catch {
+      warn('Could not restart container. Restart manually with: limbo stop && limbo start');
+    }
+  }
+}
+
 function cmdHelp() {
   console.log(`
 ${c.bold}limbo${c.reset} - personal AI memory agent
@@ -1699,6 +1804,7 @@ ${c.bold}Commands:${c.reset}
   logs          Tail container logs
   update        Pull latest image and restart
   status        Show container status
+  config        Configure optional features (voice, web-search)
   help          Show this help
 
 ${c.bold}Flags:${c.reset}
@@ -1711,6 +1817,13 @@ ${c.bold}Flags:${c.reset}
   --language <code>    Language: en, es (default: en)
   --tunnel-domain <d>  Admin: use branded subdomain for setup tunnel (e.g. limbo.tomasward.com)
 
+${c.bold}Config:${c.reset}
+  limbo config voice --enable --api-key gsk_xxx     Enable voice transcription
+  limbo config voice --disable                       Disable voice transcription
+  limbo config web-search --enable --api-key BSAxxx  Enable web search
+  limbo config web-search --disable                  Disable web search
+  limbo config voice --status                        Show feature status
+
 ${c.bold}Data directory:${c.reset} ${LIMBO_DIR}
 `);
 }
@@ -1727,6 +1840,7 @@ const [,, cmd = 'start'] = process.argv;
     case 'logs':    cmdLogs(); break;
     case 'update':  cmdUpdate(); break;
     case 'status':  cmdStatus(); break;
+    case 'config':  cmdConfig(); break;
     case 'help':
     case '--help':
     case '-h':      cmdHelp(); break;

package/mcp-server/index.js

@@ -5,6 +5,7 @@ import {
   ListToolsRequestSchema,
 } from "@modelcontextprotocol/sdk/types.js";
 
+import { buildIndex } from "./vault-index.js";
 import { vaultSearch } from "./tools/search.js";
 import { vaultRead } from "./tools/read.js";
 import { vaultWriteNote } from "./tools/write.js";
@@ -13,7 +14,7 @@ import { vaultUpdateMap } from "./tools/update-map.js";
 const server = new Server(
   {
     name: "limbo-vault",
-    version: "1.1.0",
+    version: "1.2.0",
   },
   {
     capabilities: {
@@ -172,5 +173,9 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
 
 // ── Start ───────────────────────────────────────────────────────────────────
 
+// Build in-memory index before accepting connections
+const noteCount = await buildIndex();
+process.stderr.write(`[limbo-vault] Index built: ${noteCount} notes indexed\n`);
+
 const transport = new StdioServerTransport();
 await server.connect(transport);

package/mcp-server/test/benchmark.js

@@ -0,0 +1,365 @@
+/**
+ * Benchmark: old (filesystem scan) vs new (in-memory index)
+ *
+ * Creates a temporary vault with N notes, then compares:
+ * - vault_search latency
+ * - vault_read latency
+ * - vault_write_note + subsequent search latency
+ *
+ * Run: node test/benchmark.js [noteCount]
+ *   default noteCount = 200
+ */
+
+import { mkdir, writeFile, rm } from "fs/promises";
+import { join, basename, relative } from "path";
+import { tmpdir } from "os";
+import { randomUUID } from "crypto";
+
+// ── Scaffold a temp vault ──────────────────────────────────────────────────
+
+const NOTE_COUNT = parseInt(process.argv[2] || "200", 10);
+const VAULT_DIR = join(tmpdir(), `limbo-bench-${randomUUID().slice(0, 8)}`);
+const NOTES_DIR = join(VAULT_DIR, "notes");
+const MAPS_DIR = join(VAULT_DIR, "maps");
+
+const DOMAINS = ["personal", "research", "projects", "aios", "limbo"];
+const TYPES = ["fact", "preference", "idea", "insight", "decision"];
+
+function generateNote(i) {
+  const domain = DOMAINS[i % DOMAINS.length];
+  const type = TYPES[i % TYPES.length];
+  const id = `bench-note-${String(i).padStart(4, "0")}`;
+  const title = `Benchmark note number ${i} about ${domain}`;
+  const description = `This is test note ${i} in the ${domain} domain for performance benchmarking`;
+  const body = [
+    `This note contains searchable content for domain ${domain}.`,
+    `Keywords: optimization, performance, latency, throughput, indexing.`,
+    `Note index: ${i}. UUID: ${randomUUID()}.`,
+    `The quick brown fox jumps over the lazy dog.`,
+    i % 7 === 0 ? "Special keyword: UNIQUE_NEEDLE" : "",
+    i % 3 === 0 ? `Reference to [[bench-note-${String(i - 1).padStart(4, "0")}]]` : "",
+  ].join("\n\n");
+
+  const frontmatter = [
+    "---",
+    `id: ${id}`,
+    `title: "${title}"`,
+    `description: "${description}"`,
+    `type: ${type}`,
+    `schema_version: 1`,
+    `domain: ${domain}`,
+    `created: "2026-03-19"`,
+    `source: benchmark`,
+    "topics:",
+    `  - "[[${domain}-map]]"`,
+    "---",
+  ].join("\n");
+
+  return { id, domain, content: `${frontmatter}\n\n${body}\n` };
+}
+
+async function scaffoldVault() {
+  await mkdir(NOTES_DIR, { recursive: true });
+  await mkdir(MAPS_DIR, { recursive: true });
+
+  const writes = [];
+  for (let i = 0; i < NOTE_COUNT; i++) {
+    const { id, domain, content } = generateNote(i);
+    const dir = join(NOTES_DIR, domain);
+    writes.push(
+      mkdir(dir, { recursive: true }).then(() =>
+        writeFile(join(dir, `${id}.md`), content, "utf8")
+      )
+    );
+  }
+  await Promise.all(writes);
+}
+
+async function cleanup() {
+  await rm(VAULT_DIR, { recursive: true, force: true });
+}
+
+// ── Old implementation (filesystem scan) ───────────────────────────────────
+
+import { readdir, readFile, stat } from "fs/promises";
+
+async function oldWalkNotes(dir, base = dir) {
+  const entries = [];
+  let items;
+  try {
+    items = await readdir(dir);
+  } catch {
+    return entries;
+  }
+  for (const item of items) {
+    if (item.startsWith(".") || item === "_meta") continue;
+    const full = join(dir, item);
+    let s;
+    try {
+      s = await stat(full);
+    } catch {
+      continue;
+    }
+    if (s.isDirectory()) {
+      const sub = await oldWalkNotes(full, base);
+      entries.push(...sub);
+    } else if (item.endsWith(".md")) {
+      const rel = relative(base, dir);
+      entries.push({ filePath: full, domain: rel || null });
+    }
+  }
+  return entries;
+}
+
+function oldExtractTitle(content) {
+  const fmMatch = content.match(/^---\s*\n([\s\S]*?)\n---/);
+  if (fmMatch) {
+    const titleMatch = fmMatch[1].match(/^title:\s*["']?(.+?)["']?\s*$/m);
+    if (titleMatch) return titleMatch[1];
+  }
+  return null;
+}
+
+async function oldSearch(query) {
+  const files = await oldWalkNotes(NOTES_DIR);
+  const escaped = query.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+  const regex = new RegExp(escaped, "gi");
+  const results = [];
+  for (const { filePath, domain } of files) {
+    let content;
+    try {
+      content = await readFile(filePath, "utf8");
+    } catch {
+      continue;
+    }
+    const matches = content.match(regex);
+    if (!matches) continue;
+    const noteId = basename(filePath, ".md");
+    const title = oldExtractTitle(content) || noteId;
+    results.push({ noteId, title, score: matches.length, domain });
+  }
+  results.sort((a, b) => b.score - a.score);
+  return results;
+}
+
+async function oldRead(noteId) {
+  // Fast path
+  const flatPath = join(NOTES_DIR, `${noteId}.md`);
+  try {
+    await stat(flatPath);
+    return await readFile(flatPath, "utf8");
+  } catch {}
+
+  // Recursive search
+  async function searchDir(dir) {
+    let items;
+    try {
+      items = await readdir(dir);
+    } catch {
+      return null;
+    }
+    for (const item of items) {
+      if (item.startsWith(".") || item === "_meta") continue;
+      const full = join(dir, item);
+      let s;
+      try {
+        s = await stat(full);
+      } catch {
+        continue;
+      }
+      if (s.isDirectory()) {
+        const candidate = join(full, `${noteId}.md`);
+        try {
+          await stat(candidate);
+          return await readFile(candidate, "utf8");
+        } catch {
+          const found = await searchDir(full);
+          if (found) return found;
+        }
+      }
+    }
+    return null;
+  }
+
+  return searchDir(NOTES_DIR);
+}
+
+// ── New implementation (in-memory index) ───────────────────────────────────
+
+// We inline the index here to avoid env var coupling with vault-index.js
+
+const index = new Map();
+
+function newExtractTitle(content) {
+  const fmMatch = content.match(/^---\s*\n([\s\S]*?)\n---/);
+  if (fmMatch) {
+    const titleMatch = fmMatch[1].match(/^title:\s*["']?(.+?)["']?\s*$/m);
+    if (titleMatch) return titleMatch[1];
+    const descMatch = fmMatch[1].match(/^description:\s*["']?(.+?)["']?\s*$/m);
+    if (descMatch) return descMatch[1];
+  }
+  return null;
+}
+
+async function newWalkAndIndex(dir, base = dir) {
+  let items;
+  try {
+    items = await readdir(dir);
+  } catch {
+    return;
+  }
+  const promises = [];
+  for (const item of items) {
+    if (item.startsWith(".") || item === "_meta") continue;
+    const full = join(dir, item);
+    promises.push(
+      stat(full)
+        .then((s) => {
+          if (s.isDirectory()) return newWalkAndIndex(full, base);
+          if (item.endsWith(".md")) {
+            return readFile(full, "utf8")
+              .then((content) => {
+                const noteId = basename(full, ".md");
+                const domain = relative(base, dir) || null;
+                const title = newExtractTitle(content) || noteId;
+                index.set(noteId, { path: full, title, content, domain });
+              })
+              .catch(() => {});
+          }
+        })
+        .catch(() => {})
+    );
+  }
+  await Promise.all(promises);
+}
+
+async function newBuildIndex() {
+  index.clear();
+  await newWalkAndIndex(NOTES_DIR);
+  return index.size;
+}
+
+function newSearch(query) {
+  const escaped = query.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+  const regex = new RegExp(escaped, "gi");
+  const results = [];
+  for (const [noteId, entry] of index) {
+    const matches = entry.content.match(regex);
+    if (!matches) continue;
+    results.push({ noteId, title: entry.title, score: matches.length, domain: entry.domain });
+  }
+  results.sort((a, b) => b.score - a.score);
+  return results;
+}
+
+function newRead(noteId) {
+  const entry = index.get(noteId);
+  return entry ? entry.content : null;
+}
+
+// ── Benchmark harness ──────────────────────────────────────────────────────
+
+async function timeMs(fn, iterations = 1) {
+  const times = [];
+  for (let i = 0; i < iterations; i++) {
+    const start = performance.now();
+    await fn();
+    times.push(performance.now() - start);
+  }
+  times.sort((a, b) => a - b);
+  return {
+    min: times[0],
+    median: times[Math.floor(times.length / 2)],
+    max: times[times.length - 1],
+    avg: times.reduce((a, b) => a + b, 0) / times.length,
+  };
+}
+
+function fmt(ms) {
+  if (ms < 1) return `${(ms * 1000).toFixed(0)}µs`;
+  return `${ms.toFixed(2)}ms`;
+}
+
+function printResult(label, old, now) {
+  const speedup = old.median / now.median;
+  console.log(`  ${label}`);
+  console.log(`    OLD:  median=${fmt(old.median)}  avg=${fmt(old.avg)}  min=${fmt(old.min)}  max=${fmt(old.max)}`);
+  console.log(`    NEW:  median=${fmt(now.median)}  avg=${fmt(now.avg)}  min=${fmt(now.min)}  max=${fmt(now.max)}`);
+  console.log(`    Speedup: ${speedup.toFixed(1)}x faster`);
+  console.log();
+}
+
+// ── Main ───────────────────────────────────────────────────────────────────
+
+console.log(`\n=== Limbo MCP Benchmark ===`);
+console.log(`Notes: ${NOTE_COUNT}`);
+console.log(`Vault: ${VAULT_DIR}\n`);
+
+try {
+  // Setup
+  console.log("Scaffolding vault...");
+  await scaffoldVault();
+
+  // Build index (one-time cost)
+  console.log("Building index...");
+  const buildTime = await timeMs(() => newBuildIndex(), 3);
+  console.log(`  Index build: median=${fmt(buildTime.median)} (${index.size} notes)\n`);
+
+  const ITERS = 20;
+
+  // ── Search: broad query (many matches) ─────────────────────────────────
+  console.log(`--- Search: broad query ("optimization") × ${ITERS} ---`);
+  const oldSearchBroad = await timeMs(() => oldSearch("optimization"), ITERS);
+  const newSearchBroad = await timeMs(() => newSearch("optimization"), ITERS);
+  printResult("Broad search", oldSearchBroad, newSearchBroad);
+
+  // ── Search: narrow query (few matches) ─────────────────────────────────
+  console.log(`--- Search: narrow query ("UNIQUE_NEEDLE") × ${ITERS} ---`);
+  const oldSearchNarrow = await timeMs(() => oldSearch("UNIQUE_NEEDLE"), ITERS);
+  const newSearchNarrow = await timeMs(() => newSearch("UNIQUE_NEEDLE"), ITERS);
+  printResult("Narrow search", oldSearchNarrow, newSearchNarrow);
+
+  // ── Search: no matches ─────────────────────────────────────────────────
+  console.log(`--- Search: miss ("xyzzy_nonexistent") × ${ITERS} ---`);
+  const oldSearchMiss = await timeMs(() => oldSearch("xyzzy_nonexistent"), ITERS);
+  const newSearchMiss = await timeMs(() => newSearch("xyzzy_nonexistent"), ITERS);
+  printResult("Miss search", oldSearchMiss, newSearchMiss);
+
+  // ── Read: note in subdirectory (worst case for old) ────────────────────
+  console.log(`--- Read: note in subdirectory × ${ITERS} ---`);
+  const readTarget = `bench-note-${String(Math.floor(NOTE_COUNT / 2)).padStart(4, "0")}`;
+  const oldReadSub = await timeMs(() => oldRead(readTarget), ITERS);
+  const newReadSub = await timeMs(() => newRead(readTarget), ITERS);
+  printResult(`Read "${readTarget}"`, oldReadSub, newReadSub);
+
+  // ── Read: nonexistent note ─────────────────────────────────────────────
+  console.log(`--- Read: nonexistent note × ${ITERS} ---`);
+  const oldReadMiss = await timeMs(() => oldRead("does-not-exist-999"), ITERS);
+  const newReadMiss = await timeMs(() => newRead("does-not-exist-999"), ITERS);
+  printResult("Read miss", oldReadMiss, newReadMiss);
+
+  // ── Correctness check ──────────────────────────────────────────────────
+  console.log("--- Correctness ---");
+  const oldResults = await oldSearch("optimization");
+  const newResults = newSearch("optimization");
+  const oldCount = oldResults.length;
+  const newCount = newResults.length;
+  const match = oldCount === newCount;
+  console.log(`  Old result count: ${oldCount}`);
+  console.log(`  New result count: ${newCount}`);
+  console.log(`  Match: ${match ? "PASS ✓" : "FAIL ✗"}`);
+  if (!match) {
+    console.log("  WARNING: result counts differ!");
+  }
+
+  // Verify read returns identical content
+  const oldContent = await oldRead(readTarget);
+  const newContent = newRead(readTarget);
+  const readMatch = oldContent === newContent;
+  console.log(`  Read content match: ${readMatch ? "PASS ✓" : "FAIL ✗"}`);
+  console.log();
+
+} finally {
+  await cleanup();
+  console.log("Cleaned up.\n");
+}