memex-mvp 0.5.0

package/lib/config.js

@@ -0,0 +1,179 @@
+/**
+ * memex configuration: ~/.memex/config.json
+ *
+ * Schema:
+ *   {
+ *     sources: {
+ *       claude_code:   true | false,
+ *       claude_cowork: true | false,
+ *       cursor:        true | false,
+ *       obsidian: true | false | { enabled: bool, vaults: string[] }
+ *     }
+ *   }
+ *
+ * Behavior:
+ *   - File missing → defaults below (everything ON if its data exists). Preserves
+ *     backward compat for users who installed before config was a thing.
+ *   - File present but partial → merged with defaults.
+ *   - Env var MEMEX_OBSIDIAN_VAULTS overrides config.sources.obsidian.vaults
+ *     (useful for cron/scripts without touching the file).
+ *
+ * CLI source names accept both "claude-code" and "claude_code" forms;
+ * normalizeSourceName() canonicalises to underscore (matches JSON keys).
+ */
+
+import { readFileSync, writeFileSync, existsSync, mkdirSync, renameSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join, dirname, resolve } from 'node:path';
+
+const HOME = homedir();
+const MEMEX_DIR = process.env.MEMEX_DIR || join(HOME, '.memex');
+export const CONFIG_PATH = join(MEMEX_DIR, 'config.json');
+
+export const KNOWN_SOURCES = ['claude_code', 'claude_cowork', 'cursor', 'obsidian'];
+
+/** What the daemon does when no config file exists — preserve current behavior. */
+export const DEFAULT_CONFIG = Object.freeze({
+  sources: {
+    claude_code: true,
+    claude_cowork: true,
+    cursor: true,
+    obsidian: { enabled: true, vaults: [] }, // empty vaults → autodetect
+  },
+  search: {
+    // Half-life in days for the temporal recency boost in memex_search.
+    // Score = bm25 * exp(-age_days / half_life). 30d ≈ recent week dominates,
+    // month-old halved, 3-month-old in long tail. Set to 0 to disable.
+    half_life_days: 30,
+  },
+});
+
+/** Returns the configured default half-life (days) for recency boost. 0 disables. */
+export function getSearchHalfLifeDays(config) {
+  const v = config && config.search && config.search.half_life_days;
+  if (typeof v !== 'number' || !isFinite(v) || v < 0) return 30;
+  return v;
+}
+
+/**
+ * Normalise a CLI source name. Accepts "claude-code", "claude_code", "code"
+ * (alias), "cowork" (alias). Returns canonical name or null.
+ */
+export function normalizeSourceName(input) {
+  if (!input) return null;
+  const s = String(input).toLowerCase().replace(/-/g, '_');
+  const aliases = {
+    code: 'claude_code',
+    cowork: 'claude_cowork',
+  };
+  const canonical = aliases[s] || s;
+  return KNOWN_SOURCES.includes(canonical) ? canonical : null;
+}
+
+export function loadConfig() {
+  if (!existsSync(CONFIG_PATH)) return clone(DEFAULT_CONFIG);
+  let raw;
+  try { raw = JSON.parse(readFileSync(CONFIG_PATH, 'utf-8')); }
+  catch (_) { return clone(DEFAULT_CONFIG); }
+  return mergeWithDefaults(raw, DEFAULT_CONFIG);
+}
+
+export function saveConfig(config) {
+  mkdirSync(dirname(CONFIG_PATH), { recursive: true });
+  const tmp = CONFIG_PATH + '.tmp';
+  writeFileSync(tmp, JSON.stringify(config, null, 2));
+  renameSync(tmp, CONFIG_PATH);
+}
+
+/**
+ * Is a given named source enabled?
+ *   - boolean → that
+ *   - object with .enabled → that
+ *   - undefined → default-on
+ */
+export function isSourceEnabled(name, config) {
+  const v = config.sources && config.sources[name];
+  if (v === undefined || v === null) return true;
+  if (typeof v === 'boolean') return v;
+  if (typeof v === 'object' && 'enabled' in v) return !!v.enabled;
+  return true;
+}
+
+/** Mutate config to set source enabled/disabled. Preserves nested structure for obsidian. */
+export function setSourceEnabled(name, enabled, config) {
+  if (!config.sources) config.sources = {};
+  const existing = config.sources[name];
+  if (typeof existing === 'object' && existing !== null) {
+    existing.enabled = !!enabled;
+  } else {
+    config.sources[name] = !!enabled;
+  }
+}
+
+/** Get configured Obsidian vault list (config + env var). Returns absolute paths. */
+export function obsidianVaultsFromConfig(config) {
+  const out = [];
+  const fromConfig = config.sources && config.sources.obsidian;
+  if (fromConfig && typeof fromConfig === 'object' && Array.isArray(fromConfig.vaults)) {
+    for (const v of fromConfig.vaults) out.push(expandTilde(v));
+  }
+  const fromEnv = (process.env.MEMEX_OBSIDIAN_VAULTS || '')
+    .split(',')
+    .map((s) => s.trim())
+    .filter(Boolean)
+    .map(expandTilde);
+  // Dedup, env wins
+  return [...new Set([...fromEnv, ...out])];
+}
+
+export function addObsidianVault(path, config) {
+  const abs = resolve(expandTilde(path));
+  if (!config.sources) config.sources = {};
+  if (!config.sources.obsidian || typeof config.sources.obsidian !== 'object') {
+    config.sources.obsidian = { enabled: true, vaults: [] };
+  }
+  if (!Array.isArray(config.sources.obsidian.vaults)) {
+    config.sources.obsidian.vaults = [];
+  }
+  if (!config.sources.obsidian.vaults.includes(abs)) {
+    config.sources.obsidian.vaults.push(abs);
+  }
+  return abs;
+}
+
+export function removeObsidianVault(path, config) {
+  const abs = resolve(expandTilde(path));
+  const obs = config.sources && config.sources.obsidian;
+  if (!obs || typeof obs !== 'object' || !Array.isArray(obs.vaults)) return false;
+  const before = obs.vaults.length;
+  obs.vaults = obs.vaults.filter((v) => resolve(expandTilde(v)) !== abs);
+  return obs.vaults.length !== before;
+}
+
+// -------------------- Internal helpers --------------------
+function clone(o) {
+  return JSON.parse(JSON.stringify(o));
+}
+
+function mergeWithDefaults(parsed, defaults) {
+  const out = clone(defaults);
+  if (!parsed || typeof parsed !== 'object') return out;
+  if (parsed.sources && typeof parsed.sources === 'object') {
+    for (const key of KNOWN_SOURCES) {
+      if (key in parsed.sources) out.sources[key] = parsed.sources[key];
+    }
+  }
+  if (parsed.search && typeof parsed.search === 'object') {
+    if (typeof parsed.search.half_life_days === 'number') {
+      out.search.half_life_days = parsed.search.half_life_days;
+    }
+  }
+  return out;
+}
+
+function expandTilde(p) {
+  if (!p) return p;
+  if (p === '~' || p === '~/') return HOME;
+  if (p.startsWith('~/')) return join(HOME, p.slice(2));
+  return p;
+}

package/lib/parse-cursor.js

@@ -0,0 +1,172 @@
+/**
+ * Cursor IDE history parser.
+ *
+ * Reads Cursor's local SQLite store (state.vscdb) and extracts
+ * Composer / Chat conversations as ingest-ready dialogue messages.
+ *
+ * Schema (verified 2026-05 on Cursor _v: 13 / bubble _v: 3):
+ *
+ *   cursorDiskKV table:
+ *     composerData:<composerId>          session metadata + ordered headers
+ *     bubbleId:<composerId>:<bubbleId>   individual message bubble
+ *     agentKv:* / checkpointId:*         ignored (not dialogue)
+ *     inlineDiff:* / composer.content.*  ignored
+ *
+ * Key insight from real-data probe: Cursor splits ONE logical assistant
+ * turn across MULTIPLE bubbles —
+ *   bubble.type=1 + bubble.text   = user prompt          (KEEP)
+ *   bubble.type=2 + bubble.thinking only = reasoning      (SKIP)
+ *   bubble.type=2 + bubble.text  = user-visible answer    (KEEP)
+ *   bubble.type=2 + bubble.toolFormerData = tool call    (SKIP)
+ *
+ * We keep only bubbles with a non-empty .text field. Same dialogue-only
+ * filter philosophy as our Claude Code/Cowork parser.
+ */
+
+import Database from 'better-sqlite3';
+import { homedir, platform } from 'node:os';
+import { join } from 'node:path';
+import { existsSync } from 'node:fs';
+
+export const CURSOR_DB_PATHS = {
+  darwin: join(homedir(), 'Library', 'Application Support', 'Cursor', 'User', 'globalStorage', 'state.vscdb'),
+  linux: join(homedir(), '.config', 'Cursor', 'User', 'globalStorage', 'state.vscdb'),
+  win32: join(homedir(), 'AppData', 'Roaming', 'Cursor', 'User', 'globalStorage', 'state.vscdb'),
+};
+
+export function defaultCursorDbPath() {
+  return CURSOR_DB_PATHS[platform()] || null;
+}
+
+/**
+ * Open Cursor's SQLite read-only with retry on SQLITE_BUSY.
+ * Cursor writes live; we use exponential backoff (100/300/900 ms).
+ * Returns null if the DB doesn't exist (Cursor not installed).
+ */
+export function openCursorDB(path) {
+  if (!path || !existsSync(path)) return null;
+  const delays = [100, 300, 900];
+  for (let attempt = 0; ; attempt++) {
+    try {
+      return new Database(path, { readonly: true, fileMustExist: true });
+    } catch (err) {
+      const busy = err && (err.code === 'SQLITE_BUSY' || err.code === 'SQLITE_LOCKED');
+      if (busy && attempt < delays.length) {
+        const start = Date.now();
+        // Synchronous busy-wait so this stays a sync function. Total worst-
+        // case ~1.3s, which is acceptable for a once-per-tick scan.
+        while (Date.now() - start < delays[attempt]) {}
+        continue;
+      }
+      throw err;
+    }
+  }
+}
+
+function parseValue(buf) {
+  if (!buf) return null;
+  try {
+    return JSON.parse(typeof buf === 'string' ? buf : buf.toString('utf8'));
+  } catch (_) {
+    return null;
+  }
+}
+
+/**
+ * Iterate all Composer sessions in the DB.
+ * Yields { composerId, name, createdAt, lastUpdatedAt, headers, isAgentic }.
+ */
+export function* iterComposers(db) {
+  const stmt = db.prepare(`SELECT value FROM cursorDiskKV WHERE key LIKE 'composerData:%'`);
+  for (const row of stmt.iterate()) {
+    const cd = parseValue(row.value);
+    if (!cd || !cd.composerId) continue;
+    yield {
+      composerId: cd.composerId,
+      name: cd.name || null,
+      createdAt: cd.createdAt || null,
+      lastUpdatedAt: cd.lastUpdatedAt || cd.createdAt || null,
+      isAgentic: !!cd.isAgentic,
+      headers: Array.isArray(cd.fullConversationHeadersOnly) ? cd.fullConversationHeadersOnly : [],
+    };
+  }
+}
+
+/**
+ * Extract dialogue messages from one composer.
+ * Skips thinking-only and tool-only bubbles (no .text content).
+ *
+ * Returns [{ role, text, bubbleId, ts }] in conversation order.
+ */
+export function extractDialogue(db, composer) {
+  const { composerId, createdAt, lastUpdatedAt, headers } = composer;
+  if (!headers.length) return [];
+
+  const bubbleStmt = db.prepare(`SELECT value FROM cursorDiskKV WHERE key = ?`);
+  const messages = [];
+
+  // Synthetic per-bubble timestamps. Real bubble timing is not reliably
+  // present, but createdAt → lastUpdatedAt range gives us a valid window.
+  const start = createdAt;
+  const span = Math.max(1, (lastUpdatedAt || createdAt) - start);
+  // We'll assign each KEPT message a ts that lives between start and end.
+  // To preserve order even when we skip bubbles, we use the header index
+  // for spacing.
+  const totalHeaders = headers.length;
+
+  for (let i = 0; i < headers.length; i++) {
+    const h = headers[i];
+    if (!h || !h.bubbleId) continue;
+
+    const row = bubbleStmt.get(`bubbleId:${composerId}:${h.bubbleId}`);
+    if (!row) continue; // truncated — skip silently
+
+    const b = parseValue(row.value);
+    if (!b) continue;
+
+    const text = typeof b.text === 'string' ? b.text.trim() : '';
+    if (!text) continue; // thinking-only / tool-only bubble — skip
+
+    const role = h.type === 1 ? 'user' : (h.type === 2 ? 'assistant' : null);
+    if (!role) continue;
+
+    const ts = start
+      ? Math.floor(start + (i / Math.max(1, totalHeaders - 1)) * span)
+      : null;
+
+    messages.push({
+      role,
+      text,
+      bubbleId: h.bubbleId,
+      ts, // unix ms
+    });
+  }
+
+  return messages;
+}
+
+/**
+ * Render dialogue + ai-title metadata as inbox-ready JSONL records.
+ * Records match the flat shape that memex's importClaudeCodeJsonl expects,
+ * so the inbox flow stays unified across sources.
+ *
+ * Returns: [recordObj, ...] — each is one JSONL line (caller serializes).
+ */
+export function composerToInboxRecords(composer, dialogue, prefix, shortId, hashFn) {
+  const records = [];
+  if (composer.name) {
+    records.push({ type: 'ai-title', aiTitle: composer.name });
+  }
+  for (const m of dialogue) {
+    const tsIso = m.ts ? new Date(m.ts).toISOString() : null;
+    const seed = `${m.role}|${tsIso}|${m.text.slice(0, 200)}`;
+    const msgId = hashFn(seed);
+    records.push({
+      role: m.role,
+      content: m.text,
+      timestamp: tsIso,
+      id: `${prefix}-${shortId}-${msgId}`,
+    });
+  }
+  return records;
+}

package/lib/parse-obsidian.js

@@ -0,0 +1,256 @@
+/**
+ * Obsidian vault parser.
+ *
+ * Walks a vault root directory (folder containing .obsidian/ subdir),
+ * yields one logical "conversation" per .md file. Each note becomes a
+ * single user-authored message in memex.
+ *
+ * Why one-note-per-conversation: notes don't have natural turn structure,
+ * splitting on H2 would be artificial and fragment search context.
+ * memex_search will return the whole note as one hit, with FTS5 snippet
+ * highlighting the matched terms — that's the right granularity for PKM.
+ *
+ * Privacy posture:
+ *   - Vault is opt-in; user provides path explicitly (env var or auto-detect).
+ *   - .obsidian/, .trash/, .git/ are skipped.
+ *   - Notes with frontmatter `memex: false` are skipped.
+ *   - .memexignore in vault root supports gitignore-style patterns (TODO).
+ *   - Sync-conflict files (e.g. "* (conflict).md", "*.sync-conflict-*") skipped.
+ */
+
+import { readFileSync, statSync, readdirSync, existsSync } from 'node:fs';
+import { join, relative, basename, sep } from 'node:path';
+import { homedir } from 'node:os';
+import { createHash } from 'node:crypto';
+
+/**
+ * Auto-detect Obsidian vaults in standard macOS locations.
+ * Returns array of absolute vault root paths.
+ */
+export function autodetectObsidianVaults() {
+  const candidates = [
+    join(homedir(), 'Documents'),
+    join(homedir(), 'Obsidian'),
+    join(homedir(), 'Library', 'Mobile Documents', 'iCloud~md~obsidian', 'Documents'),
+    join(homedir(), 'Documents', 'Obsidian'),
+  ];
+  const found = new Set();
+  for (const root of candidates) {
+    if (!existsSync(root)) continue;
+    walkForObsidianFolder(root, 3, found);
+  }
+  return [...found];
+}
+
+function walkForObsidianFolder(dir, depth, found) {
+  if (depth < 0) return;
+  let entries;
+  try { entries = readdirSync(dir, { withFileTypes: true }); }
+  catch (_) { return; }
+  // Is THIS dir a vault?
+  if (entries.some((e) => e.isDirectory() && e.name === '.obsidian')) {
+    found.add(dir);
+    return; // don't descend further into a vault — nested vaults are unusual
+  }
+  // Recurse one level deeper
+  for (const e of entries) {
+    if (!e.isDirectory()) continue;
+    if (e.name.startsWith('.')) continue;
+    if (e.name === 'node_modules' || e.name === 'Library') continue;
+    walkForObsidianFolder(join(dir, e.name), depth - 1, found);
+  }
+}
+
+/**
+ * Decide whether a path inside a vault should be skipped entirely.
+ */
+const SKIP_PATTERNS = [
+  /(^|\/)\.obsidian(\/|$)/,
+  /(^|\/)\.trash(\/|$)/,
+  /(^|\/)\.git(\/|$)/,
+  /(^|\/)node_modules(\/|$)/,
+  /\.DS_Store$/,
+  /\.sync-conflict-/,
+  /\(conflict\)\.md$/i,
+];
+export function shouldSkipPath(relPath) {
+  for (const re of SKIP_PATTERNS) if (re.test(relPath)) return true;
+  return false;
+}
+
+/**
+ * Walk a vault, yielding paths to .md files we should process.
+ */
+export function* walkVault(vaultRoot) {
+  const stack = [vaultRoot];
+  while (stack.length > 0) {
+    const dir = stack.pop();
+    let entries;
+    try { entries = readdirSync(dir, { withFileTypes: true }); }
+    catch (_) { continue; }
+    for (const e of entries) {
+      const full = join(dir, e.name);
+      const rel = relative(vaultRoot, full);
+      if (shouldSkipPath(rel)) continue;
+      if (e.isDirectory()) {
+        stack.push(full);
+      } else if (e.isFile() && e.name.endsWith('.md')) {
+        yield { absolute: full, relative: rel };
+      }
+    }
+  }
+}
+
+/**
+ * Minimal YAML parser for typical Obsidian frontmatter (flat key:value,
+ * inline arrays [a, b, c], multi-line "key:\n  - item" arrays). Returns
+ * an object. Unknown structures fall through silently.
+ */
+function parseSimpleYaml(text) {
+  const result = {};
+  let currentArrayKey = null;
+  for (const rawLine of text.split('\n')) {
+    const line = rawLine.replace(/\r$/, '');
+    if (!line.trim() || line.trim().startsWith('#')) continue;
+
+    // Multi-line array continuation: "  - item"
+    const arrM = line.match(/^\s+-\s+(.*)$/);
+    if (arrM && currentArrayKey && Array.isArray(result[currentArrayKey])) {
+      result[currentArrayKey].push(stripQuotes(arrM[1].trim()));
+      continue;
+    }
+
+    const kvM = line.match(/^([^:\s][^:]*):\s*(.*)$/);
+    if (!kvM) {
+      currentArrayKey = null;
+      continue;
+    }
+    const key = kvM[1].trim();
+    const value = kvM[2].trim();
+
+    if (value === '') {
+      // Multi-line array
+      currentArrayKey = key;
+      result[key] = [];
+    } else if (value.startsWith('[') && value.endsWith(']')) {
+      // Inline array
+      result[key] = value
+        .slice(1, -1)
+        .split(',')
+        .map((s) => stripQuotes(s.trim()))
+        .filter(Boolean);
+      currentArrayKey = null;
+    } else {
+      result[key] = coerceScalar(stripQuotes(value));
+      currentArrayKey = null;
+    }
+  }
+  return result;
+}
+
+function stripQuotes(s) {
+  if (!s) return s;
+  if ((s.startsWith('"') && s.endsWith('"')) || (s.startsWith("'") && s.endsWith("'"))) {
+    return s.slice(1, -1);
+  }
+  return s;
+}
+function coerceScalar(s) {
+  if (s === 'true') return true;
+  if (s === 'false') return false;
+  if (s === 'null' || s === '~') return null;
+  if (/^-?\d+$/.test(s)) return parseInt(s, 10);
+  if (/^-?\d+\.\d+$/.test(s)) return parseFloat(s);
+  return s;
+}
+
+/**
+ * Split a markdown file into { frontmatter, body }.
+ * Returns frontmatter:{} when no `--- ... ---` block at the top.
+ */
+function splitFrontmatter(raw) {
+  if (!raw.startsWith('---\n') && !raw.startsWith('---\r\n')) {
+    return { frontmatter: {}, body: raw };
+  }
+  // Find closing fence
+  const closeRe = /\n---\s*(\n|$)/;
+  const closeMatch = raw.slice(4).match(closeRe);
+  if (!closeMatch) return { frontmatter: {}, body: raw };
+  const closeIdx = 4 + closeMatch.index;
+  const yamlText = raw.slice(4, closeIdx);
+  const after = raw.slice(closeIdx + closeMatch[0].length).replace(/^\n+/, '');
+  let fm = {};
+  try { fm = parseSimpleYaml(yamlText); } catch (_) {}
+  return { frontmatter: fm, body: after };
+}
+
+function firstH1(body) {
+  const m = body.match(/^#\s+(.+?)\s*$/m);
+  return m ? m[1].trim() : null;
+}
+
+function parseMaybeDate(v) {
+  if (!v) return null;
+  if (typeof v === 'number') return v;
+  const ms = Date.parse(v);
+  return Number.isFinite(ms) ? ms : null;
+}
+
+/**
+ * Parse a single .md file in a vault.
+ * Returns { title, body, frontmatter, created, updated, hash } or null
+ * if the file should be skipped (e.g. memex: false).
+ */
+export function parseNote(filePath, vaultRoot) {
+  let raw, stat;
+  try {
+    raw = readFileSync(filePath, 'utf-8');
+    stat = statSync(filePath);
+  } catch (_) {
+    return null;
+  }
+  if (!raw.trim()) return null; // empty file — nothing to ingest
+
+  const { frontmatter, body } = splitFrontmatter(raw);
+  if (frontmatter.memex === false) return null;
+
+  const fileBase = basename(filePath, '.md');
+  const title =
+    (typeof frontmatter.title === 'string' && frontmatter.title.trim()) ||
+    firstH1(body) ||
+    fileBase;
+
+  const created = parseMaybeDate(frontmatter.created) || stat.birthtimeMs || stat.mtimeMs;
+  const updated = parseMaybeDate(frontmatter.updated || frontmatter.modified) || stat.mtimeMs;
+
+  // Stable hash for change detection (body only — frontmatter mtime changes don't trigger reindex)
+  const hash = createHash('sha1').update(body).digest('hex').slice(0, 16);
+
+  return {
+    title,
+    body,
+    frontmatter,
+    created,
+    updated,
+    hash,
+    relativePath: relative(vaultRoot, filePath),
+  };
+}
+
+/**
+ * Build a stable short id for a note within a vault.
+ * Uses sha1(vaultName + '/' + relativePath) so multi-vault setups don't collide.
+ */
+export function noteShortId(vaultRoot, relativePath) {
+  const vaultName = basename(vaultRoot);
+  const seed = `${vaultName}/${relativePath}`;
+  return createHash('sha1').update(seed).digest('hex').slice(0, 8);
+}
+
+export function vaultSlug(vaultRoot) {
+  return basename(vaultRoot)
+    .replace(/[^a-zA-Z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '')
+    .toLowerCase()
+    .slice(0, 30);
+}