pagedex 0.0.0-stage → 0.1.0

package/src/server/search-index.js

@@ -0,0 +1,338 @@
+import { createHash } from 'node:crypto';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { DatabaseSync } from 'node:sqlite';
+import { IGNORED, VAULT_ROOT, isImagePath, isProtectedPath, resolveSafe } from './vault.js';
+
+const BINARY_EXT_RE = /\.(png|jpe?g|gif|webp|svg|ico|epub|pdf|zip|tar|gz|tgz|7z|rar|docx?|xlsx?|pptx?|odt|ods|odp|mp3|wav|m4a|mp4|mov|webm|mkv|avi|dmg|exe|bin|whl|sqlite|db)$/i;
+const MAX_INDEX_BYTES = 2 * 1024 * 1024;
+
+let db = null;
+let syncedOnce = false;
+
+function isTextIndexable(relPath, size) {
+  if (isImagePath(relPath) || BINARY_EXT_RE.test(relPath)) return false;
+  return size <= MAX_INDEX_BYTES;
+}
+
+function resolveDbPath() {
+  if (process.env.PAGEDEX_INDEX_PATH) return path.resolve(process.env.PAGEDEX_INDEX_PATH);
+  const cacheDir = process.env.PAGEDEX_INDEX_DIR || path.join(os.homedir(), '.cache', 'pagedex');
+  fs.mkdirSync(cacheDir, { recursive: true });
+  const hash = createHash('sha256').update(VAULT_ROOT).digest('hex').slice(0, 16);
+  return path.join(cacheDir, `${hash}.db`);
+}
+
+function getDb() {
+  if (db) return db;
+  const dbPath = resolveDbPath();
+  db = new DatabaseSync(dbPath);
+  db.exec(`
+    PRAGMA journal_mode = WAL;
+    PRAGMA synchronous = NORMAL;
+    CREATE TABLE IF NOT EXISTS files (
+      path TEXT PRIMARY KEY,
+      name TEXT NOT NULL,
+      mtime_ms INTEGER NOT NULL,
+      size INTEGER NOT NULL,
+      is_text INTEGER NOT NULL
+    );
+    CREATE INDEX IF NOT EXISTS idx_files_mtime ON files(mtime_ms DESC, path ASC);
+    CREATE INDEX IF NOT EXISTS idx_files_name ON files(name COLLATE NOCASE ASC, path ASC);
+    CREATE VIRTUAL TABLE IF NOT EXISTS docs_fts USING fts5(
+      path,
+      content,
+      tokenize='trigram'
+    );
+  `);
+  return db;
+}
+
+function walkDiskFiles(dir = VAULT_ROOT, relBase = '', out = []) {
+  let entries;
+  try {
+    entries = fs.readdirSync(dir, { withFileTypes: true });
+  } catch {
+    return out;
+  }
+  for (const entry of entries) {
+    if (IGNORED.has(entry.name)) continue;
+    const rel = relBase ? `${relBase}/${entry.name}` : entry.name;
+    const full = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      walkDiskFiles(full, rel, out);
+    } else if (entry.isFile()) {
+      try {
+        const st = fs.statSync(full);
+        out.push({
+          path: rel,
+          name: entry.name,
+          mtimeMs: Math.round(st.mtimeMs),
+          size: st.size,
+          full
+        });
+      } catch {
+        // Ignore unreadable transient files
+      }
+    }
+  }
+  return out;
+}
+
+export function syncIndex() {
+  const conn = getDb();
+  const diskFiles = walkDiskFiles();
+  const diskMap = new Map(diskFiles.map((f) => [f.path, f]));
+
+  const existingRows = conn.prepare('SELECT path, mtime_ms, size FROM files').all();
+  const existingMap = new Map(existingRows.map((r) => [r.path, r]));
+
+  const delFile = conn.prepare('DELETE FROM files WHERE path = ?');
+  const delFts = conn.prepare('DELETE FROM docs_fts WHERE path = ?');
+  const upsertFile = conn.prepare(`
+    INSERT INTO files (path, name, mtime_ms, size, is_text)
+    VALUES (?, ?, ?, ?, ?)
+    ON CONFLICT(path) DO UPDATE SET
+      name = excluded.name,
+      mtime_ms = excluded.mtime_ms,
+      size = excluded.size,
+      is_text = excluded.is_text
+  `);
+  const insertFts = conn.prepare('INSERT INTO docs_fts (path, content) VALUES (?, ?)');
+
+  conn.exec('BEGIN');
+  try {
+    for (const row of existingRows) {
+      if (!diskMap.has(row.path)) {
+        delFile.run(row.path);
+        delFts.run(row.path);
+      }
+    }
+    for (const f of diskFiles) {
+      const prev = existingMap.get(f.path);
+      if (prev && prev.mtime_ms === f.mtimeMs && prev.size === f.size) continue;
+      const textable = isTextIndexable(f.path, f.size);
+      let content = '';
+      if (textable) {
+        try {
+          content = fs.readFileSync(f.full, 'utf8');
+        } catch {
+          content = '';
+        }
+      }
+      upsertFile.run(f.path, f.name, f.mtimeMs, f.size, textable ? 1 : 0);
+      delFts.run(f.path);
+      if (textable) {
+        insertFts.run(f.path, content);
+      }
+    }
+    conn.exec('COMMIT');
+  } catch (err) {
+    conn.exec('ROLLBACK');
+    throw err;
+  }
+  syncedOnce = true;
+  return { total: diskFiles.length };
+}
+
+function ensureSynced() {
+  if (!syncedOnce) syncIndex();
+}
+
+export function upsertFileInIndex(relPath) {
+  if (!relPath || isProtectedPath(relPath)) return;
+  const clean = relPath.replace(/^\/+/, '');
+  const full = resolveSafe(clean);
+  if (!full) return;
+
+  const conn = getDb();
+  if (!fs.existsSync(full)) {
+    removeFileFromIndex(clean);
+    return;
+  }
+  let st;
+  try {
+    st = fs.statSync(full);
+  } catch {
+    return;
+  }
+  if (st.isDirectory()) {
+    syncIndex();
+    return;
+  }
+  if (!st.isFile()) return;
+
+  const mtimeMs = Math.round(st.mtimeMs);
+  const textable = isTextIndexable(clean, st.size);
+  let content = '';
+  if (textable) {
+    try {
+      content = fs.readFileSync(full, 'utf8');
+    } catch {
+      content = '';
+    }
+  }
+
+  conn.exec('BEGIN');
+  try {
+    conn
+      .prepare(
+        `INSERT INTO files (path, name, mtime_ms, size, is_text)
+         VALUES (?, ?, ?, ?, ?)
+         ON CONFLICT(path) DO UPDATE SET
+           name = excluded.name,
+           mtime_ms = excluded.mtime_ms,
+           size = excluded.size,
+           is_text = excluded.is_text`
+      )
+      .run(clean, path.basename(clean), mtimeMs, st.size, textable ? 1 : 0);
+    conn.prepare('DELETE FROM docs_fts WHERE path = ?').run(clean);
+    if (textable) {
+      conn.prepare('INSERT INTO docs_fts (path, content) VALUES (?, ?)').run(clean, content);
+    }
+    conn.exec('COMMIT');
+  } catch (err) {
+    conn.exec('ROLLBACK');
+    throw err;
+  }
+}
+
+export function removeFileFromIndex(relPath) {
+  if (!relPath) return;
+  const clean = relPath.replace(/^\/+/, '');
+  const conn = getDb();
+  conn.prepare('DELETE FROM files WHERE path = ?').run(clean);
+  conn.prepare('DELETE FROM docs_fts WHERE path = ?').run(clean);
+}
+
+export function queryFiles({ sort = 'mtime', offset = 0, limit = 100, q = '' } = {}) {
+  ensureSynced();
+  const conn = getDb();
+  const safeOffset = Math.max(0, Number(offset) || 0);
+  const safeLimit = Math.min(1000, Math.max(1, Number(limit) || 100));
+  const orderBy = sort === 'name' ? 'name COLLATE NOCASE ASC, path ASC' : 'mtime_ms DESC, path ASC';
+  const cleanQ = String(q || '').trim().toLowerCase();
+
+  let total;
+  let rows;
+  if (cleanQ) {
+    const pattern = `%${cleanQ.replace(/[%_\\]/g, '\\$&')}%`;
+    total = conn
+      .prepare("SELECT COUNT(*) AS cnt FROM files WHERE lower(path) LIKE ? ESCAPE '\\'")
+      .get(pattern).cnt;
+    rows = conn
+      .prepare(
+        `SELECT path, mtime_ms AS mtimeMs FROM files
+         WHERE lower(path) LIKE ? ESCAPE '\\'
+         ORDER BY ${orderBy}
+         LIMIT ? OFFSET ?`
+      )
+      .all(pattern, safeLimit, safeOffset);
+  } else {
+    total = conn.prepare('SELECT COUNT(*) AS cnt FROM files').get().cnt;
+    rows = conn
+      .prepare(
+        `SELECT path, mtime_ms AS mtimeMs FROM files
+         ORDER BY ${orderBy}
+         LIMIT ? OFFSET ?`
+      )
+      .all(safeLimit, safeOffset);
+  }
+
+  return {
+    items: rows,
+    total,
+    offset: safeOffset,
+    limit: safeLimit,
+    hasMore: safeOffset + rows.length < total
+  };
+}
+
+export function searchFts(query, limit = 50) {
+  ensureSynced();
+  const q = String(query || '').trim();
+  if (!q) return [];
+  const conn = getDb();
+  const safeLimit = Math.min(200, Math.max(1, Number(limit) || 50));
+
+  if (q.length >= 3) {
+    // Trigram FTS5 expects double-quoted literal; escape internal quotes
+    const ftsLiteral = `"${q.replace(/"/g, '""')}"`;
+    const rows = conn
+      .prepare(
+        `SELECT docs_fts.path AS path,
+                COALESCE(files.mtime_ms, 0) AS mtimeMs,
+                snippet(docs_fts, 1, '', '', '...', 16) AS snippet
+         FROM docs_fts
+         LEFT JOIN files ON files.path = docs_fts.path
+         WHERE docs_fts MATCH ?
+         ORDER BY files.mtime_ms DESC, docs_fts.path ASC
+         LIMIT ?`
+      )
+      .all(ftsLiteral, safeLimit);
+
+    // Also include non-text files whose path matches q (since non-text files aren't in docs_fts)
+    if (rows.length < safeLimit) {
+      const seen = new Set(rows.map((r) => r.path));
+      const pattern = `%${q.toLowerCase().replace(/[%_\\]/g, '\\$&')}%`;
+      const pathOnly = conn
+        .prepare(
+          `SELECT path, mtime_ms AS mtimeMs FROM files
+           WHERE is_text = 0 AND lower(path) LIKE ? ESCAPE '\\'
+           ORDER BY mtime_ms DESC
+           LIMIT ?`
+        )
+        .all(pattern, safeLimit - rows.length);
+      for (const r of pathOnly) {
+        if (!seen.has(r.path)) rows.push({ path: r.path, mtimeMs: r.mtimeMs, snippet: '' });
+      }
+    }
+    return rows;
+  }
+
+  // Short queries (1-2 chars): trigram requires >=3 chars, so match path + scan recent text files
+  const lowerQ = q.toLowerCase();
+  const pattern = `%${lowerQ.replace(/[%_\\]/g, '\\$&')}%`;
+  const results = [];
+  const seen = new Set();
+
+  const pathRows = conn
+    .prepare(
+      `SELECT path, mtime_ms AS mtimeMs FROM files
+       WHERE lower(path) LIKE ? ESCAPE '\\'
+       ORDER BY mtime_ms DESC
+       LIMIT ?`
+    )
+    .all(pattern, safeLimit);
+
+  for (const r of pathRows) {
+    seen.add(r.path);
+    results.push({ path: r.path, mtimeMs: r.mtimeMs, snippet: '' });
+  }
+
+  if (results.length < safeLimit) {
+    const recentText = conn
+      .prepare('SELECT path, mtime_ms AS mtimeMs FROM files WHERE is_text = 1 ORDER BY mtime_ms DESC LIMIT 500')
+      .all();
+    for (const r of recentText) {
+      if (seen.has(r.path)) continue;
+      const full = path.join(VAULT_ROOT, r.path);
+      try {
+        const text = fs.readFileSync(full, 'utf8');
+        const idx = text.toLowerCase().indexOf(lowerQ);
+        if (idx !== -1) {
+          const snippet = text
+            .slice(Math.max(0, idx - 60), Math.min(text.length, idx + q.length + 100))
+            .replace(/\s+/g, ' ')
+            .trim();
+          results.push({ path: r.path, mtimeMs: r.mtimeMs, snippet });
+          if (results.length >= safeLimit) break;
+        }
+      } catch {
+        // Ignore transient read error
+      }
+    }
+  }
+  return results;
+}

package/src/server/vault-cache.js

@@ -0,0 +1,269 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { IGNORED, VAULT_ROOT, isImagePath, isProtectedPath, resolveSafe } from './paths.js';
+
+let vaultCache = null;
+
+export function invalidateVaultCache() {
+  vaultCache = null;
+}
+
+function sortTreeNodes(nodes) {
+  nodes.sort((a, b) => {
+    if (a.type !== b.type) return a.type === 'dir' ? -1 : 1;
+    return a.name.localeCompare(b.name);
+  });
+}
+
+function insertTreeFile(nodes, relPath, mtimeMs = 0) {
+  const parts = relPath.split('/');
+  let list = nodes;
+  for (let i = 0; i < parts.length - 1; i++) {
+    const dirName = parts[i];
+    const dirPath = parts.slice(0, i + 1).join('/');
+    let dirNode = list.find((n) => n.type === 'dir' && n.name === dirName);
+    if (!dirNode) {
+      dirNode = { type: 'dir', name: dirName, path: dirPath, mtimeMs, children: [] };
+      list.push(dirNode);
+      sortTreeNodes(list);
+    } else if (mtimeMs > (dirNode.mtimeMs || 0)) {
+      dirNode.mtimeMs = mtimeMs;
+    }
+    list = dirNode.children;
+  }
+  const fileName = parts[parts.length - 1];
+  const existing = list.find((n) => n.type === 'file' && n.path === relPath);
+  if (!existing) {
+    list.push({ type: 'file', name: fileName, path: relPath, mtimeMs });
+    sortTreeNodes(list);
+  } else {
+    existing.mtimeMs = mtimeMs;
+  }
+}
+
+function removeTreeFile(nodes, relPath) {
+  const parts = relPath.split('/');
+  if (parts.length === 1) {
+    const idx = nodes.findIndex((n) => n.type === 'file' && n.name === parts[0]);
+    if (idx !== -1) nodes.splice(idx, 1);
+    return;
+  }
+  const dirNode = nodes.find((n) => n.type === 'dir' && n.name === parts[0]);
+  if (!dirNode) return;
+  removeTreeFile(dirNode.children, parts.slice(1).join('/'));
+  dirNode.mtimeMs = dirNode.children.reduce((max, n) => Math.max(max, n.mtimeMs || 0), 0);
+  if (dirNode.children.length === 0 && !fs.existsSync(path.join(VAULT_ROOT, dirNode.path))) {
+    const idx = nodes.indexOf(dirNode);
+    if (idx !== -1) nodes.splice(idx, 1);
+  }
+}
+
+function createWikiResolver(flatFiles) {
+  const mdFiles = flatFiles.filter((p) => p.endsWith('.md'));
+  const byName = new Map();
+  const byNoExt = new Map();
+
+  for (const p of mdFiles) {
+    const noExt = p.slice(0, -3);
+    const base = path.basename(noExt).toLowerCase();
+    byNoExt.set(noExt.toLowerCase(), p);
+    if (!byName.has(base)) byName.set(base, p);
+  }
+
+  return function resolveWiki(rawTarget) {
+    const clean = rawTarget.split('|')[0].split('#')[0].trim().replace(/\.md$/i, '').replace(/^\/+/, '');
+    if (!clean) return null;
+    const lower = clean.toLowerCase();
+    if (byNoExt.has(lower)) return byNoExt.get(lower);
+    const base = path.basename(lower);
+    return byName.get(base) || null;
+  };
+}
+
+function extractWikiTargets(text, resolveWiki) {
+  const stripped = text.replace(/```[\s\S]*?```/g, '');
+  const wikiRe = /\[\[([^\]]+)\]\]/g;
+  const targets = new Set();
+  let match;
+  while ((match = wikiRe.exec(stripped)) !== null) {
+    const target = resolveWiki(match[1]);
+    if (target) targets.add(target);
+  }
+  return targets;
+}
+
+export function removeVaultCacheFile(relPath) {
+  if (!vaultCache || !relPath) return;
+  const clean = relPath.replace(/^\/+/, '');
+  const fIdx = vaultCache.flatFiles.indexOf(clean);
+  if (fIdx !== -1) vaultCache.flatFiles.splice(fIdx, 1);
+  vaultCache.fileStats = vaultCache.fileStats.filter((f) => f.path !== clean);
+  vaultCache.sortedStats = vaultCache.sortedStats.filter((f) => f.path !== clean);
+  removeTreeFile(vaultCache.tree, clean);
+  delete vaultCache.backlinks[clean];
+  for (const list of Object.values(vaultCache.backlinks)) {
+    const i = list.indexOf(clean);
+    if (i !== -1) list.splice(i, 1);
+  }
+}
+
+export function updateVaultCacheFile(relPath) {
+  if (!vaultCache || !relPath) return;
+  const clean = relPath.replace(/^\/+/, '');
+  if (isProtectedPath(clean)) return;
+  const full = resolveSafe(clean);
+  if (!full || !fs.existsSync(full)) {
+    removeVaultCacheFile(clean);
+    return;
+  }
+  let st;
+  try {
+    st = fs.statSync(full);
+  } catch {
+    return;
+  }
+  if (st.isDirectory()) {
+    invalidateVaultCache();
+    return;
+  }
+  if (!st.isFile()) return;
+
+  const mtimeMs = Math.round(st.mtimeMs);
+  if (!vaultCache.flatFiles.includes(clean)) {
+    vaultCache.flatFiles.push(clean);
+  }
+  insertTreeFile(vaultCache.tree, clean, mtimeMs);
+
+  const statEntry = vaultCache.fileStats.find((f) => f.path === clean);
+  if (statEntry) statEntry.mtimeMs = mtimeMs;
+  else vaultCache.fileStats.push({ path: clean, mtimeMs });
+
+  vaultCache.sortedStats = vaultCache.sortedStats.filter((f) => f.path !== clean);
+  vaultCache.sortedStats.unshift({ path: clean, mtimeMs });
+  vaultCache.sortedStats.sort((a, b) => b.mtimeMs - a.mtimeMs);
+
+  if (clean.endsWith('.md')) {
+    if (!vaultCache.backlinks[clean]) vaultCache.backlinks[clean] = [];
+    for (const list of Object.values(vaultCache.backlinks)) {
+      const i = list.indexOf(clean);
+      if (i !== -1) list.splice(i, 1);
+    }
+    try {
+      const text = fs.readFileSync(full, 'utf8');
+      const resolveWiki = createWikiResolver(vaultCache.flatFiles);
+      for (const target of extractWikiTargets(text, resolveWiki)) {
+        if (target !== clean) {
+          if (!vaultCache.backlinks[target]) vaultCache.backlinks[target] = [];
+          vaultCache.backlinks[target].push(clean);
+        }
+      }
+    } catch {
+      // Ignore transient read error
+    }
+  }
+}
+
+export function sortTree(nodes, sort = 'name') {
+  if (sort !== 'mtime') return nodes;
+  return [...nodes]
+    .map((n) => (n.type === 'dir' ? { ...n, children: sortTree(n.children, sort) } : n))
+    .sort((a, b) => (b.mtimeMs || 0) - (a.mtimeMs || 0) || a.name.localeCompare(b.name));
+}
+
+export function scanVault(dir = VAULT_ROOT, relBase = '') {
+  const entries = fs.readdirSync(dir, { withFileTypes: true });
+  const dirs = [];
+  const files = [];
+  const flatFiles = [];
+  const fileStats = [];
+
+  for (const entry of entries) {
+    if (IGNORED.has(entry.name)) continue;
+    const rel = relBase ? `${relBase}/${entry.name}` : entry.name;
+    const full = path.join(dir, entry.name);
+
+    if (entry.isDirectory()) {
+      const sub = scanVault(full, rel);
+      const dirMtime = sub.tree.reduce((max, n) => Math.max(max, n.mtimeMs || 0), 0);
+      dirs.push({ type: 'dir', name: entry.name, path: rel, mtimeMs: dirMtime, children: sub.tree });
+      flatFiles.push(...sub.flatFiles);
+      fileStats.push(...sub.fileStats);
+    } else if (entry.isFile()) {
+      let mtimeMs = 0;
+      try {
+        mtimeMs = Math.round(fs.statSync(full).mtimeMs);
+      } catch {
+        mtimeMs = 0;
+      }
+      files.push({ type: 'file', name: entry.name, path: rel, mtimeMs });
+      flatFiles.push(rel);
+      fileStats.push({ path: rel, mtimeMs });
+    }
+  }
+
+  dirs.sort((a, b) => a.name.localeCompare(b.name));
+  files.sort((a, b) => a.name.localeCompare(b.name));
+  return { tree: [...dirs, ...files], flatFiles, fileStats };
+}
+
+export function buildLinkIndex(flatFiles) {
+  const mdFiles = flatFiles.filter((p) => p.endsWith('.md'));
+  const resolveWiki = createWikiResolver(flatFiles);
+  const backlinks = Object.fromEntries(mdFiles.map((p) => [p, []]));
+
+  for (const src of mdFiles) {
+    const full = path.join(VAULT_ROOT, src);
+    let text = '';
+    try {
+      text = fs.readFileSync(full, 'utf8');
+    } catch {
+      continue;
+    }
+    for (const target of extractWikiTargets(text, resolveWiki)) {
+      if (target !== src) {
+        backlinks[target]?.push(src);
+      }
+    }
+  }
+
+  return { backlinks, mdFiles };
+}
+
+export function getCachedVault() {
+  if (vaultCache) return vaultCache;
+  const { tree, flatFiles, fileStats } = scanVault();
+  const { backlinks } = buildLinkIndex(flatFiles);
+  const sortedStats = [...fileStats].sort((a, b) => b.mtimeMs - a.mtimeMs);
+  vaultCache = { tree, flatFiles, fileStats, sortedStats, backlinks };
+  return vaultCache;
+}
+
+export function searchVault(query, limit = 50) {
+  const q = String(query || '').trim().toLowerCase();
+  if (!q) return [];
+  const { fileStats } = scanVault();
+  const sorted = [...fileStats].sort((a, b) => b.mtimeMs - a.mtimeMs);
+  const results = [];
+
+  for (const { path: rel, mtimeMs } of sorted) {
+    if (isImagePath(rel)) continue;
+    const full = path.join(VAULT_ROOT, rel);
+    let text = '';
+    try {
+      text = fs.readFileSync(full, 'utf8');
+    } catch {
+      continue;
+    }
+    const pathMatch = rel.toLowerCase().includes(q);
+    const idx = text.toLowerCase().indexOf(q);
+    if (pathMatch || idx !== -1) {
+      const snippet =
+        idx !== -1
+          ? text.slice(Math.max(0, idx - 60), Math.min(text.length, idx + q.length + 100)).replace(/\s+/g, ' ').trim()
+          : '';
+      results.push({ path: rel, mtimeMs, snippet });
+      if (results.length >= limit) break;
+    }
+  }
+  return results;
+}

package/src/server/vault.js

@@ -0,0 +1,4 @@
+export * from './paths.js';
+export * from './vault-cache.js';
+export * from './fs-ops.js';
+export * from './git.js';