@grainulation/silo 1.0.0

package/lib/server.js

@@ -0,0 +1,425 @@
+#!/usr/bin/env node
+/**
+ * silo serve -- local HTTP server for the silo knowledge browser UI
+ *
+ * Two-column pack browser with search, import wizard, and SSE live updates.
+ * Zero npm dependencies (node:http only).
+ *
+ * Usage:
+ *   silo serve [--port 9095] [--root /path/to/repo]
+ */
+
+import { createServer } from 'node:http';
+import { readFileSync, existsSync, readdirSync, writeFileSync, statSync } from 'node:fs';
+import { join, resolve, extname, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { createRequire } from 'node:module';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const require = createRequire(import.meta.url);
+
+// ── Crash handlers ──
+process.on('uncaughtException', (err) => {
+  process.stderr.write(`[${new Date().toISOString()}] FATAL: ${err.stack || err}\n`);
+  process.exit(1);
+});
+process.on('unhandledRejection', (reason) => {
+  process.stderr.write(`[${new Date().toISOString()}] WARN unhandledRejection: ${reason}\n`);
+});
+
+const PUBLIC_DIR = join(__dirname, '..', 'public');
+const PACKS_DIR = join(__dirname, '..', 'packs');
+
+// ── CLI args ──────────────────────────────────────────────────────────────────
+
+const args = process.argv.slice(2);
+function arg(name, fallback) {
+  const i = args.indexOf(`--${name}`);
+  return i !== -1 && args[i + 1] ? args[i + 1] : fallback;
+}
+
+const PORT = parseInt(arg('port', '9095'), 10);
+const ROOT = resolve(arg('root', process.cwd()));
+const CORS_ORIGIN = arg('cors', null);
+
+// ── Verbose logging ──────────────────────────────────────────────────────────
+
+const verbose = process.argv.includes('--verbose') || process.argv.includes('-v');
+function vlog(...a) {
+  if (!verbose) return;
+  const ts = new Date().toISOString();
+  process.stderr.write(`[${ts}] silo: ${a.join(' ')}\n`);
+}
+
+// ── Routes manifest ──────────────────────────────────────────────────────────
+
+const ROUTES = [
+  { method: 'GET', path: '/health', description: 'Health check endpoint' },
+  { method: 'GET', path: '/events', description: 'SSE event stream for live updates' },
+  { method: 'GET', path: '/api/packs', description: 'List all available knowledge packs' },
+  { method: 'GET', path: '/api/packs/:name', description: 'Get pack details and claims by name' },
+  { method: 'POST', path: '/api/import', description: 'Import a pack into a target claims file' },
+  { method: 'GET', path: '/api/search', description: 'Search claims by ?q, ?type, ?evidence, ?limit' },
+  { method: 'POST', path: '/api/refresh', description: 'Reload packs from disk' },
+  { method: 'GET', path: '/api/docs', description: 'This API documentation page' },
+];
+
+// ── Load existing CJS modules via createRequire ──────────────────────────────
+
+const { Store } = require('./store.js');
+const { Search } = require('./search.js');
+const { Packs } = require('./packs.js');
+const { ImportExport } = require('./import-export.js');
+
+const store = new Store();
+const search = new Search(store);
+const packs = new Packs(store);
+const io = new ImportExport(store);
+
+// ── State ─────────────────────────────────────────────────────────────────────
+
+let state = {
+  packs: [],
+  searchResults: [],
+};
+
+const sseClients = new Set();
+
+function broadcast(event) {
+  const data = `data: ${JSON.stringify(event)}\n\n`;
+  for (const res of sseClients) {
+    try { res.write(data); } catch { sseClients.delete(res); }
+  }
+}
+
+// ── Data loading ──────────────────────────────────────────────────────────────
+
+function loadPacks() {
+  return packs.list();
+}
+
+function refreshState() {
+  state.packs = loadPacks();
+  broadcast({ type: 'state', data: state });
+}
+
+// ── MIME types ────────────────────────────────────────────────────────────────
+
+const MIME = {
+  '.html': 'text/html; charset=utf-8',
+  '.css': 'text/css; charset=utf-8',
+  '.js': 'application/javascript; charset=utf-8',
+  '.json': 'application/json; charset=utf-8',
+  '.svg': 'image/svg+xml',
+  '.png': 'image/png',
+};
+
+// ── Helpers ───────────────────────────────────────────────────────────────────
+
+function jsonResponse(res, code, data) {
+  res.writeHead(code, { 'Content-Type': 'application/json; charset=utf-8' });
+  res.end(JSON.stringify(data));
+}
+
+function readBody(req) {
+  return new Promise((resolve, reject) => {
+    const chunks = [];
+    let size = 0;
+    req.on('data', c => { size += c.length; if (size > 1048576) { resolve(null); req.destroy(); return; } chunks.push(c); });
+    req.on('end', () => {
+      try { resolve(JSON.parse(Buffer.concat(chunks).toString())); }
+      catch { resolve(null); }
+    });
+    req.on('error', () => resolve(null));
+  });
+}
+
+// ── HTTP server ───────────────────────────────────────────────────────────────
+
+const server = createServer(async (req, res) => {
+  const url = new URL(req.url, `http://localhost:${PORT}`);
+
+  // CORS (only when --cors is passed)
+  if (CORS_ORIGIN) {
+    res.setHeader('Access-Control-Allow-Origin', CORS_ORIGIN);
+    res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
+    res.setHeader('Access-Control-Allow-Headers', 'Content-Type');
+  }
+
+  if (req.method === 'OPTIONS' && CORS_ORIGIN) {
+    res.writeHead(204);
+    res.end();
+    return;
+  }
+
+  vlog('request', req.method, url.pathname);
+
+  // ── API: docs ──
+  if (req.method === 'GET' && url.pathname === '/api/docs') {
+    const html = `<!DOCTYPE html><html><head><title>silo API</title>
+<style>body{font-family:system-ui;background:#0a0e1a;color:#e8ecf1;max-width:800px;margin:40px auto;padding:0 20px}
+table{width:100%;border-collapse:collapse}th,td{padding:8px 12px;border-bottom:1px solid #1e293b;text-align:left}
+th{color:#9ca3af}code{background:#1e293b;padding:2px 6px;border-radius:4px;font-size:13px}</style></head>
+<body><h1>silo API</h1><p>${ROUTES.length} endpoints</p>
+<table><tr><th>Method</th><th>Path</th><th>Description</th></tr>
+${ROUTES.map(r => '<tr><td><code>'+r.method+'</code></td><td><code>'+r.path+'</code></td><td>'+r.description+'</td></tr>').join('')}
+</table></body></html>`;
+    res.writeHead(200, { 'Content-Type': 'text/html' });
+    res.end(html);
+    return;
+  }
+
+  // ── Health check ──
+  if (req.method === 'GET' && url.pathname === '/health') {
+    jsonResponse(res, 200, { status: 'ok', uptime: process.uptime(), packs: state.packs.length });
+    return;
+  }
+
+  // ── SSE endpoint ──
+  if (req.method === 'GET' && url.pathname === '/events') {
+    res.writeHead(200, {
+      'Content-Type': 'text/event-stream',
+      'Cache-Control': 'no-cache',
+      'Connection': 'keep-alive',
+    });
+    res.write(`data: ${JSON.stringify({ type: 'state', data: state })}\n\n`);
+    const heartbeat = setInterval(() => {
+      try { res.write(': heartbeat\n\n'); } catch { clearInterval(heartbeat); }
+    }, 15000);
+    sseClients.add(res);
+    vlog('sse', `client connected (${sseClients.size} total)`);
+    req.on('close', () => { clearInterval(heartbeat); sseClients.delete(res); vlog('sse', `client disconnected (${sseClients.size} total)`); });
+    return;
+  }
+
+  // ── API: list packs ──
+  if (req.method === 'GET' && url.pathname === '/api/packs') {
+    const packList = loadPacks();
+    jsonResponse(res, 200, { packs: packList });
+    return;
+  }
+
+  // ── API: get pack details ──
+  if (req.method === 'GET' && url.pathname.startsWith('/api/packs/')) {
+    const name = decodeURIComponent(url.pathname.slice('/api/packs/'.length));
+    if (!name) { jsonResponse(res, 400, { error: 'missing pack name' }); return; }
+
+    const pack = packs.get(name);
+    if (!pack) { jsonResponse(res, 404, { error: `pack "${name}" not found` }); return; }
+
+    jsonResponse(res, 200, {
+      id: name,
+      name: pack.name,
+      description: pack.description,
+      version: pack.version,
+      claimCount: (pack.claims || []).length,
+      claims: pack.claims || [],
+    });
+    return;
+  }
+
+  // ── API: import pack ──
+  if (req.method === 'POST' && url.pathname === '/api/import') {
+    try {
+      const body = await readBody(req);
+      const { pack: packName, targetDir } = body;
+      if (!packName || !targetDir) {
+        jsonResponse(res, 400, { error: 'missing pack or targetDir' });
+        return;
+      }
+
+      const absTarget = resolve(targetDir);
+      if (!absTarget.startsWith(ROOT)) {
+        jsonResponse(res, 400, { error: 'target directory must be within the project root' });
+        return;
+      }
+
+      const targetPath = resolve(absTarget, 'claims.json');
+      // Ensure the target file exists
+      if (!existsSync(targetPath)) {
+        writeFileSync(targetPath, '[]', 'utf-8');
+      }
+
+      const result = io.pull(packName, targetPath);
+      refreshState();
+      jsonResponse(res, 200, {
+        success: true,
+        imported: result.imported,
+        skippedDuplicates: result.skippedDuplicates,
+        totalClaims: result.totalClaims,
+      });
+    } catch (err) {
+      jsonResponse(res, 500, { error: err.message });
+    }
+    return;
+  }
+
+  // ── API: search ──
+  if (req.method === 'GET' && url.pathname === '/api/search') {
+    const q = url.searchParams.get('q') || '';
+    const type = url.searchParams.get('type') || undefined;
+    const evidence = url.searchParams.get('evidence') || undefined;
+    const limit = parseInt(url.searchParams.get('limit') || '20', 10);
+
+    if (!q) {
+      jsonResponse(res, 200, { query: q, results: [] });
+      return;
+    }
+
+    // Search built-in packs too
+    const results = [];
+
+    // Search silo store
+    const storeResults = search.query(q, { type, evidence, limit });
+    results.push(...storeResults);
+
+    // Also search directly in packs/ directory
+    if (existsSync(PACKS_DIR)) {
+      for (const file of readdirSync(PACKS_DIR)) {
+        if (!file.endsWith('.json')) continue;
+        try {
+          const data = JSON.parse(readFileSync(join(PACKS_DIR, file), 'utf-8'));
+          const packName = data.name || file.replace('.json', '');
+          for (const claim of data.claims || []) {
+            if (type && claim.type !== type) continue;
+            if (evidence && claim.evidence !== evidence) continue;
+            const searchable = [
+              claim.content || claim.text || '',
+              claim.type || '',
+              claim.topic || '',
+              (claim.tags || []).join(' '),
+            ].join(' ').toLowerCase();
+
+            const tokens = q.toLowerCase().split(/\s+/).filter(t => t.length > 1);
+            let score = 0;
+            for (const token of tokens) {
+              if (searchable.includes(token)) {
+                score += 1;
+                if (searchable.includes(` ${token} `) || searchable.startsWith(`${token} `)) {
+                  score += 0.5;
+                }
+              }
+            }
+            if (score > 0) {
+              results.push({ claim, collection: `pack:${packName}`, score });
+            }
+          }
+        } catch { /* skip malformed */ }
+      }
+    }
+
+    // Deduplicate by claim ID, keep highest score
+    const seen = new Map();
+    for (const r of results) {
+      const key = r.claim.id || JSON.stringify(r.claim);
+      if (!seen.has(key) || seen.get(key).score < r.score) {
+        seen.set(key, r);
+      }
+    }
+
+    const deduped = [...seen.values()]
+      .sort((a, b) => b.score - a.score)
+      .slice(0, limit);
+
+    jsonResponse(res, 200, { query: q, results: deduped });
+    return;
+  }
+
+  // ── API: refresh ──
+  if (req.method === 'POST' && url.pathname === '/api/refresh') {
+    refreshState();
+    jsonResponse(res, 200, state);
+    return;
+  }
+
+  // ── Static files ──
+  let filePath = url.pathname === '/' ? '/index.html' : url.pathname;
+  const resolved = resolve(PUBLIC_DIR, '.' + filePath);
+  if (!resolved.startsWith(PUBLIC_DIR)) {
+    res.writeHead(403);
+    res.end('forbidden');
+    return;
+  }
+
+  if (existsSync(resolved) && statSync(resolved).isFile()) {
+    const ext = extname(resolved);
+    const mime = MIME[ext] || 'application/octet-stream';
+    try {
+      const content = readFileSync(resolved);
+      res.writeHead(200, { 'Content-Type': mime });
+      res.end(content);
+    } catch {
+      res.writeHead(500);
+      res.end('read error');
+    }
+    return;
+  }
+
+  // ── 404 ──
+  res.writeHead(404, { 'Content-Type': 'text/plain' });
+  res.end('not found');
+});
+
+// ── File watching (fingerprint-based polling) ─────────────────────────────────
+
+let lastFingerprint = '';
+function computeFingerprint() {
+  const parts = [];
+  try {
+    if (existsSync(PACKS_DIR)) {
+      for (const file of readdirSync(PACKS_DIR)) {
+        if (!file.endsWith('.json')) continue;
+        try { const s = statSync(join(PACKS_DIR, file)); parts.push(file + ':' + s.mtimeMs); } catch { /* skip */ }
+      }
+    }
+  } catch { /* skip */ }
+  return parts.join('|');
+}
+
+function startWatcher() {
+  lastFingerprint = computeFingerprint();
+  setInterval(() => {
+    const fp = computeFingerprint();
+    if (fp !== lastFingerprint) {
+      lastFingerprint = fp;
+      refreshState();
+      vlog('watcher', 'packs changed, state refreshed');
+    }
+  }, 5000);
+}
+
+// ── Graceful shutdown ─────────────────────────────────────────────────────────
+const shutdown = (signal) => {
+  console.log(`\nsilo: ${signal} received, shutting down...`);
+  for (const res of sseClients) { try { res.end(); } catch {} }
+  sseClients.clear();
+  server.close(() => process.exit(0));
+  setTimeout(() => process.exit(1), 5000);
+};
+process.on('SIGTERM', () => shutdown('SIGTERM'));
+process.on('SIGINT', () => shutdown('SIGINT'));
+
+// ── Start ─────────────────────────────────────────────────────────────────────
+
+refreshState();
+startWatcher();
+
+server.on('error', (err) => {
+  if (err.code === 'EADDRINUSE') {
+    console.error(`silo: port ${PORT} is already in use. Try --port <other>.`);
+  } else if (err.code === 'EACCES') {
+    console.error(`silo: port ${PORT} requires elevated privileges.`);
+  } else {
+    console.error(`silo: server error: ${err.message}`);
+  }
+  process.exit(1);
+});
+
+server.listen(PORT, '127.0.0.1', () => {
+  vlog('listen', `port=${PORT}`, `root=${ROOT}`);
+  console.log(`silo: serving on http://localhost:${PORT}`);
+  console.log(`  packs: ${state.packs.length} available`);
+  console.log(`  root:  ${ROOT}`);
+});
+
+export { server, PORT };

package/lib/store.js

@@ -0,0 +1,145 @@
+/**
+ * store.js — Local claim/template storage (filesystem-based)
+ *
+ * Silo stores everything as JSON files in a local directory (~/.silo by default).
+ * No database, no dependencies — just the filesystem.
+ */
+
+const fs = require('node:fs');
+const path = require('node:path');
+const crypto = require('node:crypto');
+
+const DEFAULT_SILO_DIR = path.join(require('node:os').homedir(), '.silo');
+
+class Store {
+  constructor(siloDir = DEFAULT_SILO_DIR) {
+    this.root = siloDir;
+    this.claimsDir = path.join(siloDir, 'claims');
+    this.templatesDir = path.join(siloDir, 'templates');
+    this.packsDir = path.join(siloDir, 'packs');
+    this.indexPath = path.join(siloDir, 'index.json');
+  }
+
+  /** Ensure the silo directory structure exists. */
+  init() {
+    for (const dir of [this.root, this.claimsDir, this.templatesDir, this.packsDir]) {
+      if (!fs.existsSync(dir)) {
+        fs.mkdirSync(dir, { recursive: true });
+      }
+    }
+    if (!fs.existsSync(this.indexPath)) {
+      this._writeJSON(this.indexPath, {
+        version: 1,
+        created: new Date().toISOString(),
+        collections: [],
+      });
+    }
+    return this;
+  }
+
+  /** Store a collection of claims under a name. */
+  storeClaims(name, claims, meta = {}) {
+    this.init();
+    const id = this._slugify(name);
+    const entry = {
+      id,
+      name,
+      type: 'claims',
+      claimCount: claims.length,
+      hash: this._hash(JSON.stringify(claims)),
+      storedAt: new Date().toISOString(),
+      ...meta,
+    };
+    const filePath = path.join(this.claimsDir, `${id}.json`);
+    this._writeJSON(filePath, { meta: entry, claims });
+    this._addToIndex(entry);
+    return entry;
+  }
+
+  /** Retrieve claims by collection name/id. Verifies integrity if hash exists. */
+  getClaims(nameOrId) {
+    const id = this._slugify(nameOrId);
+    const filePath = path.join(this.claimsDir, `${id}.json`);
+    if (!fs.existsSync(filePath)) return null;
+    const data = this._readJSON(filePath);
+    if (data.meta && data.meta.hash && data.claims) {
+      const actual = this._hash(JSON.stringify(data.claims));
+      // Support both old 12-char and new 64-char hashes
+      if (data.meta.hash !== actual && !actual.startsWith(data.meta.hash)) {
+        data._integrityWarning = `Hash mismatch: expected ${data.meta.hash.slice(0, 12)}..., got ${actual.slice(0, 12)}...`;
+      }
+    }
+    return data;
+  }
+
+  /** Verify integrity of all stored collections. Returns array of {id, ok, warning?}. */
+  verifyAll() {
+    this.init();
+    const results = [];
+    const index = this._readJSON(this.indexPath);
+    for (const entry of (index.collections || [])) {
+      const data = this.getClaims(entry.id);
+      if (!data) {
+        results.push({ id: entry.id, ok: false, warning: 'Collection file missing' });
+      } else {
+        results.push({ id: entry.id, ok: !data._integrityWarning, warning: data._integrityWarning || null });
+      }
+    }
+    return results;
+  }
+
+  /** List all stored collections. */
+  list() {
+    this.init();
+    const index = this._readJSON(this.indexPath);
+    return index.collections || [];
+  }
+
+  /** Remove a collection by name/id. */
+  remove(nameOrId) {
+    const id = this._slugify(nameOrId);
+    for (const dir of [this.claimsDir, this.templatesDir, this.packsDir]) {
+      const filePath = path.join(dir, `${id}.json`);
+      if (fs.existsSync(filePath)) {
+        fs.unlinkSync(filePath);
+      }
+    }
+    this._removeFromIndex(id);
+    return true;
+  }
+
+  // --- Internal helpers ---
+
+  _readJSON(filePath) {
+    return JSON.parse(fs.readFileSync(filePath, 'utf-8'));
+  }
+
+  _writeJSON(filePath, data) {
+    const tmp = filePath + '.tmp.' + process.pid;
+    fs.writeFileSync(tmp, JSON.stringify(data, null, 2) + '\n', 'utf-8');
+    fs.renameSync(tmp, filePath);
+  }
+
+  _hash(str) {
+    return crypto.createHash('sha256').update(str).digest('hex');
+  }
+
+  _slugify(str) {
+    return str.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
+  }
+
+  _addToIndex(entry) {
+    const index = this._readJSON(this.indexPath);
+    index.collections = index.collections.filter((c) => c.id !== entry.id);
+    index.collections.push(entry);
+    this._writeJSON(this.indexPath, index);
+  }
+
+  _removeFromIndex(id) {
+    const index = this._readJSON(this.indexPath);
+    index.collections = index.collections.filter((c) => c.id !== id);
+    this._writeJSON(this.indexPath, index);
+  }
+}
+
+module.exports = { Store, DEFAULT_SILO_DIR };