codebase-onboarder 0.1.0

package/server/index.js

@@ -0,0 +1,156 @@
+// Onboarder's server: where it lives on disk, and how it starts.
+//
+// The work is elsewhere. `router.js` holds the route table and the request
+// gates; each endpoint has its own module beside it (`apiScan`, `apiFile`,
+// `apiDocs`, `llmProxy`, `apiSettings`); `static.js` serves the page and the
+// shared engine; `sessions.js` remembers which directory a scan came from.
+// What is left here is the part that has to know about the filesystem it was
+// installed into, and the difference between being imported and being run.
+//
+// Where it binds comes from the settings file (`server/config.js`): local mode
+// is 127.0.0.1 and nothing else, self-hosted mode binds what it was given and
+// turns on the access-key gate in the router. A bare `createServer()` — what
+// the tests drive — never touches that file.
+
+import http from 'node:http';
+import path from 'node:path';
+import { spawn } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+
+import { createRouter } from './router.js';
+import { installExitCleanup } from './sessions.js';
+import { createLogger } from './logger.js';
+import { createMcpRunner } from './mcp/runner.js';
+import { configPath, readSettings, serverUrls } from './config.js';
+import { tunnelStatus } from './tunnel.js';
+
+const logger = createLogger();
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+const PROJECT_ROOT = path.resolve(HERE, '..');
+
+// The MCP runner is created here, not inside the router, and is put on the config
+// object the router already passes to every handler. Two reasons: exactly one
+// server can be supervised per process (a second button press must be a toggle,
+// not a second child), and a test can supply its own runner in place of this one
+// without the router knowing the difference.
+const mcp = createMcpRunner({
+  onLog: (entry) => logger.info('mcp', entry.line),
+});
+
+export const CONFIG = {
+  projectRoot: PROJECT_ROOT,
+  publicDir: path.join(PROJECT_ROOT, 'public'),
+  // Served at `/shared/`, which is what lets the browser and the server import
+  // the same analyzer. See `static.js` for why that mapping constrains the code.
+  sharedDir: path.join(PROJECT_ROOT, 'shared'),
+  mcp,
+};
+
+// The tests want a server they can put on an ephemeral port; `npm start` wants
+// one from the settings file. Same router either way.
+export function createServer(config = CONFIG) {
+    const router = createRouter(config);
+  return http.createServer((req, res) => {
+    const start = Date.now();
+    res.on('finish', () => {
+      logger.http({ method: req.method, path: req.url, status: res.statusCode, ms: Date.now() - start });
+    });
+    router(req, res);
+  });
+}
+
+// What the terminal shows once the socket is listening. A pure string builder
+// so the CLI prints exactly this too, and so a test can read it.
+export function startupBanner(settings, { configFile } = {}) {
+  const urls = serverUrls(settings);
+  const lines = ['', '  Onboarder is up.'];
+  lines.push(settings.mode === 'self-hosted'
+    ? '  Mode    self-hosted — the network can reach this; every API call needs the access key'
+    : '  Mode    local — only this machine can reach it');
+  lines.push('  Local   ' + urls.local);
+  if (urls.network) lines.push('  Network ' + urls.network);
+  if (urls.domain) lines.push('  Domain  ' + urls.domain);
+  if (settings.mode === 'self-hosted' && !settings.accessKey) {
+    lines.push('  WARNING self-hosted with no access key — every API call is refused until one is set.');
+    lines.push('          Run `onboarder setup` or `onboarder config key rotate`.');
+  }
+  const tunnels = tunnelStatus(settings);
+  for (const name of ['cloudflare', 'tailscale']) {
+    const t = tunnels[name];
+    if (!t.enabled) continue;
+    lines.push(t.installed
+      ? `  Tunnel  ${name}: ${t.command}`
+      : `  Tunnel  ${name} is enabled but its CLI is not installed — ${t.install}`);
+  }
+  if (configFile) lines.push('  Config  ' + configFile);
+  lines.push('');
+  return lines.join('\n');
+}
+
+// Ask the OS to open the app. Best-effort and detached: a missing opener on a
+// headless box is not a reason the server failed to start.
+export function openInBrowser(url) {
+  const cmd = process.platform === 'darwin' ? 'open' : process.platform === 'win32' ? 'cmd' : 'xdg-open';
+  const args = process.platform === 'win32' ? ['/c', 'start', '', url] : [url];
+  try {
+    const child = spawn(cmd, args, { detached: true, stdio: 'ignore' });
+    child.on('error', () => {});
+    child.unref();
+  } catch {
+    /* no opener — the URL is on the screen already */
+  }
+}
+
+// The real boot: read the settings, bind what they say, print the banner.
+// Both `node server/index.js` and `onboarder start` land here.
+export async function startServer({ configFile = configPath(), openBrowser, log = console.log } = {}) {
+  const settings = await readSettings(configFile);
+  // Environment wins over the file: PORT=8080 npm start has worked since
+  // before settings existed, and a flag you can see beats a file you cannot.
+  const host = process.env.HOST || settings.host;
+  const port = Number(process.env.PORT) || settings.port;
+  const live = { ...settings, host, port };
+
+  const server = createServer({
+    ...CONFIG,
+    configPath: configFile,
+    getSettings: () => readSettings(configFile),
+    boot: { host, port },
+  });
+
+  installExitCleanup();
+  // The MCP child has to go with the web server. `installExitCleanup` already
+  // handles cloned scan directories on SIGINT/SIGTERM; the child is registered
+  // on the same process events so a Ctrl-C in the terminal does not leave an
+  // orphan holding the port-less stdio pipe open.
+  for (const signal of ['SIGINT', 'SIGTERM']) {
+    process.on(signal, () => {
+      if (mcp.running) mcp.stop().catch(() => {});
+    });
+  }
+
+  await new Promise((resolve, reject) => {
+    server.once('error', reject);
+    server.listen(port, host, resolve);
+  });
+
+  log(startupBanner(live, { configFile }));
+
+  const shouldOpen = openBrowser ?? (live.autoOpen && process.stdout.isTTY && !process.env.NO_OPEN);
+  // Remote visitors still need the key; what opens locally is the loopback URL,
+  // which in local mode needs nothing and in self-hosted mode asks for the key.
+  if (shouldOpen) openInBrowser(serverUrls(live).local);
+  return { server, settings: live, host, port };
+}
+
+// `node server/index.js` listens. Importing this module — which the tests do, to
+// drive the real router — does not, and does not install signal handlers either:
+// a test runner should keep its own Ctrl-C.
+const THIS_FILE = fileURLToPath(import.meta.url);
+if (process.argv[1] && path.resolve(process.argv[1]) === THIS_FILE) {
+  startServer().catch((err) => {
+    console.error('  Onboarder did not start: ' + (err.message || err));
+    process.exitCode = 1;
+  });
+}

package/server/llmProxy.js

@@ -0,0 +1,162 @@
+// Forwards chat-completion requests to any OpenAI-compatible endpoint and
+// streams the answer straight back. The API key passes through in memory
+// only — it is never written anywhere, and never appears in logs.
+
+import { sendError, sendJSON } from './http.js';
+
+const UPSTREAM_TIMEOUT_MS = 60_000;
+const MAX_MESSAGES_BYTES = 120_000;
+
+const EXTRA_ALLOWED = new Set(['reasoning', 'temperature', 'top_p', 'presence_penalty', 'frequency_penalty']);
+
+export function detectProvider(apiKey, baseUrl) {
+  if (apiKey?.startsWith('sk-ant-')) return 'anthropic';
+  if (apiKey?.startsWith('AIza')) return 'gemini';
+  return 'openai';
+}
+
+// Extracts the provider's request id from an upstream response. Each
+// provider spells it differently; the order in this list is "what we have
+// seen", not "what is correct" — a missing id is fine and collapses to
+// null. The function is pure and exported for tests; the `headers` argument
+// is anything with a `.get(name)` method, which both `Response.headers` and
+// a hand-rolled plain object can satisfy in the test suite.
+export function extractRequestId(headers) {
+  if (!headers || typeof headers.get !== 'function') return null;
+  return headers.get('x-request-id')
+    || headers.get('request-id')
+    || headers.get('x-amzn-requestid')
+    || headers.get('x-goog-request-id')
+    || null;
+}
+
+export async function proxyChat(res, body) {
+  const { baseUrl, apiKey, model, messages } = body || {};
+  const stream = body.stream !== false;
+  const maxTokens = Math.min(Number(body.max_tokens) || 700, 4096);
+
+  if (!model || typeof model !== 'string') {
+    return sendError(res, 400, 'A model name is needed.');
+  }
+  if (!Array.isArray(messages) || !messages.length) {
+    return sendError(res, 400, 'No messages to send.');
+  }
+  if (JSON.stringify(messages).length > MAX_MESSAGES_BYTES) {
+    return sendError(res, 413, 'That prompt is too large. Try a smaller file.');
+  }
+
+  const provider = detectProvider(apiKey, baseUrl);
+
+  let url;
+  let headers = { 'content-type': 'application/json' };
+  let payload = { stream };
+  const isAzure = /\.openai\.azure\.com|\.cognitiveservices\.azure\.com/i.test(baseUrl || '');
+
+  if (provider === 'anthropic') {
+    url = 'https://api.anthropic.com/v1/messages';
+    headers['x-api-key'] = apiKey;
+    headers['anthropic-version'] = '2023-06-01';
+    
+    let system = '';
+    const anthropicMessages = [];
+    for (const m of messages) {
+      if (m.role === 'system') system += m.content + '\n';
+      else anthropicMessages.push({ role: m.role === 'user' ? 'user' : 'assistant', content: m.content });
+    }
+    payload = {
+      model,
+      max_tokens: maxTokens,
+      messages: anthropicMessages,
+      system: system.trim() || undefined,
+      stream
+    };
+  } else if (provider === 'gemini') {
+    const streamSuffix = stream ? 'streamGenerateContent?alt=sse' : 'generateContent';
+    url = `https://generativelanguage.googleapis.com/v1beta/models/${model}:${streamSuffix}&key=${apiKey}`;
+    
+    let system_instruction = null;
+    const contents = [];
+    for (const m of messages) {
+      if (m.role === 'system') {
+        system_instruction = { parts: [{ text: m.content }] };
+      } else {
+        contents.push({ role: m.role === 'assistant' ? 'model' : 'user', parts: [{ text: m.content }] });
+      }
+    }
+    payload = {
+      contents,
+      system_instruction,
+      generationConfig: { maxOutputTokens: maxTokens }
+    };
+  } else {
+    // OpenAI default
+    if (!baseUrl || !/^https?:\/\/\S+$/.test(baseUrl)) {
+      return sendError(res, 400, 'A valid base URL is needed — something like https://api.openai.com/v1.');
+    }
+    url = baseUrl.replace(/\/+$/, '') + '/chat/completions';
+    if (apiKey) {
+      if (isAzure) headers['api-key'] = apiKey;
+      else headers.authorization = 'Bearer ' + apiKey;
+    }
+    payload = { model, messages, stream };
+    if (isAzure) payload.max_completion_tokens = maxTokens;
+    else payload.max_tokens = maxTokens;
+    for (const key of Object.keys(body)) {
+      if (EXTRA_ALLOWED.has(key)) payload[key] = body[key];
+    }
+  }
+
+  let upstream;
+  try {
+    upstream = await fetch(url, {
+      method: 'POST',
+      headers,
+      body: JSON.stringify(payload),
+      signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS),
+    });
+  } catch (err) {
+    const why = err.name === 'TimeoutError' ? 'The endpoint took too long to answer.' : 'Could not reach ' + url + '.';
+    return sendError(res, 502, why);
+  }
+
+  if (!upstream.ok) {
+    const text = await upstream.text().catch(() => '');
+    let detail = text.slice(0, 400);
+    try {
+      const parsed = JSON.parse(text);
+      detail = parsed.error?.message || detail;
+    } catch {}
+    // The provider's request id, when it sends one. Most providers do —
+    // OpenAI, Anthropic, Google all set one of these on every response —
+    // and "ask the user to paste this" is far more useful than "ask the
+    // user to describe what they did". `extractRequestId` is the one place
+    // that knows the spelling; the response body only carries the field
+    // when there is one to carry, so missing ids stay out of the payload.
+    const requestId = extractRequestId(upstream.headers);
+    const body = {
+      error: `The provider answered ${upstream.status}.`,
+      detail,
+    };
+    if (requestId) body.requestId = requestId;
+    return sendJSON(res, upstream.status, body);
+  }
+
+  if (!stream) {
+    const json = await upstream.json().catch(() => null);
+    return sendJSON(res, 200, json || {});
+  }
+
+  res.writeHead(200, {
+    'content-type': 'text/event-stream',
+    'cache-control': 'no-cache',
+    connection: 'keep-alive',
+  });
+  res.on('close', () => upstream.body?.cancel().catch(() => {}));
+
+  try {
+    for await (const chunk of upstream.body) {
+      if (!res.write(chunk)) await new Promise((r) => res.once('drain', r));
+    }
+  } catch {}
+  res.end();
+}

package/server/logger.js

@@ -0,0 +1,29 @@
+const levels = { debug: 0, info: 1, warn: 2, error: 3 };
+
+export function createLogger(level = process.env.LOG_LEVEL || 'info') {
+  const minLevel = levels[level] ?? levels.info;
+  
+  function log(lvl, msg, extra = {}) {
+    if (levels[lvl] < minLevel) return;
+    const entry = {
+      ts: new Date().toISOString(),
+      level: lvl,
+      msg,
+      ...extra
+    };
+    const out = JSON.stringify(entry) + '\\n';
+    if (lvl === 'warn' || lvl === 'error') {
+      process.stderr.write(out);
+    } else {
+      process.stdout.write(out);
+    }
+  }
+
+  return {
+    debug: (msg, extra) => log('debug', msg, extra),
+    info: (msg, extra) => log('info', msg, extra),
+    warn: (msg, extra) => log('warn', msg, extra),
+    error: (msg, extra) => log('error', msg, extra),
+    http: (req) => log('info', 'HTTP Request', req)
+  };
+}

package/server/mcp/analysis.js

@@ -0,0 +1,209 @@
+// The analysis every MCP tool reads from, computed once per repository.
+//
+// The HTTP app already knows this shape: `apiScan.js` returns `scan`, `facts` and
+// `manifest`, and each view derives the rest on demand. An MCP client has no
+// session to hold a scan id in and no view to trigger those derivations, so this
+// module does the whole derivation up front, once, and hands the result to every
+// tool. The tools are then pure projections — a tool cannot disagree with another
+// about what the repository looks like, because there is only one answer cached.
+//
+// It reuses the same `shared/` analyzers the browser runs, rather than a second
+// implementation: a number in an MCP response is the same number the UI shows.
+
+import { promises as fs } from 'node:fs';
+import path from 'node:path';
+
+import { scanRepo } from '../../shared/analyzer/scan.js';
+import { computeFacts, scanIndex } from '../../shared/analyzer/graph.js';
+import { detectManifest } from '../../shared/analyzer/services.js';
+import { analyzeStack } from '../../shared/analyzer/stack.js';
+import { computeLayers, detectPatterns, couplingMatrix } from '../../shared/analyzer/patterns.js';
+import { analyzeHealth } from '../../shared/analyzer/health.js';
+import { summarizeSecurity } from '../../shared/analyzer/security.js';
+import { explainOverview, explainFile, explainFolder, scanCaveats } from '../../shared/analyzer/explainLocal.js';
+import { docFileRow, fileFactsLine, fileStaticDoc, folderStaticDoc } from '../../shared/analyzer/docs.js';
+import { baseName, dirOf } from '../../shared/analyzer/pathUtil.js';
+import { buildSearchIndex } from '../searchIndex.js';
+import { searchDocuments } from '../apiSearch.js';
+import { nodeFileSource } from '../fileSourceNode.js';
+import { gitLog, parseGitLog } from '../gitHistory.js';
+import { analyzeHistory, unavailableHistory } from '../../shared/analyzer/history.js';
+import { expandHome, resolveInside } from '../paths.js';
+
+// How many repositories stay warm. One is the common case — a client points at a
+// repo and works in it — but an agent comparing two projects should not pay for a
+// rescan when it alternates between them. Each entry holds a parsed file list, so
+// the cap is about memory, not count.
+const MAX_CACHED = 4;
+
+const cache = new Map();
+
+// Root path -> the finished analysis. Resolving the path first means `/Users/x/~
+// repo` and `/Users/x/repo` are one entry, not two that evict each other.
+export function cachedAnalysis(root) {
+  return cache.get(path.resolve(root)) || null;
+}
+
+export function cachedRoots() {
+  return [...cache.keys()];
+}
+
+export function clearAnalysisCache() {
+  cache.clear();
+}
+
+// A tool error is a message the agent can act on, not a stack trace. The protocol
+// layer turns these into `isError` results so the model sees the sentence and can
+// correct itself, rather than the call failing opaquely.
+export class ToolError extends Error {
+  constructor(message) {
+    super(message);
+    this.name = 'ToolError';
+  }
+}
+
+function buildPatterns(scan, facts, manifest) {
+  const layersInfo = computeLayers(scan, facts);
+  return {
+    layersInfo,
+    findings: detectPatterns(scan, facts, manifest, layersInfo),
+    coupling: couplingMatrix(scan, facts),
+  };
+}
+
+async function collectHistory(root, scan) {
+  try {
+    const log = await gitLog(root);
+    if (!log.ok) return unavailableHistory(log.reason);
+    return analyzeHistory(scan, parseGitLog(log.text), { totalCommits: log.totalCommits });
+  } catch {
+    return unavailableHistory('The history could not be read — the scan itself is unaffected.');
+  }
+}
+
+// The entry point. `force` is how `onboarder_rescan` differs from every other
+// tool: the same call twice returns the same answer until it is asked not to.
+export async function analyzeRepository(rootInput, { force = false } = {}) {
+  const trimmed = String(rootInput || '').trim();
+  if (!trimmed) throw new ToolError('A repository path is required.');
+
+  const root = expandHome(trimmed);
+  if (!force) {
+    const hit = cache.get(root);
+    if (hit) {
+      // Re-insert so the eviction below drops a genuinely cold repo rather than
+      // whichever one happened to be scanned first.
+      cache.delete(root);
+      cache.set(root, hit);
+      return hit;
+    }
+  }
+
+  const stat = await fs.stat(root).catch(() => null);
+  if (!stat?.isDirectory()) {
+    throw new ToolError(`No folder at ${root}. Check the path and try again.`);
+  }
+
+  const source = nodeFileSource(root);
+  const scan = await scanRepo(source);
+  const manifest = await detectManifest(source);
+  const facts = computeFacts(scan, manifest);
+
+  // Built here rather than on first search for the same reason the HTTP route
+  // builds it at scan time: a search that reads every file is a search whose cost
+  // depends on what was asked, and an agent asks repeatedly.
+  const searchIndexData = await buildSearchIndex(source, scan.files.map((f) => f.path));
+
+  const analysis = {
+    root,
+    scannedAt: Date.now(),
+    scan,
+    facts,
+    manifest,
+    stack: analyzeStack(manifest, scan.stats.languages),
+    patterns: buildPatterns(scan, facts, manifest),
+    health: analyzeHealth(scan, facts),
+    security: summarizeSecurity(scan),
+    history: await collectHistory(root, scan),
+    searchIndex: searchIndexData,
+    // The prose the overview panel shows, computed here so an agent gets the same
+    // onboarding summary a person reads rather than a pile of raw arrays.
+    overview: explainOverview(scan, facts, manifest),
+    caveats: scanCaveats(scan),
+  };
+
+  cache.set(root, analysis);
+  // Map preserves insertion order, so the first key is the least recently used.
+  while (cache.size > MAX_CACHED) cache.delete(cache.keys().next().value);
+  return analysis;
+}
+
+
+// Files anywhere under a folder, parsed or not. `scanIndex().filesIn` is an exact
+// directory lookup over the *parsed* files, which is the right thing for a diagram
+// of one directory and the wrong thing for "what is in this folder" — an agent
+// asking about `public` wants the stylesheet and the icons too.
+export function filesUnder(analysis, folder) {
+  const prefix = folder ? String(folder).replace(/\/+$/, '') + '/' : '';
+  return analysis.scan.allFiles.filter((p) => p.startsWith(prefix));
+}
+
+// The files under a folder that the parser understood, as file objects. The same
+// containment rule, so the two cannot disagree about what a folder holds.
+export function parsedFilesUnder(analysis, folder) {
+  const paths = new Set(filesUnder(analysis, folder));
+  return analysis.scan.files.filter((f) => paths.has(f.path));
+}
+
+// Resolves a repository-relative path to an absolute one, refusing anything that
+// lands outside the root. This is the same check `/api/file` makes, and it is the
+// only thing standing between an agent and the rest of the disk: an MCP client is
+// another program on this machine, not a browser tab, so the cross-origin guards
+// do not apply here. Containment is the boundary, not the scan.
+export function resolveIn(analysis, relPath) {
+  if (typeof relPath !== 'string' || !relPath.trim()) {
+    throw new ToolError('A file path is required.');
+  }
+  const abs = resolveInside(analysis.root, relPath.trim());
+  if (!abs) throw new ToolError(`${relPath} is outside ${analysis.root}.`);
+  return { abs, rel: path.relative(analysis.root, abs).split(path.sep).join('/') };
+}
+
+// The same, but also insists the parser looked at the file. The analysis tools
+// (`explain_file`, `explain_folder`) need a parsed file because every field they
+// return came from parsing it; saying "package.json is not a file in the scanned
+// repository" for a JSON manifest would be technically true and practically
+// useless, so those tools check this and `read_file` does not.
+export function fileIn(analysis, relPath) {
+  const { abs, rel } = resolveIn(analysis, relPath);
+  const file = scanIndex(analysis.scan).fileAt(rel);
+  if (!file) {
+    throw new ToolError(`${rel} is not a source file in the scan. Use onboarder_read_file to read it as text.`);
+  }
+  return { abs, rel, file };
+}
+
+export async function readFileText(analysis, relPath) {
+  const { abs, rel } = resolveIn(analysis, relPath);
+  const stat = await fs.stat(abs).catch(() => null);
+  if (!stat?.isFile()) throw new ToolError(`${rel} could not be read.`);
+  // Matches the HTTP route's ceiling. A tool that returns a 200 MB file has not
+  // answered the question, it has exhausted the context window.
+  if (stat.size > 200 * 1024) {
+    throw new ToolError(`${rel} is ${Math.round(stat.size / 1024)} KB — too large to return whole.`);
+  }
+  return { path: rel, text: await fs.readFile(abs, 'utf8') };
+}
+
+// Re-exported so tools build their payloads from the same helpers the views do,
+// rather than re-deriving "which files are in this folder" a third time.
+//
+// `importsOf`/`importers` are the plain-object maps `computeFacts` returns, not
+// methods on the scan index — the index only answers questions about files.
+export {
+  scanIndex, dirOf, baseName,
+  explainFile, explainFolder,
+  docFileRow, fileFactsLine, fileStaticDoc, folderStaticDoc,
+  searchDocuments,
+};
+