codebase-onboarder 0.1.0

package/server/fileSourceNode.js

@@ -0,0 +1,44 @@
+// FileSource over node:fs. Paths crossing the boundary are repo-relative
+// POSIX style; this adapter is the only place that knows about the real
+// filesystem layout.
+
+import { promises as fs } from 'node:fs';
+import path from 'node:path';
+
+import { resolveInside } from './paths.js';
+
+export function nodeFileSource(rootAbs) {
+  const toAbs = (rel) => {
+    // `list('')` is how the scanner asks for the top of the repo. Everywhere
+    // else an empty path is a caller mistake, which is why resolveInside
+    // refuses it rather than quietly meaning "the root".
+    if (rel === '' || rel == null) return rootAbs;
+    // The scanner only ever produces paths from directory listings, so this
+    // should never fire — but it's the same check the HTTP routes use, and a
+    // free one.
+    const abs = resolveInside(rootAbs, String(rel));
+    if (!abs) throw new Error('Path escapes the repository root');
+    return abs;
+  };
+
+  return {
+    kind: 'node',
+    root: rootAbs,
+    name: path.basename(rootAbs),
+
+    async list(dir) {
+      const entries = await fs.readdir(toAbs(dir), { withFileTypes: true });
+      const out = [];
+      for (const e of entries) {
+        if (e.isSymbolicLink()) continue; // no loops, no surprises
+        const rel = dir ? dir + '/' + e.name : e.name;
+        out.push({ name: e.name, path: rel, type: e.isDirectory() ? 'dir' : 'file' });
+      }
+      return out;
+    },
+
+    async read(rel) {
+      return fs.readFile(toAbs(rel), 'utf8');
+    },
+  };
+}

package/server/gitClone.js

@@ -0,0 +1,95 @@
+// Shallow-cloning for the git-URL flow. The URL is validated, then passed to
+// git as an argument array — never through a shell — so nothing in it gets
+// executed. Clones land in the OS temp dir and are removed on request.
+
+import { spawn } from 'node:child_process';
+import { promises as fs, rmSync } from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import crypto from 'node:crypto';
+
+const GIT_URL_RE = /^(https?:\/\/[^\s]+|git@[^\s:]+:[^\s]+|ssh:\/\/[^\s]+)$/;
+const CLONE_TIMEOUT_MS = 180_000;
+
+// Where clones go, and the only shape of path the removals below will touch.
+// One definition: a delete that walks a directory tree should not be guarded by
+// a condition restated at each call site.
+const CLONE_PREFIX = path.join(os.tmpdir(), 'onboarder-');
+
+export function isCloneDir(dir) {
+  return typeof dir === 'string' && dir.startsWith(CLONE_PREFIX) && dir !== CLONE_PREFIX;
+}
+
+export function assertGitUrl(url) {
+  if (typeof url !== 'string' || !GIT_URL_RE.test(url.trim())) {
+    throw new Error('That doesn’t look like a git URL. https://, ssh:// or git@ forms work.');
+  }
+  return url.trim();
+}
+
+// The name people recognize, from any URL shape we accept:
+// https://github.com/org/repo.git, ssh://host/org/repo, git@host:org/repo
+export function repoNameFromUrl(url) {
+  const clean = String(url).trim().replace(/\.git$/, '').replace(/\/+$/, '');
+  const tail = !clean.includes('://') && clean.includes(':') ? clean.split(':').pop() : clean;
+  return tail.split('/').pop() || 'repo';
+}
+
+export async function cloneRepo(url) {
+  const id = crypto.randomBytes(6).toString('hex');
+  const dir = CLONE_PREFIX + id;
+  try {
+    // Blobless rather than shallow: `--depth 1` threw the history away, and
+    // history is half of what this tool has to say (churn × complexity).
+    // `--filter=blob:none` fetches every commit and tree but no file versions —
+    // full history at a fraction of a full clone's weight; blobs arrive on
+    // demand if anything ever checks one out. `--single-branch` keeps it to the
+    // default branch: the newcomer cares about main, not every fork line.
+    await run('git', ['clone', '--filter=blob:none', '--single-branch', '--quiet', url, dir]);
+  } catch (err) {
+    await fs.rm(dir, { recursive: true, force: true }).catch(() => {});
+    throw err;
+  }
+  return { id, dir };
+}
+
+export async function removeClone(dir) {
+  if (!isCloneDir(dir)) return;
+  await fs.rm(dir, { recursive: true, force: true }).catch(() => {});
+}
+
+// The version an exit handler can actually finish. A process on its way out will
+// not come back to resolve a promise, so the sweep on shutdown has to be
+// synchronous or it does not happen at all.
+export function removeCloneSync(dir) {
+  if (!isCloneDir(dir)) return;
+  try {
+    rmSync(dir, { recursive: true, force: true });
+  } catch {
+    /* going down anyway; a leftover temp dir is not worth a crash on exit */
+  }
+}
+
+function run(cmd, args) {
+  return new Promise((resolve, reject) => {
+    const child = spawn(cmd, args, { stdio: ['ignore', 'ignore', 'pipe'] });
+    let stderr = '';
+    child.stderr.on('data', (chunk) => {
+      stderr += chunk;
+      if (stderr.length > 4000) stderr = stderr.slice(-4000);
+    });
+    const timer = setTimeout(() => {
+      child.kill('SIGKILL');
+      reject(new Error('git clone took too long and was stopped.'));
+    }, CLONE_TIMEOUT_MS);
+    child.on('error', (err) => {
+      clearTimeout(timer);
+      reject(new Error('Could not run git: ' + err.message));
+    });
+    child.on('close', (code) => {
+      clearTimeout(timer);
+      if (code === 0) resolve();
+      else reject(new Error('git clone failed. ' + (stderr.trim().split('\n').pop() || '')));
+    });
+  });
+}

package/server/gitDiff.js

@@ -0,0 +1,184 @@
+// Git diff & refs provider.
+// Executes git subcommands safely via child_process and parses patch hunks.
+
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+
+const execFileAsync = promisify(execFile);
+const MAX_DIFF_BYTES = 5 * 1024 * 1024; // 5 MB cap
+
+export async function getGitRefs(root) {
+  try {
+    const [branchRes, tagRes, headRes, logRes] = await Promise.all([
+      execFileAsync('git', ['branch', '-a', '--format=%(refname:short)'], { cwd: root }).catch(() => ({ stdout: '' })),
+      execFileAsync('git', ['tag', '--sort=-creatordate'], { cwd: root }).catch(() => ({ stdout: '' })),
+      execFileAsync('git', ['rev-parse', '--abbrev-ref', 'HEAD'], { cwd: root }).catch(() => ({ stdout: 'main\n' })),
+      execFileAsync('git', ['log', '-n', '40', '--format=%H|%h|%s|%an|%aI'], { cwd: root }).catch(() => ({ stdout: '' })),
+    ]);
+
+    const branches = branchRes.stdout.split('\n').map((b) => b.trim()).filter(Boolean);
+    const tags = tagRes.stdout.split('\n').map((t) => t.trim()).filter(Boolean);
+    const current = headRes.stdout.trim() || 'HEAD';
+
+    const commits = logRes.stdout.split('\n').filter(Boolean).map((line) => {
+      const [hash, short, message, author, date] = line.split('|');
+      return { hash, short, message, author, date };
+    });
+
+    return {
+      available: true,
+      branches: [...new Set(branches)],
+      tags,
+      current,
+      commits,
+    };
+  } catch (err) {
+    return { available: false, reason: err.message };
+  }
+}
+
+export async function getGitDiff(root, options = {}) {
+  const { base, head, file } = options;
+  const args = ['diff', '--no-color', '-U3'];
+
+  if (base && head) {
+    args.push(`${base}...${head}`);
+  } else if (base) {
+    args.push(base);
+  } else {
+    args.push('HEAD');
+  }
+
+  if (file) {
+    args.push('--', file);
+  }
+
+  try {
+    const { stdout } = await execFileAsync('git', args, {
+      cwd: root,
+      maxBuffer: MAX_DIFF_BYTES,
+    });
+    return parseUnifiedDiff(stdout);
+  } catch (err) {
+    // If git diff exited with empty or specific error, fallback to HEAD~1..HEAD or working tree
+    try {
+      const fallback = await execFileAsync('git', ['diff', '--no-color', '-U3'], {
+        cwd: root,
+        maxBuffer: MAX_DIFF_BYTES,
+      });
+      return parseUnifiedDiff(fallback.stdout);
+    } catch (e) {
+      return { files: [], stats: { filesChanged: 0, additions: 0, deletions: 0 }, raw: '' };
+    }
+  }
+}
+
+export function parseUnifiedDiff(diffText) {
+  if (!diffText || typeof diffText !== 'string') {
+    return { files: [], stats: { filesChanged: 0, additions: 0, deletions: 0 }, raw: '' };
+  }
+
+  const files = [];
+  const lines = diffText.split('\n');
+  let currentFile = null;
+  let currentHunk = null;
+  let totalAdditions = 0;
+  let totalDeletions = 0;
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+
+    // File header: diff --git a/path b/path
+    if (line.startsWith('diff --git ')) {
+      const m = line.match(/^diff --git a\/(.+?) b\/(.+)$/);
+      const oldPath = m ? m[1] : '';
+      const newPath = m ? m[2] : '';
+      currentFile = {
+        oldPath,
+        newPath,
+        status: 'modified',
+        additions: 0,
+        deletions: 0,
+        hunks: [],
+      };
+      files.push(currentFile);
+      currentHunk = null;
+      continue;
+    }
+
+    if (!currentFile) continue;
+
+    if (line.startsWith('new file mode ')) {
+      currentFile.status = 'added';
+      continue;
+    }
+    if (line.startsWith('deleted file mode ')) {
+      currentFile.status = 'deleted';
+      continue;
+    }
+    if (line.startsWith('rename from ')) {
+      currentFile.status = 'renamed';
+      currentFile.oldPath = line.slice(12).trim();
+      continue;
+    }
+    if (line.startsWith('rename to ')) {
+      currentFile.newPath = line.slice(10).trim();
+      continue;
+    }
+
+    // Hunk header: @@ -1,5 +1,6 @@ optional context
+    if (line.startsWith('@@ ')) {
+      const match = line.match(/^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@(.*)$/);
+      if (match) {
+        const oldStart = parseInt(match[1], 10);
+        const oldLines = match[2] !== undefined ? parseInt(match[2], 10) : 1;
+        const newStart = parseInt(match[3], 10);
+        const newLines = match[4] !== undefined ? parseInt(match[4], 10) : 1;
+        currentHunk = {
+          header: line,
+          oldStart,
+          oldLines,
+          newStart,
+          newLines,
+          heading: match[5]?.trim() || '',
+          lines: [],
+        };
+        currentFile.hunks.push(currentHunk);
+      }
+      continue;
+    }
+
+    if (!currentHunk) continue;
+
+    if (line.startsWith('+') && !line.startsWith('+++')) {
+      currentFile.additions++;
+      totalAdditions++;
+      currentHunk.lines.push({
+        type: 'add',
+        text: line.slice(1),
+      });
+    } else if (line.startsWith('-') && !line.startsWith('---')) {
+      currentFile.deletions++;
+      totalDeletions++;
+      currentHunk.lines.push({
+        type: 'del',
+        text: line.slice(1),
+      });
+    } else if (line.startsWith(' ')) {
+      currentHunk.lines.push({
+        type: 'context',
+        text: line.slice(1),
+      });
+    }
+  }
+
+  return {
+    files,
+    stats: {
+      filesChanged: files.length,
+      additions: totalAdditions,
+      deletions: totalDeletions,
+    },
+    raw: diffText,
+  };
+}

package/server/gitHistory.js

@@ -0,0 +1,110 @@
+// Reading git history for a scanned root. The same discipline as gitClone.js:
+// git is spawned with an argument array — never through a shell — and its
+// output is parsed, not interpreted. A root without history is an expected
+// outcome, not an error: the browser folder-pick flow never touches git, a
+// downloaded tarball has no .git, and a plain folder may simply not be a
+// checkout. All of those come back as { available: false, reason } so the UI
+// can say why instead of showing zeros.
+
+import { spawn } from 'node:child_process';
+
+// How far back the window goes. Two thousand commits is years of work on most
+// repos and keeps the log parse bounded; `totalCommits` travels alongside so
+// truncation is stated, not silent.
+const LOG_CAP = 2000;
+const TIMEOUT_MS = 30_000;
+
+// Record separator \x1e between commits, unit separator \x1f between fields.
+// Filenames are one per line after the header. A filename containing \x1e or
+// \x1f is legal on paper and does not happen in practice — the same trade every
+// machine-readable git format makes (documented limitation, like .gitignore
+// negation in scan.js).
+const LOG_FORMAT = '%x1e%H%x1f%an%x1f%ae%x1f%aI';
+
+export async function gitLog(root) {
+  let countText;
+  try {
+    countText = await run(['-C', root, 'rev-list', '--count', 'HEAD']);
+  } catch (err) {
+    return { ok: false, reason: explain(err) };
+  }
+  const totalCommits = parseInt(countText.trim(), 10) || 0;
+  if (!totalCommits) {
+    return { ok: false, reason: 'The repository exists but has no commits yet.' };
+  }
+
+  let text;
+  try {
+    text = await run([
+      '-C', root, 'log',
+      '--max-count=' + LOG_CAP,
+      '--pretty=format:' + LOG_FORMAT,
+      '--name-only',
+      '--no-renames',
+      'HEAD',
+    ]);
+  } catch (err) {
+    return { ok: false, reason: explain(err) };
+  }
+
+  return { ok: true, text, totalCommits };
+}
+
+// Pure, so the format contract is testable without a git binary.
+export function parseGitLog(text) {
+  const commits = [];
+  for (const record of String(text).split('\x1e')) {
+    if (!record.trim()) continue;
+    const lines = record.split('\n');
+    const [hash, name, email, date] = lines[0].split('\x1f');
+    if (!hash) continue;
+    const files = [];
+    for (let i = 1; i < lines.length; i++) {
+      const f = lines[i].trim();
+      if (f) files.push(f);
+    }
+    commits.push({ hash, author: { name: name || '', email: email || '' }, date: date || '', files });
+  }
+  return commits;
+}
+
+function explain(err) {
+  const message = err.message || '';
+  if (err.code === 'ENOENT' || /spawn git ENOENT/.test(message)) {
+    return 'git is not installed or not on PATH, so there is no history to read.';
+  }
+  if (/not a git repository/i.test(message)) {
+    return 'No .git here — this folder is not a checkout, so there is no history to read.';
+  }
+  if (/does not have any commits yet/i.test(message)) {
+    return 'The repository exists but has no commits yet.';
+  }
+  return 'git would not say: ' + message.split('\n').pop();
+}
+
+function run(args) {
+  return new Promise((resolve, reject) => {
+    const child = spawn('git', args, { stdio: ['ignore', 'pipe', 'pipe'] });
+    let out = '';
+    let stderr = '';
+    child.stdout.on('data', (chunk) => { out += chunk; });
+    child.stderr.on('data', (chunk) => {
+      stderr += chunk;
+      if (stderr.length > 2000) stderr = stderr.slice(-2000);
+    });
+    const timer = setTimeout(() => {
+      child.kill('SIGKILL');
+      reject(new Error('git took too long and was stopped.'));
+    }, TIMEOUT_MS);
+    child.on('error', (err) => {
+      clearTimeout(timer);
+      err.message = 'Could not run git: ' + err.message;
+      reject(err);
+    });
+    child.on('close', (code) => {
+      clearTimeout(timer);
+      if (code === 0) resolve(out);
+      else reject(new Error(stderr.trim() || ('git exited ' + code)));
+    });
+  });
+}

package/server/htmlText.js

@@ -0,0 +1,44 @@
+// HTML to readable text, for the docs fetch. Not a parser and not trying to be:
+// the output is fed to a language model as context, so what matters is that the
+// prose survives and the markup, scripts and styles do not.
+//
+// Order is the whole trick. Script and style bodies go first — their contents are
+// text, not tags, so stripping tags first would leave a page of minified
+// JavaScript behind. The title is lifted before the general tag strip, because
+// after it there is no way to tell the title from the first paragraph. Entities
+// are decoded last, so a `<script>` written *about* HTML in the docs
+// cannot turn back into markup that the earlier passes would have removed.
+
+// Enough context for a model to summarise from without sending a whole site.
+const MAX_TEXT = 14000;
+
+const ENTITIES = {
+  '&nbsp;': ' ', '&amp;': '&', '&lt;': '<', '&gt;': '>', '&quot;': '"', '&#39;': "'",
+};
+
+// Horizontal space, newlines excluded. The non-breaking space belongs here:
+// `&nbsp;` is decoded to a plain space below, but pages carry the raw character
+// as well and it should not survive as indentation.
+const RUN_OF_SPACE = /[ \t\u00a0]+/g;
+const AROUND_NEWLINE = /[ \t\u00a0]*\n[ \t\u00a0]*/g;
+
+export function htmlToText(html, limit = MAX_TEXT) {
+  let s = String(html);
+  s = s.replace(/<script[\s\S]*?<\/script>/gi, ' ');
+  s = s.replace(/<style[\s\S]*?<\/style>/gi, ' ');
+
+  const rawTitle = (s.match(/<title[^>]*>([\s\S]*?)<\/title>/i) || [, ''])[1];
+  const title = rawTitle.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim();
+
+  s = s.replace(/<[^>]+>/g, ' ');
+  s = s.replace(/&nbsp;|&amp;|&lt;|&gt;|&quot;|&#39;/g, (m) => ENTITIES[m]);
+
+  // Newlines survive as paragraph breaks \u2014 a docs page read as one long line
+  // loses the structure a summary needs \u2014 but everything else collapses. Every
+  // tag became a space, so a break arrives as " \n\n\n " and the indentation of
+  // the source becomes indentation of the text; trimming around each newline is
+  // what makes the break itself clean.
+  s = s.replace(RUN_OF_SPACE, ' ').replace(AROUND_NEWLINE, '\n').replace(/\n{3,}/g, '\n\n').trim();
+
+  return { title, text: s.slice(0, limit) };
+}

package/server/http.js

@@ -0,0 +1,55 @@
+// Reading a request body and writing a response — the two things every route
+// does. Pulled out of the router so a handler can be read on its own, and so
+// the body limit is one number in one place rather than a check repeated per
+// route and forgotten on the next one.
+
+// A scan request carries a path or a URL; an explain request carries a prompt
+// and a slice of facts. Two megabytes is far more than either needs, and the
+// point is to have a ceiling at all: a local server with no auth should not let
+// a stray fetch hold memory open.
+export const MAX_BODY_BYTES = 2 * 1024 * 1024;
+
+// Resolves to the parsed JSON body, or `{}` for an empty one — a POST with no
+// body is a valid "just the defaults" request and every caller would otherwise
+// need the same guard. Rejects on oversized or malformed input; the router turns
+// that into a 500 with the message.
+export function readBody(req, limit = MAX_BODY_BYTES) {
+  return new Promise((resolve, reject) => {
+    let size = 0;
+    const chunks = [];
+    req.on('data', (chunk) => {
+      size += chunk.length;
+      if (size > limit) {
+        reject(new Error('Request body too large.'));
+        req.destroy();
+        return;
+      }
+      chunks.push(chunk);
+    });
+    req.on('end', () => {
+      try {
+        resolve(chunks.length ? JSON.parse(Buffer.concat(chunks).toString('utf8')) : {});
+      } catch {
+        reject(new Error('The request body was not valid JSON.'));
+      }
+    });
+    req.on('error', reject);
+  });
+}
+
+export function sendJSON(res, status, data) {
+  res.writeHead(status, { 'content-type': 'application/json; charset=utf-8' });
+  res.end(JSON.stringify(data));
+}
+
+// Every error the UI shows comes back through here, so the shape is fixed:
+// `{ error }` with a sentence a person can act on. The front end reads `.error`
+// and nothing else.
+export function sendError(res, status, message) {
+  sendJSON(res, status, { error: message });
+}
+
+export function sendText(res, status, text) {
+  res.writeHead(status, { 'content-type': 'text/plain; charset=utf-8' });
+  res.end(text);
+}

package/server/httpGuards.js

@@ -0,0 +1,87 @@
+// Two questions worth asking about a request before it is allowed to do
+// anything: did it arrive at a name that really is this machine, and did it
+// come from this app's own page?
+//
+// Neither is authentication — a single-user local tool has nobody to
+// authenticate. Both exist to stop some *other* page the user happens to have
+// open from driving this server on their behalf. Binding to 127.0.0.1 keeps
+// the network out; it does nothing about the browser, which is already inside.
+
+const LOOPBACK_HOSTS = new Set(['localhost', '127.0.0.1', '[::1]', '::1']);
+
+// "localhost:4310" -> "localhost", "[::1]:4310" -> "[::1]"
+function hostnameOf(hostHeader) {
+  const raw = String(hostHeader ?? '').trim().toLowerCase();
+  if (raw.startsWith('[')) {
+    const close = raw.indexOf(']');
+    return close === -1 ? raw : raw.slice(0, close + 1);
+  }
+  const colon = raw.lastIndexOf(':');
+  return colon === -1 ? raw : raw.slice(0, colon);
+}
+
+// DNS rebinding, the attack this server's loopback bind does not cover: a page
+// on evil.com whose DNS answer flips to 127.0.0.1 reaches us anyway, and from
+// then on the browser treats evil.com as the same origin as everything we
+// return — so the origin check below is looking at `same-origin` and sees
+// nothing wrong. What still gives it away is the Host header, which the page
+// cannot set and which still says evil.com.
+//
+// `extraHosts` is the self-hosted escape hatch: the domain and bind address
+// from the settings file, so a server that was deliberately put on the network
+// answers to its own name and nothing else. A local server passes nothing and
+// keeps the loopback-only rule. Entries may be exact names or `*.suffix`
+// wildcards — enabled tunnels put their random upstream names here, which
+// cannot be known ahead of time.
+//
+// Returns null when the request is fine, or a short reason when it isn't.
+export function rebindingReason(req, extraHosts = []) {
+  const host = req.headers?.host;
+  if (!host) return 'The request arrived with no Host header.';
+  const name = hostnameOf(host);
+  if (LOOPBACK_HOSTS.has(name)) return null;
+  for (const extra of extraHosts) {
+    if (extra.startsWith('*.')) {
+      // "*.trycloudflare.com" matches "abc.trycloudflare.com" but not the bare
+      // suffix itself and not "evil-trycloudflare.com".
+      if (name.endsWith(extra.slice(1)) && name.length > extra.length) return null;
+    } else if (name === extra) {
+      return null;
+    }
+  }
+  return `It arrived addressed to ${name}.`;
+}
+
+// Cross-site requests: any page in the browser can POST here without being
+// able to read the answer, and "without being able to read the answer" is not
+// the same as harmless — /api/scan walks a directory the caller names, and
+// /api/explain will POST to any URL the caller names.
+//
+// Two signals, and a request only has to fail one to be refused:
+//
+//   Sec-Fetch-Site  set by the browser, unsettable by page script. `none`
+//                   means the user did it themselves — typed the URL, opened
+//                   a bookmark — which no attacking page can arrange.
+//   Origin          sent on every fetch that isn't GET/HEAD. Compared against
+//                   this server's own origin, which is `http://` plus the Host
+//                   we just confirmed is loopback.
+//
+// A request carrying neither header isn't a browser at all — curl, a test, a
+// script — and there is no cross-site story for those, so they pass.
+export function crossOriginReason(req) {
+  const site = req.headers?.['sec-fetch-site'];
+  if (site && site !== 'same-origin' && site !== 'none') {
+    return `The browser reported it as ${site}.`;
+  }
+  const origin = req.headers?.origin;
+  if (origin) {
+    // The host was just vetted by the rebinding check, so the origin only has
+    // to agree with it. Both schemes are accepted: a self-hosted server behind
+    // a tunnel is reached over https while the page's fetches still arrive
+    // here addressed to the same host the tunnel forwarded.
+    const host = String(req.headers.host ?? '').trim().toLowerCase();
+    const expected = new Set(['http://' + host, 'https://' + host]);
+    if (!expected.has(String(origin).trim().toLowerCase())) return `It came from ${origin}.`;
+  }
+  return null;
+}