codebase-onboarder 0.1.0

package/server/tools/install.js

@@ -0,0 +1,246 @@
+// Installing an engine from the GUI.
+//
+// The engine's founding rule was "detect, never install" — the person
+// self-hosting decides what goes on the machine. This module relaxes *who
+// runs the command*, not *what runs*: the install plans below are fixed data,
+// one per tool per way of installing, and the GUI's Install button is the
+// person deciding. No plan takes user input; every step is an argument array
+// spawned without a shell, exactly like the analyzers themselves.
+//
+// Plans are ordered by preference and each names the launcher it needs
+// (`brew`, `pip`, `npm`, `winget`, `curl`, …). At install time the first plan
+// whose launcher is found on PATH runs; the rest are skipped. That is what
+// makes one codebase serve all three platforms — a Mac with Homebrew, a
+// Windows box with winget, and a bare Linux server with nothing but curl all
+// get a working path, and the log says plainly which one was taken.
+
+import { spawn } from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+import { findOnPath, onboarderBinDir, spawnArgv } from './platform.js';
+import { clearDetectionCache } from '../tools.js';
+
+// Gitleaks has no package-manager story on Linux, so the fallback is its
+// official release asset, fetched over TLS from GitHub and unpacked into
+// ~/.onboarder/bin — a directory `findOnPath` already checks. The version is
+// pinned on purpose: a moving "latest" URL is a supply-chain surprise.
+const GITLEAKS_VERSION = '8.24.3';
+
+function gitleaksAsset(platform, arch) {
+  const osPart = { darwin: 'darwin', linux: 'linux', win32: 'windows' }[platform];
+  const archPart = arch === 'arm64' ? 'arm64' : 'x64';
+  const ext = platform === 'win32' ? 'zip' : 'tar.gz';
+  return {
+    url: `https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/gitleaks_${GITLEAKS_VERSION}_${osPart}_${archPart}.${ext}`,
+    file: `gitleaks.${ext}`,
+    binary: platform === 'win32' ? 'gitleaks.exe' : 'gitleaks',
+  };
+}
+
+// A plan step is `{ argv }`, built from a context so paths stay absolute.
+// `needs` is the launcher binary that must exist for the whole plan to run.
+const INSTALL_PLANS = {
+  semgrep: [
+    { id: 'brew', label: 'Homebrew', needs: 'brew', platforms: ['darwin'],
+      steps: (bin) => [[bin, ['install', 'semgrep']]] },
+    { id: 'pipx', label: 'pipx', needs: 'pipx',
+      steps: (bin) => [[bin, ['install', 'semgrep']]] },
+    { id: 'uv', label: 'uv', needs: 'uv',
+      steps: (bin) => [[bin, ['tool', 'install', 'semgrep']]] },
+    { id: 'pip', label: 'pip (user site)', needs: 'python3', platforms: ['darwin', 'linux'],
+      steps: (bin) => [[bin, ['-m', 'pip', 'install', '--user', 'semgrep']]] },
+    { id: 'pip', label: 'pip (user site)', needs: 'python', platforms: ['win32'],
+      steps: (bin) => [[bin, ['-m', 'pip', 'install', '--user', 'semgrep']]] },
+  ],
+  vulture: [
+    { id: 'brew', label: 'Homebrew', needs: 'brew', platforms: ['darwin'],
+      steps: (bin) => [[bin, ['install', 'vulture']]] },
+    { id: 'pipx', label: 'pipx', needs: 'pipx',
+      steps: (bin) => [[bin, ['install', 'vulture']]] },
+    { id: 'uv', label: 'uv', needs: 'uv',
+      steps: (bin) => [[bin, ['tool', 'install', 'vulture']]] },
+    { id: 'pip', label: 'pip (user site)', needs: 'python3', platforms: ['darwin', 'linux'],
+      steps: (bin) => [[bin, ['-m', 'pip', 'install', '--user', 'vulture']]] },
+    { id: 'pip', label: 'pip (user site)', needs: 'python', platforms: ['win32'],
+      steps: (bin) => [[bin, ['-m', 'pip', 'install', '--user', 'vulture']]] },
+  ],
+  knip: [
+    { id: 'npm', label: 'npm (global)', needs: 'npm',
+      steps: (bin) => [[bin, ['install', '-g', 'knip']]] },
+  ],
+  depcheck: [
+    { id: 'npm', label: 'npm (global)', needs: 'npm',
+      steps: (bin) => [[bin, ['install', '-g', 'depcheck']]] },
+  ],
+  gitleaks: [
+    { id: 'brew', label: 'Homebrew', needs: 'brew', platforms: ['darwin'],
+      steps: (bin) => [[bin, ['install', 'gitleaks']]] },
+    { id: 'winget', label: 'winget', needs: 'winget', platforms: ['win32'],
+      steps: (bin) => [[bin, ['install', '--id', 'Gitleaks.Gitleaks', '-e',
+        '--accept-package-agreements', '--accept-source-agreements']]] },
+    { id: 'download', label: 'GitHub release download', needs: 'curl',
+      steps: (bin, ctx) => {
+        const asset = gitleaksAsset(ctx.platform, ctx.arch);
+        const archive = path.join(ctx.tmpDir, asset.file);
+        return [
+          [bin, ['-fSL', '-o', archive, asset.url]],
+          // bsdtar ships with Windows 10+ and reads zip; GNU tar reads tar.gz.
+          // One command shape serves both.
+          ['tar', ['-xf', archive, '-C', ctx.binDir]],
+        ];
+      },
+      after: (ctx) => {
+        const asset = gitleaksAsset(ctx.platform, ctx.arch);
+        const binPath = path.join(ctx.binDir, asset.binary);
+        if (ctx.platform !== 'win32') fs.chmodSync(binPath, 0o755);
+        return binPath;
+      },
+    },
+  ],
+};
+// The plans a tool can use on this platform, in preference order. Pure —
+// exported for the status endpoint and for tests.
+export function plansFor(toolId, platform = process.platform) {
+  return (INSTALL_PLANS[toolId] || [])
+    .filter((plan) => !plan.platforms || plan.platforms.includes(platform));
+}
+
+// What the GUI's install dialog shows before the button is pressed: for each
+// plan, whether its launcher exists here. Injectable `find` keeps it testable.
+export function planAvailability(toolId, options = {}) {
+  const platform = options.platform || process.platform;
+  const find = options.find || ((bin) => findOnPath(bin));
+  return plansFor(toolId, platform).map((plan) => ({
+    id: plan.id,
+    label: plan.label,
+    needs: plan.needs,
+    ready: !!find(plan.needs),
+  }));
+}
+
+// One install at a time per tool — two Install clicks must not run pip against
+// itself. Held here rather than in the handler so the rule survives a second
+// caller.
+const installing = new Set();
+
+export function isInstalling(toolId) {
+  return installing.has(toolId);
+}
+
+const STEP_TIMEOUT_MS = 10 * 60 * 1000; // pip on a slow link is still finite
+
+// Spawn one step, forwarding output line by line. Resolves with the exit
+// status; never rejects — a failed plan is data for the next plan, and the
+// whole log goes to the GUI either way.
+function runStep(command, args, onLog) {
+  const { command: cmd, args: argv } = spawnArgv([command, ...args]);
+  return new Promise((resolve) => {
+    let child;
+    try {
+      child = spawn(cmd, argv, { stdio: ['ignore', 'pipe', 'pipe'] });
+    } catch (err) {
+      onLog(`could not start ${cmd}: ${err.message}`);
+      resolve(-1);
+      return;
+    }
+    let pending = '';
+    const feed = (chunk) => {
+      pending += chunk.toString();
+      const lines = pending.split(/\r?\n/);
+      pending = lines.pop();
+      for (const line of lines) if (line.trim()) onLog(line);
+    };
+    child.stdout.on('data', feed);
+    child.stderr.on('data', feed);
+    const timer = setTimeout(() => {
+      child.kill('SIGKILL');
+      onLog(`${cmd} took too long and was stopped.`);
+    }, STEP_TIMEOUT_MS);
+    child.on('error', (err) => {
+      clearTimeout(timer);
+      onLog(`could not run ${cmd}: ${err.message}`);
+      resolve(-1);
+    });
+    child.on('close', (code) => {
+      clearTimeout(timer);
+      if (pending.trim()) onLog(pending);
+      resolve(code ?? -1);
+    });
+  });
+}
+
+// Install a tool, streaming progress through `onEvent({ type, ... })`:
+//   { type: 'log', line }         — one line of output from the installer
+//   { type: 'plan', id, label }   — which plan is being attempted
+//   { type: 'done', ok, error?, method? }
+// Resolves with the same shape as the final `done` event.
+export async function installTool(toolId, onEvent = () => {}) {
+  const log = (line) => onEvent({ type: 'log', line });
+  const finish = (result) => { onEvent({ type: 'done', ...result }); return result; };
+
+  const plans = plansFor(toolId);
+  if (!plans.length) {
+    return finish({ ok: false, error: `Onboarder does not know how to install "${toolId}".` });
+  }
+  if (installing.has(toolId)) {
+    return finish({ ok: false, error: `${toolId} is already being installed.` });
+  }
+  installing.add(toolId);
+
+  try {
+    const ctx = {
+      platform: process.platform,
+      arch: process.arch,
+      tmpDir: os.tmpdir(),
+      binDir: onboarderBinDir(),
+    };
+
+    let attempted = 0;
+    for (const plan of plans) {
+      const launcher = findOnPath(plan.needs);
+      if (!launcher) {
+        log(`— skipping ${plan.label}: no ${plan.needs} on this machine.`);
+        continue;
+      }
+      attempted += 1;
+      onEvent({ type: 'plan', id: plan.id, label: plan.label });
+      log(`Installing ${toolId} with ${plan.label}…`);
+
+      if (plan.id === 'download') fs.mkdirSync(ctx.binDir, { recursive: true });
+
+      let failed = false;
+      for (const [command, args] of plan.steps(launcher, ctx)) {
+        log(`$ ${command} ${args.join(' ')}`);
+        const status = await runStep(command, args, log);
+        if (status !== 0) {
+          log(`✗ that step exited ${status}.`);
+          failed = true;
+          break;
+        }
+      }
+      if (failed) continue;
+
+      try {
+        const binPath = plan.after ? plan.after(ctx) : null;
+        if (binPath) log(`Installed to ${binPath}`);
+      } catch (err) {
+        log(`✗ unpack failed: ${err.message}`);
+        continue;
+      }
+
+      clearDetectionCache();
+      log('✓ done. Re-checking the engine list…');
+      return finish({ ok: true, method: plan.label });
+    }
+
+    const why = attempted
+      ? 'Every available installer failed — the log above says where.'
+      : 'No installer this machine has can install it. Try the manual command in the engine list.';
+    return finish({ ok: false, error: why });
+  } finally {
+    installing.delete(toolId);
+  }
+}
+

package/server/tools/parse.js

@@ -0,0 +1,170 @@
+// Parsing the dialects. Every analyzer speaks its own JSON (or SARIF); the UI
+// speaks one. This file is the translation layer, and it is pure — it takes a
+// string a tool printed and returns findings, so it can be tested in Node with
+// fixtures and needs no process, no filesystem, no PATH.
+//
+// The single target shape is the one the built-in scanner already emits
+// (`shared/analyzer/security.js`), so the existing security view needs no new
+// kind of row:
+//
+//   { rule, severity, category, message, line, excerpt, source, tool }
+//
+// `severity` is one of the built-in five. `source` is 'builtin' or 'external'
+// and `tool` names which engine answered, so the UI can say "found by Semgrep"
+// rather than letting a finding appear from nowhere.
+
+export const SEVERITY_ORDER = ['info', 'low', 'medium', 'high', 'critical'];
+
+export function normalizeSeverity(raw) {
+  const s = String(raw || '').toLowerCase();
+  if (s === 'critical' || s === 'error') return s === 'error' ? 'high' : 'critical';
+  if (s === 'high') return 'high';
+  if (s === 'medium' || s === 'warning' || s === 'warn' || s === 'moderate') return 'medium';
+  if (s === 'low' || s === 'note') return 'low';
+  return 'info';
+}
+
+function clipExcerpt(text) {
+  return String(text ?? '').replace(/\s+/g, ' ').trim().slice(0, 90);
+}
+
+// ---- Semgrep / Opengrep ----------------------------------------------------
+// `semgrep --json` → { results: [ { check_id, path, start:{line}, extra:
+// { severity, message, lines, metadata:{category} } } ] }. Opengrep is the same
+// JSON. `rel` strips the scan root so paths match the repo-relative form the
+// rest of the app uses.
+export function parseSemgrep(json, rel) {
+  const results = Array.isArray(json && json.results) ? json.results : [];
+  const findings = [];
+  for (const r of results) {
+    const extra = r.extra || {};
+    findings.push({
+      rule: String(r.check_id || 'semgrep-rule'),
+      severity: normalizeSeverity(extra.severity),
+      category: (extra.metadata && extra.metadata.category) || 'security',
+      message: clipExcerpt(extra.message || r.check_id),
+      path: rel(r.path),
+      line: r.start && r.start.line ? r.start.line : 1,
+      excerpt: clipExcerpt(extra.lines),
+      source: 'external',
+      tool: 'semgrep',
+    });
+  }
+  return findings;
+}
+
+// ---- Gitleaks ---------------------------------------------------------------
+// `gitleaks --report-format json` → an array of secrets: [ { RuleID,
+// Description, File, StartLine, Match, Secret } ]. The finding reports *that* a
+// secret exists, never the secret — `Secret` is read for nothing.
+export function parseGitleaks(json, rel) {
+  const list = Array.isArray(json) ? json : [];
+  const findings = [];
+  for (const r of list) {
+    findings.push({
+      rule: 'secret:' + (r.RuleID || 'unknown'),
+      severity: 'critical',
+      category: 'secret',
+      message: clipExcerpt(r.Description || 'A secret is committed to the source.'),
+      path: rel(r.File),
+      line: r.StartLine || 1,
+      excerpt: 'committed credential (value redacted)',
+      source: 'external',
+      tool: 'gitleaks',
+    });
+  }
+  return findings;
+}
+
+// ---- Knip -------------------------------------------------------------------
+// `knip --reporter json` groups dead code by kind: { files:[], dependencies:[],
+// unused:[{name,file,line}], ... }. Each becomes a finding so the dead-code
+// pass can be read in one list; the kind is the rule.
+export function parseKnip(json, rel) {
+  const findings = [];
+  const push = (rule, category, severity, path, line, message) => {
+    findings.push({ rule, category, severity, message, path: rel(path), line: line || 1, excerpt: '', source: 'external', tool: 'knip' });
+  };
+
+  for (const file of (json && json.files) || []) {
+    push('knip:unused-file', 'dead-file', 'medium', file, 1, 'This file is never imported anywhere.');
+  }
+  for (const dep of (json && json.dependencies) || []) {
+    push('knip:unused-dependency', 'dead-dependency', 'low', dep, 1, `Dependency "${dep}" is declared but never imported.`);
+  }
+  for (const u of (json && json.unused) || []) {
+    push('knip:unused-export', 'dead-export', 'low', u.file, u.line, `Export "${u.name}" is never used.`);
+  }
+  for (const u of (json && json.unresolved) || []) {
+    push('knip:unresolved', 'dead-import', 'medium', u.file, u.line, `Import "${u.name}" does not resolve to anything.`);
+  }
+  return findings;
+}
+
+// ---- Vulture ----------------------------------------------------------------
+// `vulture` prints lines like `src/util.py:42: unused function 'helper' (60%
+// confidence)`. There is no stable JSON, so we parse the text — which is why the
+// parser is a pure function over a string, and why the line shape is pinned by
+// a test rather than by reading the binary's source.
+const VULTURE_RE = /^(.+?):(\d+):\s+(.+?)\s+\((\d+)% confidence\)/;
+
+export function parseVulture(text, rel) {
+  const findings = [];
+  for (const line of String(text || '').split('\n')) {
+    const m = line.match(VULTURE_RE);
+    if (!m) continue;
+    const confidence = Number(m[4]);
+    if (confidence < 60) continue; // below vulture's own floor, it is guessing
+    findings.push({
+      rule: 'vulture:' + (m[3].split(' ')[0] || 'unused'),
+      severity: confidence >= 90 ? 'medium' : 'low',
+      category: 'dead-code',
+      message: clipExcerpt(m[3]) + ` (${confidence}% confident)`,
+      path: rel(m[1]),
+      line: Number(m[2]) || 1,
+      excerpt: '',
+      source: 'external',
+      tool: 'vulture',
+    });
+  }
+  return findings;
+}
+
+// ---- Depcheck ---------------------------------------------------------------
+// `depcheck --json` → { dependencies:[], devDependencies:[], missing:{} }.
+export function parseDepcheck(json) {
+  const findings = [];
+  const push = (name, dev) => findings.push({
+    rule: 'depcheck:unused',
+    severity: 'low',
+    category: 'dead-dependency',
+    message: `Package "${name}" is declared in package.json but never imported.`,
+    path: 'package.json',
+    line: 1,
+    excerpt: '',
+    source: 'external',
+    tool: 'depcheck',
+    dev,
+  });
+  for (const name of (json && json.dependencies) || []) push(name, false);
+  for (const name of (json && json.devDependencies) || []) push(name, true);
+  return findings;
+}
+
+// Merge the external passes into one list, keeping the built-in scan's findings
+// untouched. Dedup on (path, line, rule): two engines that flag the same line
+// for the same reason are one problem, not two.
+export function mergeFindings(...lists) {
+  const seen = new Set();
+  const out = [];
+  for (const list of lists) {
+    for (const f of list || []) {
+      const key = `${f.path}:${f.line}:${f.rule}`;
+      if (seen.has(key)) continue;
+      seen.add(key);
+      out.push(f);
+    }
+  }
+  return out;
+}
+

package/server/tools/platform.js

@@ -0,0 +1,91 @@
+// Cross-platform binary lookup.
+//
+// The engine used to ask the Unix `which` command where a tool lives. That is
+// the right answer on macOS and Linux and *no* answer on Windows, where the
+// command is `where` and an npm-installed tool is a `npm.cmd` shim, not an
+// ELF file. Doing the lookup by hand — split PATH, try the PATHEXT suffixes
+// on Windows — keeps one code path for all three platforms, keeps it testable
+// (`exists` is injectable), and removes a spawned process from detection.
+//
+// `findOnPath` also checks a couple of well-known directories that are on a
+// user's interactive PATH but often not on the server's — Homebrew on Apple
+// Silicon is the classic case — and the folder Onboarder's own installer
+// drops downloaded binaries into, so a tool installed from the GUI is found
+// on the next status check without a server restart.
+
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+// Where `install.js` puts binaries it downloads itself (gitleaks on Linux and
+// Windows). One fixed place, under the user's home, never the scanned repo.
+export function onboarderBinDir(home = os.homedir()) {
+  return path.join(home, '.onboarder', 'bin');
+}
+
+// The names a command can go by on disk. Windows resolves `npm` to the first
+// of `npm.exe`, `npm.cmd`, … on PATHEXT; POSIX looks for the name as-is.
+export function candidateNames(cmd, platform = process.platform, pathext = '') {
+  if (platform !== 'win32') return [cmd];
+  const exts = (pathext || process.env.PATHEXT || '.EXE;.CMD;.BAT;.COM')
+    .split(';').map((e) => e.trim()).filter(Boolean);
+  const lower = cmd.toLowerCase();
+  if (exts.some((e) => lower.endsWith(e.toLowerCase()))) return [cmd];
+  return [cmd, ...exts.map((e) => cmd + e.toLowerCase())];
+}
+
+// Directories worth checking beyond PATH itself. The Homebrew pair matters on
+// a Mac launched from Spotlight rather than a shell; the Onboarder bin dir is
+// where the GUI installer puts self-downloaded tools.
+export function extraSearchDirs(platform = process.platform, home = os.homedir()) {
+  const dirs = [onboarderBinDir(home)];
+  if (platform === 'darwin') dirs.push('/opt/homebrew/bin', '/usr/local/bin');
+  if (platform === 'linux') dirs.push('/usr/local/bin', path.join(home, '.local', 'bin'));
+  if (platform === 'win32') {
+    const local = process.env.LOCALAPPDATA;
+    if (local) dirs.push(path.join(local, 'Programs', 'Gitleaks'));
+  }
+  return dirs;
+}
+
+// Split a PATH string the way the platform does. Exported for tests.
+export function splitPath(pathValue, platform = process.platform) {
+  return String(pathValue || '')
+    .split(platform === 'win32' ? ';' : ':')
+    .map((d) => d.trim())
+    .filter(Boolean);
+}
+
+// Find `cmd` on PATH (plus the extra dirs). Returns the absolute path, or
+// null. Pure apart from the `exists` probe, which defaults to the disk and is
+// injectable so the tests never touch a real filesystem layout.
+export function findOnPath(cmd, options = {}) {
+  const platform = options.platform || process.platform;
+  const envPath = options.envPath !== undefined ? options.envPath : process.env.PATH;
+  const pathext = options.pathext !== undefined ? options.pathext : process.env.PATHEXT;
+  const exists = options.exists || ((p) => fs.existsSync(p));
+  const home = options.home || os.homedir();
+
+  const dirs = [...splitPath(envPath, platform), ...extraSearchDirs(platform, home)];
+  for (const dir of dirs) {
+    for (const name of candidateNames(cmd, platform, pathext)) {
+      const full = path.join(dir, name);
+      try {
+        if (exists(full)) return full;
+      } catch {
+        /* an unreadable dir is just not where the tool lives */
+      }
+    }
+  }
+  return null;
+}
+
+// A `.cmd`/`.bat` shim cannot be spawned directly on Windows (Node refuses
+// with EINVAL): it needs cmd.exe. `spawnArgv` rewrites the argv for that case
+// and leaves everything else untouched, so callers keep passing plain arrays.
+export function spawnArgv(argv, platform = process.platform) {
+  if (platform !== 'win32') return { command: argv[0], args: argv.slice(1) };
+  if (!/\.(cmd|bat)$/i.test(argv[0] || '')) return { command: argv[0], args: argv.slice(1) };
+  const quoted = argv.map((a) => `"${String(a).replace(/"/g, '\\"')}"`).join(' ');
+  return { command: 'cmd.exe', args: ['/d', '/s', '/c', quoted] };
+}