@sikkuumiii/defuse 0.0.0-stage → 0.5.0

package/src/parse/languages.ts

@@ -0,0 +1,232 @@
+/**
+ * LANGUAGE REGISTRY
+ * =================
+ *
+ * A scanner has to answer "what language is this file?" before it can answer
+ * "is this file dangerous?". This file is the single place that knows.
+ *
+ * ARCHITECTURAL DECISION (matters in Phase 5, language expansion):
+ * languages are DATA, not code. Adding Ruby later means adding one object to
+ * the array below plus one query file per rule - it does not mean editing the
+ * parser, the engine, the CLI, or any existing rule. If adding a language ever
+ * requires touching the engine, we designed it wrong.
+ *
+ * WHY WASM GRAMMARS INSTEAD OF NATIVE ONES:
+ * Tree-sitter ships two ways to run a grammar - a native C module you compile
+ * on your machine, or a WebAssembly (.wasm) file that runs anywhere. We chose
+ * WASM for three reasons:
+ *   1. No compiler toolchain needed. `npm install` just works, on any OS.
+ *   2. The SAME grammar files run in a browser. Phase 2 is a browser UI - it
+ *      will run this identical engine instead of a reimplementation of it.
+ *   3. The tree-sitter-wasms package already ships 36 grammars, so Phase 5's
+ *      language expansion is mostly unlocking files we already have on disk.
+ */
+
+/** The languages Defuse supports. */
+export type LanguageId =
+  | 'javascript'
+  | 'typescript'
+  | 'python'
+  | 'java'
+  | 'go'
+  | 'php'
+  | 'c'
+  | 'cpp';
+
+export interface Dialect {
+  /** File extensions handled by this dialect. */
+  readonly extensions: readonly string[];
+  /** Grammar name inside tree-sitter-wasms, i.e. tree-sitter-<name>.wasm */
+  readonly grammar: string;
+}
+
+export interface LanguageSpec {
+  readonly id: LanguageId;
+  readonly displayName: string;
+  /** Primary extensions -> primary grammar. */
+  readonly extensions: readonly string[];
+  readonly grammar: string;
+  /**
+   * Variants of the same language that need a DIFFERENT grammar file but share
+   * rules. e.g. .tsx needs the tsx grammar, but a SQL-injection rule written for
+   * TypeScript matches .tsx code perfectly well.
+   */
+  readonly dialects?: readonly Dialect[];
+  /** Interpreter names seen in a `#!` shebang line, for extensionless scripts. */
+  readonly shebangs?: readonly string[];
+  /** Honest notes about what this language's support does and does not include. */
+  readonly notes: string;
+}
+
+export const LANGUAGES: readonly LanguageSpec[] = [
+  {
+    id: 'javascript',
+    displayName: 'JavaScript',
+    extensions: ['.js', '.mjs', '.cjs', '.jsx'],
+    grammar: 'javascript',
+    shebangs: ['node'],
+    notes:
+      'The javascript grammar parses JSX natively, so .jsx needs no separate grammar. ' +
+      'Vue/Svelte single-file components are NOT supported - their <script> blocks are ' +
+      'not extracted. Documented gap, not a silent one.',
+  },
+  {
+    id: 'typescript',
+    displayName: 'TypeScript',
+    extensions: ['.ts', '.mts', '.cts'],
+    grammar: 'typescript',
+    dialects: [{ extensions: ['.tsx'], grammar: 'tsx' }],
+    notes:
+      'Type annotations are parsed but NOT used for analysis. Knowing that ' +
+      'a parameter is typed `string` tells us nothing about whether it is attacker-' +
+      'controlled, so we deliberately do not pretend it does. .d.ts files are skipped.',
+  },
+  {
+    id: 'python',
+    displayName: 'Python',
+    extensions: ['.py', '.pyi'],
+    grammar: 'python',
+    shebangs: ['python', 'python2', 'python3'],
+    notes:
+      'Python 3 grammar. Python 2 print-statements will produce parse errors, which we ' +
+      'report rather than swallow.',
+  },
+  {
+    id: 'java',
+    displayName: 'Java',
+    extensions: ['.java'],
+    grammar: 'java',
+    notes:
+      'Plain Java only. .jsp, .kt (Kotlin) and .scala are separate languages and are not ' +
+      'covered by these rules even though a Java-ish pattern might appear in them.',
+  },
+  {
+    id: 'php',
+    displayName: 'PHP',
+    extensions: ['.php', '.phtml'],
+    grammar: 'php',
+    notes:
+      'PHP source only. A .php file is parsed as a whole document, so HTML outside ' +
+      '<?php ?> is text rather than markup - an XSS sink written in raw HTML with a ' +
+      'short echo tag (<?= $x ?>) IS matched, but HTML attributes around it are not ' +
+      'analysed. Blade (.blade.php) and Twig (.twig) templates are NOT parsed.',
+  },
+  {
+    id: 'go',
+    displayName: 'Go',
+    extensions: ['.go'],
+    grammar: 'go',
+    notes:
+      'Go source only; go.mod, go.sum and generated protobuf files are skipped as noise.',
+  },
+  {
+    id: 'c',
+    displayName: 'C',
+    extensions: ['.c'],
+    grammar: 'c',
+    notes:
+      'MEMORY SAFETY IS NOT ANALYSED. C is famous for buffer overflows, use-after-free, ' +
+      'double-free and integer overflow, and this engine models none of them - it tracks ' +
+      'values, not sizes, allocations or pointer aliasing. What IS analysed is the same ' +
+      'thing analysed in every other language: attacker-controlled data reaching a ' +
+      'dangerous call. In C that means shells (system, popen, exec*), format strings ' +
+      '(printf-family, where a tainted FORMAT argument is CWE-134 and a taint engine ' +
+      'catches it exactly), and copies with no length bound (strcpy, strcat, sprintf, ' +
+      'gets). A finding that data reaches an unbounded copy is NOT a claim that the ' +
+      'destination is too small - that needs size analysis we do not have - so the ' +
+      'wording says "reaches a copy with no length bound" and means only that. ' +
+      'THE PREPROCESSOR IS NOT RUN: #include is not followed, #ifdef branches are all ' +
+      'parsed as though taken, and a macro that hides a sink (#define RUN(x) system(x)) ' +
+      'is invisible.',
+  },
+  {
+    id: 'cpp',
+    displayName: 'C++',
+    extensions: ['.cpp', '.cc', '.cxx', '.c++', '.hpp', '.hh', '.hxx', '.h'],
+    grammar: 'cpp',
+    notes:
+      'Every C limitation applies, plus: templates are parsed but not instantiated, so a ' +
+      'sink reached only through a template specialisation is missed, and operator ' +
+      'overloading is not resolved - a custom operator<< that executes a command reads ' +
+      'as an ordinary stream write. ' +
+      'THE .h EXTENSION IS TREATED AS C++, deliberately and against appearances. It is ' +
+      'ambiguous - most .h files in the world are C - but the two grammars are not ' +
+      'symmetric: measured on this project\'s own fixtures, the C++ grammar parses C ' +
+      'with zero error nodes while the C grammar produces eleven on C++. Choosing the ' +
+      'superset costs a C project a cosmetic mislabel in the per-language counts; ' +
+      'choosing C would cost a C++ header every std:: sink in it, silently. A wrong ' +
+      'label is better than a missing finding.',
+  },
+];
+
+/** Extension -> { language, grammar-to-use }. Built once at module load. */
+const BY_EXTENSION = new Map<string, { spec: LanguageSpec; grammar: string }>();
+for (const spec of LANGUAGES) {
+  for (const ext of spec.extensions) {
+    BY_EXTENSION.set(ext, { spec, grammar: spec.grammar });
+  }
+  for (const dialect of spec.dialects ?? []) {
+    for (const ext of dialect.extensions) {
+      BY_EXTENSION.set(ext, { spec, grammar: dialect.grammar });
+    }
+  }
+}
+
+export interface LanguageMatch {
+  readonly spec: LanguageSpec;
+  /** Which .wasm grammar to actually load (may differ from spec.grammar for dialects). */
+  readonly grammar: string;
+  /** How we decided. Reported in --json so detection is auditable, never magic. */
+  readonly detectedBy: 'extension' | 'shebang';
+}
+
+/** Files we never scan, because scanning them is noise, not security. */
+const SKIP_FILE_PATTERNS: readonly RegExp[] = [
+  /\.d\.ts$/,
+  /\.min\.js$/,
+  /\.bundle\.js$/,
+  /\.pb\.go$/,
+  /_pb2\.py$/,
+  /\.generated\./,
+];
+
+/**
+ * Decide which language a file is.
+ *
+ * Step 1: file extension. Boring, fast, right ~99% of the time.
+ * Step 2: if there is no useful extension, read the first line and look for a
+ *         "shebang" - the `#!/usr/bin/env python3` line that tells Unix which
+ *         interpreter to run. This is how we catch a CLI script named `deploy`
+ *         with no extension at all.
+ *
+ * Returns null for "we don't handle this", which is a normal answer, not an error.
+ */
+export function detectLanguage(filePath: string, firstLine?: string): LanguageMatch | null {
+  const lower = filePath.toLowerCase();
+
+  for (const pattern of SKIP_FILE_PATTERNS) {
+    if (pattern.test(lower)) return null;
+  }
+
+  const dot = lower.lastIndexOf('.');
+  const slash = Math.max(lower.lastIndexOf('/'), lower.lastIndexOf('\\'));
+  if (dot > slash) {
+    const hit = BY_EXTENSION.get(lower.slice(dot));
+    if (hit) return { spec: hit.spec, grammar: hit.grammar, detectedBy: 'extension' };
+  }
+
+  if (firstLine?.startsWith('#!')) {
+    // "#!/usr/bin/env python3" -> we want the word "python3"
+    const words = firstLine.slice(2).trim().split(/[\s/]+/);
+    for (const spec of LANGUAGES) {
+      if (spec.shebangs?.some((s) => words.includes(s))) {
+        return { spec, grammar: spec.grammar, detectedBy: 'shebang' };
+      }
+    }
+  }
+
+  return null;
+}
+
+/** Every extension we will attempt to open. Used by the file walker. */
+export const SCANNABLE_EXTENSIONS: readonly string[] = [...BY_EXTENSION.keys()];

package/src/parse/location.ts

@@ -0,0 +1,44 @@
+/**
+ * Turning a tree-sitter node into a human-readable, reproducible location.
+ *
+ * Tree-sitter counts rows and columns from 0. Editors, terminals and humans
+ * count from 1. Every off-by-one bug in a scanner's output lives right here, so
+ * the conversion happens in exactly ONE place: this file.
+ */
+
+import type { Node } from 'web-tree-sitter';
+import type { CodeLocation } from '../core/finding.js';
+
+const MAX_SNIPPET = 160;
+
+/** Walk up the tree collecting node types: "program > call_expression > ...". */
+export function astPath(node: Node, maxDepth = 5): string {
+  const parts: string[] = [];
+  let current: Node | null = node;
+  while (current && parts.length < maxDepth) {
+    parts.unshift(current.type);
+    current = current.parent;
+  }
+  return parts.join(' > ');
+}
+
+/**
+ * Build the location record that goes into a finding.
+ * `file` is passed in because a node does not know which file it came from -
+ * tree-sitter only ever sees a string of text.
+ */
+export function nodeLocation(node: Node, file: string): CodeLocation {
+  const raw = node.text ?? '';
+  const collapsed = raw.replace(/\s+/g, ' ').trim();
+  return {
+    file,
+    startLine: node.startPosition.row + 1,
+    startColumn: node.startPosition.column + 1,
+    endLine: node.endPosition.row + 1,
+    endColumn: node.endPosition.column + 1,
+    astNodeType: node.type,
+    astNodePath: astPath(node),
+    snippet:
+      collapsed.length > MAX_SNIPPET ? `${collapsed.slice(0, MAX_SNIPPET - 1)}…` : collapsed,
+  };
+}

package/src/parse/node-grammars.ts

@@ -0,0 +1,41 @@
+/**
+ * THE NODE SHELL for grammar loading and file reading.
+ *
+ * Everything in here is the half of the tool that only makes sense on a
+ * machine with a filesystem. The browser has its own equivalent in
+ * ui/worker.js, and the analysis code in between never learns which one it got.
+ */
+
+import { readFile } from 'node:fs/promises';
+import { createRequire } from 'node:module';
+import path from 'node:path';
+import { configureParsing, parseSourceFile, type ParseOutcome } from './parser.js';
+
+const require = createRequire(import.meta.url);
+
+/** Folder inside node_modules that holds the prebuilt .wasm grammar files. */
+const GRAMMAR_DIR = path.join(
+  path.dirname(require.resolve('tree-sitter-wasms/package.json')),
+  'out',
+);
+
+/** Point the parser at node_modules. Called once, before any scan. */
+export function useNodeGrammars(): void {
+  configureParsing({
+    loadGrammar: (grammar) => path.join(GRAMMAR_DIR, `tree-sitter-${grammar}.wasm`),
+  });
+}
+
+/** Read a file off disk, then hand the text to the platform-agnostic parser. */
+export async function parseFile(filePath: string): Promise<ParseOutcome> {
+  try {
+    const source = await readFile(filePath, 'utf8');
+    return parseSourceFile(filePath, source);
+  } catch (error) {
+    return {
+      status: 'skipped',
+      path: filePath,
+      reason: `unreadable: ${(error as Error).message}`,
+    };
+  }
+}

package/src/parse/parser.ts

@@ -0,0 +1,262 @@
+/**
+ * WHAT IS AN AST? (read this once, then the rest of the project makes sense)
+ * =========================================================================
+ *
+ * When you write:
+ *
+ *     const total = price + tax;
+ *
+ * your computer does not see a sentence. It sees a TREE. "AST" stands for
+ * Abstract Syntax Tree: "abstract" because it throws away things that don't
+ * change meaning (spaces, most punctuation), "syntax" because it captures the
+ * grammar of the language, and "tree" because code nests inside code.
+ *
+ * That line becomes something like:
+ *
+ *     lexical_declaration
+ *      +- variable_declarator
+ *          +- name: identifier          "total"
+ *          +- value: binary_expression
+ *              +- left:     identifier  "price"
+ *              +- operator: "+"
+ *              +- right:    identifier  "tax"
+ *
+ * Why this matters enormously for a security scanner:
+ *
+ * The old way (what most cheap scanners do, and what earlier versions of this
+ * project did) is to search the raw TEXT with regular expressions. Text search
+ * cannot tell the difference between:
+ *
+ *     db.query("SELECT * FROM users WHERE id = " + userId)   <- real problem
+ *     // db.query("SELECT * FROM users WHERE id = " + userId)  <- a comment
+ *     log("db.query is unsafe when you do \" + userId\"")      <- a string
+ *
+ * All three contain the same characters. Only one is a vulnerability. An AST
+ * knows which is which, because the parser already decided "this is a comment",
+ * "this is a string literal", "this is a real function call". We get that
+ * accuracy for free, and it is the entire reason we use tree-sitter.
+ *
+ * Tree-sitter is Github's parser toolkit. It is fast, it is error-tolerant
+ * (it produces a usable tree even for code that doesn't compile - important,
+ * because real repositories contain broken files), and it has mature grammars
+ * for dozens of languages that all behave the same way.
+ */
+
+import { Language, Parser, type Node, type Tree } from 'web-tree-sitter';
+import { detectLanguage, type LanguageSpec } from './languages.js';
+
+/**
+ * WHERE DO THE .wasm GRAMMARS COME FROM?
+ *
+ * On Node they are files in node_modules. In a browser they are URLs fetched
+ * over HTTP. That is the ONLY difference between the two platforms in the whole
+ * analysis path, so it is the only thing abstracted: everything else - parsing,
+ * rules, taint - is identical code running in both places.
+ *
+ * A loader returns whatever `Language.load` accepts: a path, a URL, or bytes.
+ */
+export type GrammarLoader = (grammar: string) => string | Uint8Array | Promise<string | Uint8Array>;
+
+let grammarLoader: GrammarLoader | null = null;
+let engineInit: (() => Promise<void>) | null = null;
+
+/** Called once by whichever shell is in charge (CLI or browser worker). */
+export function configureParsing(options: {
+  loadGrammar: GrammarLoader;
+  init?: () => Promise<void>;
+}): void {
+  grammarLoader = options.loadGrammar;
+  engineInit = options.init ?? null;
+  engineReady = null;
+  parserCache.clear();
+  languageCache.clear();
+}
+
+let engineReady: Promise<void> | null = null;
+
+/**
+ * Boot the WebAssembly runtime. Must happen once before any parsing.
+ * We memoise the promise so calling this from ten places still only boots once.
+ */
+export async function initEngine(): Promise<void> {
+  engineReady ??= engineInit ? engineInit() : Parser.init();
+  await engineReady;
+}
+
+/** Grammar name -> loaded Parser. Loading a grammar costs ~10-40ms, so we cache. */
+const parserCache = new Map<string, Parser>();
+
+async function getParser(grammar: string): Promise<Parser> {
+  const cached = parserCache.get(grammar);
+  if (cached) return cached;
+
+  await initEngine();
+  if (!grammarLoader) {
+    throw new Error(
+      'No grammar loader configured. Call configureParsing() first - ' +
+        'src/parse/node-grammars.ts for the CLI, ui/worker.js for the browser.',
+    );
+  }
+  const language = await Language.load(await grammarLoader(grammar));
+  const parser = new Parser();
+  parser.setLanguage(language);
+  parserCache.set(grammar, parser);
+  return parser;
+}
+
+/** The tree-sitter Language object, needed to compile queries. */
+const languageCache = new Map<string, Language>();
+
+export async function getLanguage(grammar: string): Promise<Language> {
+  const cached = languageCache.get(grammar);
+  if (cached) return cached;
+  const parser = await getParser(grammar);
+  const language = parser.language;
+  if (!language) throw new Error(`Grammar ${grammar} loaded but exposed no language`);
+  languageCache.set(grammar, language);
+  return language;
+}
+
+/** A place where tree-sitter could not make sense of the source. */
+export interface ParseIssue {
+  readonly kind: 'ERROR' | 'MISSING';
+  readonly line: number;
+  readonly column: number;
+  readonly text: string;
+}
+
+/** Everything downstream code needs about one parsed file. */
+export interface ParsedFile {
+  readonly path: string;
+  readonly language: LanguageSpec;
+  /** May differ from language.grammar for dialects (.tsx uses the tsx grammar). */
+  readonly grammar: string;
+  readonly detectedBy: 'extension' | 'shebang';
+  readonly source: string;
+  readonly tree: Tree;
+  readonly root: Node;
+  /**
+   * Parse problems, reported not hidden. A file with issues is still scanned -
+   * tree-sitter recovers - but the report says so, because findings from a
+   * partially-understood file deserve less trust.
+   */
+  readonly parseIssues: readonly ParseIssue[];
+}
+
+export type ParseOutcome =
+  | { readonly status: 'parsed'; readonly file: ParsedFile }
+  | { readonly status: 'skipped'; readonly path: string; readonly reason: string };
+
+const MAX_REPORTED_ISSUES = 25;
+
+/**
+ * Walk the whole tree looking for the two ways tree-sitter admits defeat:
+ *   ERROR   - "I found characters here I cannot fit into the grammar"
+ *   MISSING - "the grammar says something must be here; I inserted a fake one"
+ *
+ * We use an explicit stack rather than recursion because deeply nested files
+ * (minified code, generated code) can blow the JavaScript call stack.
+ */
+function collectParseIssues(root: Node): ParseIssue[] {
+  if (!root.hasError) return [];
+  const issues: ParseIssue[] = [];
+  const stack: Node[] = [root];
+
+  while (stack.length > 0 && issues.length < MAX_REPORTED_ISSUES) {
+    const node = stack.pop();
+    if (!node) break;
+
+    if (node.type === 'ERROR' || node.isMissing) {
+      issues.push({
+        kind: node.isMissing ? 'MISSING' : 'ERROR',
+        line: node.startPosition.row + 1,
+        column: node.startPosition.column + 1,
+        text: (node.text ?? '').replace(/\s+/g, ' ').trim().slice(0, 60),
+      });
+      // Don't descend into an error subtree; its children are noise.
+      continue;
+    }
+
+    // Only descend where an error actually lives - a huge speed win.
+    if (node.hasError) {
+      for (let i = node.childCount - 1; i >= 0; i--) {
+        const child = node.child(i);
+        if (child) stack.push(child);
+      }
+    }
+  }
+  return issues;
+}
+
+/** Parse a string of code you already have in memory. Used by tests and by `ast`. */
+export async function parseSource(source: string, grammar: string): Promise<Tree> {
+  const parser = await getParser(grammar);
+  const tree = parser.parse(source);
+  if (!tree) throw new Error(`Parser returned no tree for grammar ${grammar}`);
+  return tree;
+}
+
+/**
+ * Parse WITHOUT awaiting, for a grammar that has already been loaded.
+ *
+ * WHY THIS EXISTS. A syntax tree lives in WebAssembly memory, which the
+ * JavaScript garbage collector cannot see into and will never reclaim; the only
+ * way to free one is to call tree.delete() by hand. Holding every tree in a
+ * repository at once therefore costs about sixty times the source - measured,
+ * not guessed - and that is the ceiling a large scan dies against.
+ *
+ * The fix is to free trees and parse a file again if something needs it later.
+ * "Later" is inside the tracer, which is synchronous all the way down, so the
+ * re-parse has to be synchronous too.
+ *
+ * It is safe to demand that here because loading is what is actually async, and
+ * by the time anything re-parses, every grammar in the scan has already been
+ * loaded during the first pass. If that assumption ever breaks, this throws
+ * rather than silently returning nothing - a missing tree would look exactly
+ * like a file with no vulnerabilities in it.
+ */
+export function parseSourceSync(source: string, grammar: string): Tree {
+  const parser = parserCache.get(grammar);
+  if (!parser) {
+    throw new Error(
+      `Grammar ${grammar} is not loaded, so it cannot be parsed synchronously. ` +
+        'Every grammar must be loaded by the first (async) pass before anything re-parses.',
+    );
+  }
+  const tree = parser.parse(source);
+  if (!tree) throw new Error(`Parser returned no tree for grammar ${grammar}`);
+  return tree;
+}
+
+/**
+ * Work out a file's language and parse it, from text already in memory.
+ * No filesystem, no platform assumptions - this is what the browser calls too.
+ */
+export async function parseSourceFile(filePath: string, source: string): Promise<ParseOutcome> {
+  // A binary file that happens to end in .js would waste a parse and produce
+  // garbage findings. NUL bytes are the cheap, reliable binary tell.
+  if (source.includes('\u0000')) {
+    return { status: 'skipped', path: filePath, reason: 'binary file' };
+  }
+
+  const firstLine = source.slice(0, source.indexOf('\n') + 1 || 200);
+  const match = detectLanguage(filePath, firstLine);
+  if (!match) {
+    return { status: 'skipped', path: filePath, reason: 'unsupported file type' };
+  }
+
+  const tree = await parseSource(source, match.grammar);
+  return {
+    status: 'parsed',
+    file: {
+      path: filePath,
+      language: match.spec,
+      grammar: match.grammar,
+      detectedBy: match.detectedBy,
+      source,
+      tree,
+      root: tree.rootNode,
+      parseIssues: collectParseIssues(tree.rootNode),
+    },
+  };
+}