@sikkuumiii/defuse 0.0.0-stage → 0.5.0

package/src/engine/shapes.ts

@@ -0,0 +1,348 @@
+/**
+ * THE SHAPE LAYER
+ * ===============
+ *
+ * This is the translator between "five different languages" and "one set of
+ * rules". It uses tree-sitter's QUERY LANGUAGE, which is worth understanding
+ * because it is the only unfamiliar syntax in this project.
+ *
+ * A tree-sitter query is a pattern written as nested parentheses that describes
+ * a shape of tree, with @names marking the parts you want back:
+ *
+ *     (call_expression
+ *       function: (member_expression
+ *         object: (_) @receiver
+ *         property: (property_identifier) @method)
+ *       arguments: (arguments) @args) @call
+ *
+ * Read it as: "find a call_expression; its `function` child must be a
+ * member_expression (that's the `a.b` shape); give me whatever is on the left
+ * of the dot as @receiver, the name after the dot as @method, the argument list
+ * as @args, and the whole call as @call."
+ *
+ * `(_)` means "any node". Field names before the colon (`function:`,
+ * `object:`) are the ROLE labels you saw in the AST dump. Matching on roles
+ * instead of child positions is what makes these patterns stable.
+ *
+ * Every language below gets its own query because the grammars use different
+ * node names for the same idea - JavaScript says `call_expression`, Python says
+ * `call`, Java says `method_invocation`. The OUTPUT is identical for all five,
+ * and that output is what rules see.
+ *
+ * ADDING A LANGUAGE LATER means adding two query strings to this file. No rule
+ * changes.
+ */
+
+import { Query, type Node, type QueryMatch } from 'web-tree-sitter';
+import type { Assignment, CallSite, Shape } from '../rules/contract.js';
+import type { LanguageId } from '../parse/languages.js';
+import { getLanguage, type ParsedFile } from '../parse/parser.js';
+
+/* ---------------------------------------------------------------------------
+ * CALL SITES: "something is being called with arguments"
+ * ------------------------------------------------------------------------ */
+
+/** Shared by C and C++, because C++ is a superset of these two shapes. */
+const C_CALL_QUERY = `
+  (call_expression
+    function: (identifier) @method
+    arguments: (argument_list) @args) @call
+  (call_expression
+    function: (field_expression argument: (_) @receiver field: (field_identifier) @method)
+    arguments: (argument_list) @args) @call
+`;
+
+const CALL_QUERIES: Record<LanguageId, string> = {
+  javascript: `
+    (call_expression
+      function: (member_expression object: (_) @receiver property: (property_identifier) @method)
+      arguments: (arguments) @args) @call
+    (call_expression
+      function: (identifier) @method
+      arguments: (arguments) @args) @call
+    (new_expression
+      constructor: (_) @method
+      arguments: (arguments) @args) @call
+  `,
+  // TypeScript's grammar is JavaScript's grammar plus types, so the call shapes
+  // are byte-for-byte identical. Kept as a separate entry rather than an alias
+  // so that a future TS-only pattern has an obvious home.
+  typescript: `
+    (call_expression
+      function: (member_expression object: (_) @receiver property: (property_identifier) @method)
+      arguments: (arguments) @args) @call
+    (call_expression
+      function: (identifier) @method
+      arguments: (arguments) @args) @call
+    (new_expression
+      constructor: (_) @method
+      arguments: (arguments) @args) @call
+  `,
+  python: `
+    (call
+      function: (attribute object: (_) @receiver attribute: (identifier) @method)
+      arguments: (argument_list) @args) @call
+    (call
+      function: (identifier) @method
+      arguments: (argument_list) @args) @call
+  `,
+  java: `
+    (method_invocation
+      object: (_) @receiver
+      name: (identifier) @method
+      arguments: (argument_list) @args) @call
+    (method_invocation
+      name: (identifier) @method
+      arguments: (argument_list) @args) @call
+    (object_creation_expression
+      type: (_) @method
+      arguments: (argument_list) @args) @call
+  `,
+  go: `
+    (call_expression
+      function: (selector_expression operand: (_) @receiver field: (field_identifier) @method)
+      arguments: (argument_list) @args) @call
+    (call_expression
+      function: (identifier) @method
+      arguments: (argument_list) @args) @call
+  `,
+  // PHP has three call shapes: a bare function, a method on an object, and a
+  // static method on a class. `echo` is deliberately NOT here - it is a
+  // statement, not a call, and the XSS rule handles it separately.
+  php: `
+    (function_call_expression
+      function: (name) @method
+      arguments: (arguments) @args) @call
+    (member_call_expression
+      object: (_) @receiver
+      name: (name) @method
+      arguments: (arguments) @args) @call
+    (scoped_call_expression
+      scope: (_) @receiver
+      name: (name) @method
+      arguments: (arguments) @args) @call
+    (object_creation_expression
+      (name) @method
+      (arguments) @args) @call
+  `,
+
+  /*
+   * C. Two shapes only, because C has two: a bare call, and a call through a
+   * struct member. Every node type and field name below was read off the real
+   * grammar with a query probe rather than assumed - the same discipline the
+   * PHP entry learned the hard way, when `if_statement` turned out to use
+   * `body` where every other grammar uses `consequence`.
+   */
+  c: C_CALL_QUERY,
+
+  /*
+   * C++ is C plus namespaces. `std::system(cmd)` parses as a
+   * qualified_identifier, NOT an identifier, so a query that only matched the
+   * C shapes would miss it - and reaching system() through <cstdlib> is the
+   * stylistically CORRECT way to do it in C++, which means the well-written
+   * code would be the code that slipped through.
+   */
+  cpp: `
+    ${C_CALL_QUERY}
+    (call_expression
+      function: (qualified_identifier name: (identifier) @method)
+      arguments: (argument_list) @args) @call
+  `,
+};
+
+/* ---------------------------------------------------------------------------
+ * ASSIGNMENTS: "a name is being bound to a value"
+ *
+ * Deliberately broad: it covers `x = v`, `const x = v`, object/dict entries
+ * `{ key: v }`, and keyword arguments `f(key=v)`. Hardcoded credentials hide in
+ * all of those, and so does `element.innerHTML = ...`.
+ * ------------------------------------------------------------------------ */
+
+const JS_ASSIGN_QUERY = `
+  (assignment_expression left: (_) @target right: (_) @value) @assign
+  (variable_declarator name: (_) @target value: (_) @value) @assign
+  (pair key: (_) @target value: (_) @value) @assign
+  (augmented_assignment_expression left: (_) @target right: (_) @value) @assign
+`;
+
+const C_ASSIGN_QUERY = `
+  (assignment_expression left: (_) @target right: (_) @value) @assign
+  (init_declarator declarator: (_) @target value: (_) @value) @assign
+`;
+
+const ASSIGNMENT_QUERIES: Record<LanguageId, string> = {
+  php: `
+    (assignment_expression left: (_) @target right: (_) @value) @assign
+    (property_element (variable_name) @target (_) @value) @assign
+    (echo_statement "echo" @target (_) @value) @assign
+  `,
+  javascript: JS_ASSIGN_QUERY,
+  typescript: `
+    ${JS_ASSIGN_QUERY}
+    (public_field_definition name: (_) @target value: (_) @value) @assign
+  `,
+  python: `
+    (assignment left: (_) @target right: (_) @value) @assign
+    (pair key: (_) @target value: (_) @value) @assign
+    (keyword_argument name: (identifier) @target value: (_) @value) @assign
+  `,
+  java: `
+    (variable_declarator name: (identifier) @target value: (_) @value) @assign
+    (assignment_expression left: (_) @target right: (_) @value) @assign
+  `,
+  go: `
+    (short_var_declaration left: (expression_list) @target right: (expression_list) @value) @assign
+    (assignment_statement left: (expression_list) @target right: (expression_list) @value) @assign
+    (var_spec name: (identifier) @target value: (expression_list) @value) @assign
+    (const_spec name: (identifier) @target value: (expression_list) @value) @assign
+    (keyed_element (literal_element (identifier) @target) (literal_element (_) @value)) @assign
+  `,
+
+  /*
+   * C and C++ share these. `init_declarator` is the declaration-with-value
+   * form (`char *p = getenv(...)`), and its @target captures the DECLARATOR
+   * rather than a bare name - so `*name = getenv("USER")` yields a target of
+   * `*name`, pointer star included. The tracer strips it; see POINTER_PREFIX
+   * in taint/tracer.ts. Recorded here because it is the kind of detail that
+   * silently halves a language's coverage if nobody notices it.
+   */
+  c: C_ASSIGN_QUERY,
+  cpp: C_ASSIGN_QUERY,
+};
+
+/* ------------------------------------------------------------------------ */
+
+/** Compiled queries are expensive to build and cheap to reuse. Cache per grammar. */
+const queryCache = new Map<string, Query>();
+
+async function compile(grammar: string, key: string, source: string): Promise<Query> {
+  const cacheKey = `${grammar}:${key}`;
+  const cached = queryCache.get(cacheKey);
+  if (cached) return cached;
+  const language = await getLanguage(grammar);
+  const query = new Query(language, source);
+  queryCache.set(cacheKey, query);
+  return query;
+}
+
+function collapse(text: string | null | undefined): string {
+  return (text ?? '').replace(/\s+/g, ' ').trim();
+}
+
+/** Pull one capture out of a match. Returns null when the capture is optional. */
+function captured(match: QueryMatch, name: string): Node | null {
+  for (const capture of match.captures) {
+    if (capture.name === name) return capture.node;
+  }
+  return null;
+}
+
+/**
+ * Argument lists contain punctuation as well as expressions. `namedChildren`
+ * gives us only the meaningful ones - see the "named vs anonymous" note in
+ * astDump.ts.
+ */
+function argumentNodes(argsNode: Node | null): Node[] {
+  if (!argsNode) return [];
+  return argsNode.namedChildren
+    .filter((n): n is Node => n !== null)
+    // PHP wraps every argument in an `argument` node, so the expression a rule
+    // wants to examine sits one level down. Without this unwrap every PHP
+    // argument reads as the node type `argument` and no content check matches.
+    .map((n) => (n.type === 'argument' ? (n.namedChildren.find((c) => c !== null) ?? n) : n));
+}
+
+/**
+ * Go writes `a, b := f(), g()` - both sides are lists. When the lists line up
+ * we pair them; when they don't (a multi-return call) we bind every name to the
+ * single value node, which is honest enough for signature matching and is
+ * flagged as a known imprecision in the Go coverage notes.
+ */
+function goPairs(target: Node, value: Node): Array<{ t: Node; v: Node }> {
+  const targets = target.type === 'expression_list' ? target.namedChildren : [target];
+  const values = value.type === 'expression_list' ? value.namedChildren : [value];
+  const pairs: Array<{ t: Node; v: Node }> = [];
+  for (let i = 0; i < targets.length; i++) {
+    const t = targets[i];
+    if (!t) continue;
+    const v = values.length === targets.length ? values[i] : values[0];
+    if (v) pairs.push({ t, v });
+  }
+  return pairs;
+}
+
+/** Last dotted segment: "config.db.password" -> "password". */
+function lastSegment(text: string): string {
+  const cleaned = text.replace(/\[["']?|["']?\]/g, '.').replace(/\.+$/, '');
+  const parts = cleaned.split('.');
+  return (parts[parts.length - 1] ?? cleaned).trim();
+}
+
+/**
+ * Lower one parsed file into the flat list of shapes rules will see.
+ * Order is source order, which makes reports read top-to-bottom like the file.
+ */
+export async function extractShapes(file: ParsedFile, language: LanguageId): Promise<Shape[]> {
+  const shapes: Shape[] = [];
+  const seen = new Set<number>();
+
+  const callQuery = await compile(file.grammar, 'calls', CALL_QUERIES[language]);
+  for (const match of callQuery.matches(file.root)) {
+    const call = captured(match, 'call');
+    const method = captured(match, 'method');
+    if (!call || !method) continue;
+    // Java's two method_invocation patterns overlap (object is optional), so the
+    // same node can match twice. Dedupe by node id.
+    if (seen.has(call.id)) continue;
+    seen.add(call.id);
+
+    const receiver = captured(match, 'receiver');
+    const receiverText = collapse(receiver?.text);
+    const calleeName = collapse(method.text);
+    // Java's `method_invocation` has no `function` field - the receiver and the
+    // name are separate children - so we rebuild "receiver.name" by hand there.
+    const functionNode = call.childForFieldName('function') ?? call.childForFieldName('constructor');
+    const site: CallSite = {
+      kind: 'call',
+      node: call,
+      calleeName,
+      receiverText,
+      calleeText: collapse(
+        functionNode?.text ?? (receiverText ? `${receiverText}.${calleeName}` : calleeName),
+      ),
+      args: argumentNodes(captured(match, 'args')),
+    };
+    shapes.push(site);
+  }
+
+  const assignQuery = await compile(file.grammar, 'assignments', ASSIGNMENT_QUERIES[language]);
+  for (const match of assignQuery.matches(file.root)) {
+    const node = captured(match, 'assign');
+    const target = captured(match, 'target');
+    const value = captured(match, 'value');
+    if (!node || !target || !value) continue;
+    if (seen.has(node.id)) continue;
+    seen.add(node.id);
+
+    const pairs = language === 'go' ? goPairs(target, value) : [{ t: target, v: value }];
+    for (const pair of pairs) {
+      const targetText = collapse(pair.t.text);
+      shapes.push({
+        kind: 'assignment',
+        node,
+        targetText,
+        targetName: lastSegment(targetText).replace(/^["']|["']$/g, ''),
+        value: pair.v,
+      } satisfies Assignment);
+    }
+  }
+
+  shapes.sort((a, b) => a.node.startIndex - b.node.startIndex);
+  return shapes;
+}
+
+/** Exposed so a self-check can compile every query against every grammar. */
+export const SHAPE_QUERY_SOURCES = {
+  calls: CALL_QUERIES,
+  assignments: ASSIGNMENT_QUERIES,
+} as const;

package/src/engine/walk.ts

@@ -0,0 +1,196 @@
+/**
+ * FILE DISCOVERY
+ *
+ * Turning "scan this path" into "here is the list of files". Boring code, but
+ * two decisions in it matter:
+ *
+ * 1. We skip dependency and build directories. Scanning node_modules finds
+ *    thousands of "vulnerabilities" in other people's code that you cannot fix
+ *    and did not write. Dependency risk is a real and separate problem
+ *    (Software Composition Analysis) and it deserves its own tool, not noise
+ *    inside this one.
+ *
+ * 2. We record the extensions we skipped. That list feeds the coverage report,
+ *    so a scan of a repo that is 70% Ruby says "we did not look at 412 .rb
+ *    files" instead of quietly reporting "no issues found".
+ */
+
+import { readdir, stat } from 'node:fs/promises';
+import path from 'node:path';
+import { runtimeFileKind } from '../platform/runtime-version.js';
+
+const IGNORED_DIRECTORIES = new Set([
+  '.git', '.hg', '.svn',
+  'node_modules', 'bower_components', 'jspm_packages',
+  'dist', 'build', 'out', 'target', 'bin', 'obj',
+  'vendor', 'third_party',
+  '.venv', 'venv', 'env', '__pycache__', '.tox', 'site-packages', '.mypy_cache',
+  '.next', '.nuxt', '.svelte-kit', '.turbo', '.parcel-cache',
+  'coverage', '.nyc_output', '.gradle', '.m2',
+  '.idea', '.vscode', '.cache', '.pytest_cache',
+]);
+
+/** 2 MB. Anything larger is almost always generated, minified or a data blob. */
+const MAX_FILE_BYTES = 2 * 1024 * 1024;
+
+export interface WalkResult {
+  readonly files: readonly string[];
+  /** Extension -> how many files with it we passed over. Feeds the coverage report. */
+  readonly skippedByExtension: ReadonlyMap<string, number>;
+  readonly oversizedFiles: readonly string[];
+  /** How many files --exclude removed. Printed, never silent. */
+  readonly excludedCount: number;
+  /**
+   * Directories --exclude pruned. Counted separately because we stop walking
+   * them, so we honestly do not know how many files were inside. Saying
+   * "0 files excluded" after skipping a whole tree would be its own small lie.
+   */
+  readonly excludedDirectories: readonly string[];
+  /**
+   * Files that declare which PHP the project runs on - Dockerfiles, compose
+   * files, composer.json, `.php-version`. Not source code, so not in `files`;
+   * the runtime version check reads them (platform/runtime-version.ts). The
+   * same directories are skipped as for code, so a dependency's composer.json
+   * under vendor/ is never mistaken for the project's own.
+   */
+  readonly runtimeFiles: readonly string[];
+}
+
+export interface WalkOptions {
+  /** Extensions we are able to parse. Everything else is counted and skipped. */
+  readonly scannable: readonly string[];
+  readonly maxBytes?: number;
+  /** Glob-ish patterns from --exclude. */
+  readonly exclude?: readonly string[];
+}
+
+/**
+ * Turn a glob into a regular expression.
+ *
+ * Supported, deliberately a small subset:
+ *   **   any number of path segments
+ *   *    anything except a path separator
+ *   ?    one character
+ * A pattern with no wildcards and no separator (e.g. `vendor`) matches any path
+ * SEGMENT with that name, which is what people mean by `--exclude=vendor`.
+ *
+ * Everything is matched against the path with forward slashes, so the same
+ * pattern works on Windows and Unix.
+ */
+export function globToRegExp(pattern: string): RegExp {
+  const cleaned = pattern.trim().replace(/\\/g, '/').replace(/\/+$/, '');
+  const bare = !/[*?/]/.test(cleaned);
+
+  const escaped = cleaned
+    .replace(/[.+^${}()|[\]\\]/g, '\\$&')
+    .replace(/\*\*/g, '\u0000')
+    .replace(/\*/g, '[^/]*')
+    .replace(/\?/g, '[^/]')
+    .replace(/\u0000/g, '.*');
+
+  // `vendor` -> matches the segment anywhere in the path.
+  if (bare) return new RegExp(`(^|/)${escaped}(/|$)`);
+  // `src/**/test.js` or `*.spec.ts` -> match anywhere the shape fits.
+  return new RegExp(`(^|/)${escaped}$|^${escaped}$`);
+}
+
+export async function collectFiles(target: string, options: WalkOptions): Promise<WalkResult> {
+  const scannable = new Set(options.scannable.map((e) => e.toLowerCase()));
+  const maxBytes = options.maxBytes ?? MAX_FILE_BYTES;
+  const excludePatterns = (options.exclude ?? []).map(globToRegExp);
+  let excludedCount = 0;
+  const excludedDirectories: string[] = [];
+
+  /** Matched against the path relative to the scan target, slashes normalised. */
+  const isExcluded = (fullPath: string): boolean => {
+    if (excludePatterns.length === 0) return false;
+    const relative = path.relative(target, fullPath).split(path.sep).join('/');
+    return excludePatterns.some((pattern) => pattern.test(relative));
+  };
+
+  const files: string[] = [];
+  const skipped = new Map<string, number>();
+  const oversized: string[] = [];
+  const runtimeFiles: string[] = [];
+
+  const info = await stat(target);
+  if (info.isFile()) {
+    // An explicitly named file is always scanned - if the user pointed at it,
+    // they meant it, even if it is huge or in node_modules.
+    files.push(target);
+    return {
+      files,
+      skippedByExtension: skipped,
+      oversizedFiles: oversized,
+      excludedCount,
+      excludedDirectories,
+      runtimeFiles: runtimeFileKind(path.basename(target)) ? [target] : [],
+    };
+  }
+
+  const queue: string[] = [target];
+  while (queue.length > 0) {
+    const dir = queue.pop();
+    if (!dir) break;
+
+    let entries;
+    try {
+      entries = await readdir(dir, { withFileTypes: true });
+    } catch {
+      continue; // unreadable directory - permissions, broken symlink, race
+    }
+
+    for (const entry of entries) {
+      const full = path.join(dir, entry.name);
+
+      if (entry.isDirectory()) {
+        if (IGNORED_DIRECTORIES.has(entry.name)) continue;
+        if (entry.name.startsWith('.') && entry.name !== '.') continue;
+        // Pruning an excluded directory here saves walking its whole subtree.
+        if (isExcluded(full)) {
+          excludedDirectories.push(path.relative(target, full).split(path.sep).join('/'));
+          continue;
+        }
+        queue.push(full);
+        continue;
+      }
+      if (!entry.isFile()) continue;
+
+      if (isExcluded(full)) {
+        excludedCount++;
+        continue;
+      }
+
+      if (runtimeFileKind(entry.name)) runtimeFiles.push(full);
+
+      const ext = path.extname(entry.name).toLowerCase();
+      if (!scannable.has(ext)) {
+        if (ext) skipped.set(ext, (skipped.get(ext) ?? 0) + 1);
+        continue;
+      }
+
+      try {
+        const fileInfo = await stat(full);
+        if (fileInfo.size > maxBytes) {
+          oversized.push(full);
+          continue;
+        }
+      } catch {
+        continue;
+      }
+
+      files.push(full);
+    }
+  }
+
+  files.sort();
+  excludedDirectories.sort();
+  return {
+    files,
+    skippedByExtension: skipped,
+    oversizedFiles: oversized,
+    excludedCount,
+    excludedDirectories,
+    runtimeFiles,
+  };
+}

package/src/parse/astDump.ts

@@ -0,0 +1,85 @@
+/**
+ * A human-readable printer for syntax trees.
+ *
+ * This exists purely so you can SEE what the parser sees. Run:
+ *     npm run ast -- somefile.js
+ * and compare the output to the source. Once the shape of the tree stops
+ * looking mysterious, writing detection rules becomes ordinary programming.
+ *
+ * Two concepts show up in the output:
+ *
+ *   NAMED vs ANONYMOUS nodes. `db.query(x)` contains the parentheses and the
+ *   dot as nodes too, but they carry no meaning of their own - tree-sitter calls
+ *   those "anonymous". By default we hide them, because rules almost never care.
+ *
+ *   FIELDS. Some children have a role, not just a position. In
+ *   `binary_expression`, the children are labelled `left`, `operator`, `right`.
+ *   Fields are how a rule says "the thing being called" instead of "child #0",
+ *   which is what makes rules survive grammar updates.
+ */
+
+import type { Node } from 'web-tree-sitter';
+
+export interface AstDumpOptions {
+  /** Show punctuation/keyword nodes too. Default false. */
+  readonly includeAnonymous?: boolean;
+  /** Stop after this depth. Default 12. */
+  readonly maxDepth?: number;
+  /** Stop after this many lines, so a huge file doesn't flood the terminal. */
+  readonly maxLines?: number;
+  /** Show the source text of leaf nodes. Default true. */
+  readonly showText?: boolean;
+}
+
+export function dumpAst(root: Node, options: AstDumpOptions = {}): string {
+  const includeAnonymous = options.includeAnonymous ?? false;
+  const maxDepth = options.maxDepth ?? 12;
+  const maxLines = options.maxLines ?? 400;
+  const showText = options.showText ?? true;
+
+  const lines: string[] = [];
+
+  const walk = (node: Node, depth: number, fieldName: string | null, prefix: string, isLast: boolean): void => {
+    if (lines.length >= maxLines) return;
+
+    const connector = depth === 0 ? '' : isLast ? '└─ ' : '├─ ';
+    const label = fieldName ? `${fieldName}: ${node.type}` : node.type;
+    const position = `[${node.startPosition.row + 1}:${node.startPosition.column + 1}]`;
+
+    // Show source text only for leaves - printing it for a whole function is noise.
+    let text = '';
+    if (showText && node.namedChildCount === 0) {
+      const raw = (node.text ?? '').replace(/\s+/g, ' ').trim();
+      if (raw.length > 0) text = `  "${raw.length > 48 ? `${raw.slice(0, 47)}…` : raw}"`;
+    }
+
+    lines.push(`${prefix}${connector}${label} ${position}${text}`);
+
+    if (depth >= maxDepth) {
+      if (node.namedChildCount > 0) {
+        lines.push(`${prefix}${isLast ? '   ' : '│  '}└─ … (${node.namedChildCount} more, depth limit)`);
+      }
+      return;
+    }
+
+    const children: Array<{ node: Node; field: string | null }> = [];
+    for (let i = 0; i < node.childCount; i++) {
+      const child = node.child(i);
+      if (!child) continue;
+      if (!includeAnonymous && !child.isNamed) continue;
+      children.push({ node: child, field: node.fieldNameForChild(i) });
+    }
+
+    const childPrefix = depth === 0 ? '' : prefix + (isLast ? '   ' : '│  ');
+    children.forEach((child, index) => {
+      walk(child.node, depth + 1, child.field, childPrefix, index === children.length - 1);
+    });
+  };
+
+  walk(root, 0, null, '', true);
+
+  if (lines.length >= maxLines) {
+    lines.push(`… output truncated at ${maxLines} lines (use --max-lines to raise)`);
+  }
+  return lines.join('\n');
+}