@massu/core 1.2.1 → 1.4.0-soak.0

package/src/detect/adapters/python-fastapi.ts

@@ -0,0 +1,223 @@
+// Copyright (c) 2026 Massu. All rights reserved.
+// Licensed under BSL 1.1 - see LICENSE file for details.
+
+/**
+ * Plan 3b — Phase 1: FastAPI AST adapter.
+ *
+ * Extracts:
+ *   - auth_dep: name passed to `Depends(...)` in router files
+ *   - api_prefix_base: first path segment of `APIRouter(prefix="/...")`
+ *   - test_async_pattern: `@pytest.mark.asyncio` (with or without parens)
+ *
+ * Confidence rules:
+ *   - 'high' if the auth dep is found exactly ONCE in routers/ and matches
+ *     known FastAPI signatures.
+ *   - 'medium' if found in non-routers/ paths (e.g., a deps.py module).
+ *   - 'low' if multiple candidate auth deps are found (ambiguous — but still
+ *     emitted so the user can see what was found).
+ *   - 'none' if no `Depends(...)` calls found AND no `APIRouter(prefix=)` —
+ *     adapter doesn't apply, regex fallback takes over.
+ *
+ * Does NOT use regex on file content — only Tree-sitter S-expression queries
+ * compiled via `query-helpers.ts`. Regex would be the regex-fallback path.
+ */
+
+import { Parser } from 'web-tree-sitter';
+import type { CodebaseAdapter, AdapterResult, DetectionSignals, Provenance, SourceFile } from './types.ts';
+import { runQuery, InvalidQueryError } from './query-helpers.ts';
+import { loadGrammar } from './tree-sitter-loader.ts';
+import { isParsableSource, MAX_AST_FILE_BYTES } from './parse-guard.ts';
+
+// ============================================================
+// Tree-sitter S-expression queries
+// ============================================================
+
+/**
+ * Auth dependency: catches `Depends(get_current_user)`, `Depends(require_tier_or_guardian)`,
+ * etc. Anchored on the canonical `Depends` call shape.
+ *
+ * Per the spec doc §3, predicate constraints (#eq?) keep the query from
+ * matching arbitrary `<x>(<y>)` calls.
+ */
+const AUTH_DEP_QUERY = `
+(call
+  function: (identifier) @_callee (#eq? @_callee "Depends")
+  arguments: (argument_list
+    (identifier) @auth_dep))
+`;
+
+/**
+ * APIRouter prefix: `APIRouter(prefix="/api/orders", ...)`. Captures the
+ * string literal so the runner can split off the base segment.
+ */
+const API_PREFIX_QUERY = `
+(call
+  function: (identifier) @_callee (#eq? @_callee "APIRouter")
+  arguments: (argument_list
+    (keyword_argument
+      name: (identifier) @_kw (#eq? @_kw "prefix")
+      value: (string) @prefix_value)))
+`;
+
+/**
+ * `@pytest.mark.asyncio` decorator. Captures the decorator name string for
+ * provenance; the value field is fixed as the canonical form.
+ */
+const PYTEST_ASYNCIO_QUERY = `
+(decorator
+  (attribute
+    object: (attribute
+      object: (identifier) @_pkg (#eq? @_pkg "pytest")
+      attribute: (identifier) @_mark (#eq? @_mark "mark"))
+    attribute: (identifier) @_marker (#eq? @_marker "asyncio"))) @decorator
+`;
+
+// ============================================================
+// Adapter
+// ============================================================
+
+export const pythonFastApiAdapter: CodebaseAdapter = {
+  id: 'python-fastapi',
+  languages: ['python'],
+
+  matches(signals: DetectionSignals): boolean {
+    // Cheap signal-only check. No file IO. Match if:
+    //   1. pyproject.toml mentions fastapi (raw text contains 'fastapi'), OR
+    //   2. project has a routers/ directory (FastAPI convention), OR
+    //   3. project has app/ + python files at top level
+    const pyToml = signals.pyprojectToml as { __raw?: string } | undefined;
+    if (pyToml?.__raw && /\bfastapi\b/i.test(pyToml.__raw)) return true;
+    if (signals.presentDirs.has('routers')) return true;
+    if (signals.presentDirs.has('app') && signals.presentFiles.has('main.py')) return true;
+    return false;
+  },
+
+  async introspect(files: SourceFile[], _rootDir: string): Promise<AdapterResult> {
+    if (files.length === 0) {
+      return { conventions: {}, provenance: [], confidence: 'none' };
+    }
+
+    let language;
+    try {
+      language = await loadGrammar('python');
+    } catch (e) {
+      // Grammar unavailable → adapter returns 'none' so regex fallback takes
+      // over. The runner's stderr line is emitted at the introspector tier.
+      return { conventions: {}, provenance: [], confidence: 'none' };
+    }
+
+    const parser = new Parser();
+    parser.setLanguage(language);
+
+    // Per-field collection: { value -> { fileLine, queryName } }
+    const authDeps = new Map<string, { line: number; file: string }>();
+    const prefixBases = new Map<string, { line: number; file: string }>();
+    const testAsyncPatterns = new Map<string, { line: number; file: string }>();
+
+    try {
+      for (const file of files) {
+        // Phase 3.5 fix: defense-in-depth size + depth gate at adapter
+        // tier (the runner also gates, but adapters may be invoked
+        // directly from tests/CLI).
+        const skip = isParsableSource(file.content, file.size);
+        if (skip) {
+          process.stderr.write(
+            `[massu/ast] WARN: python-fastapi skipping ${file.path}: ${skip.reason} (${skip.detail}). Cap=${MAX_AST_FILE_BYTES}. (Phase 3.5 mitigation)\n`,
+          );
+          continue;
+        }
+        try {
+          // Auth dep
+          for (const hit of runQuery(parser, file.content, AUTH_DEP_QUERY, 'fastapi-auth-dep', file.path)) {
+            const name = hit.captures.auth_dep;
+            if (name && !authDeps.has(name)) {
+              authDeps.set(name, { line: hit.line, file: file.path });
+            }
+          }
+          // API prefix
+          for (const hit of runQuery(parser, file.content, API_PREFIX_QUERY, 'fastapi-api-prefix', file.path)) {
+            const raw = hit.captures.prefix_value;
+            if (!raw) continue;
+            // Strip enclosing quotes (string node text includes them)
+            const literal = raw.replace(/^['"]/, '').replace(/['"]$/, '');
+            const base = extractPrefixBase(literal);
+            if (base && !prefixBases.has(base)) {
+              prefixBases.set(base, { line: hit.line, file: file.path });
+            }
+          }
+          // pytest.mark.asyncio
+          for (const hit of runQuery(parser, file.content, PYTEST_ASYNCIO_QUERY, 'fastapi-pytest-asyncio', file.path)) {
+            const pat = '@pytest.mark.asyncio';
+            if (!testAsyncPatterns.has(pat)) {
+              testAsyncPatterns.set(pat, { line: hit.line, file: file.path });
+            }
+          }
+        } catch (e) {
+          if (e instanceof InvalidQueryError) {
+            // Compile-time failure of OUR query is a developer bug — surface it.
+            throw e;
+          }
+          // Per-file parse error: skip this file, keep going. Tree-sitter is
+          // error-tolerant so this is rare; usually means we got a binary or
+          // a non-Python file mislabeled.
+          continue;
+        }
+      }
+    } finally {
+      try { parser.delete(); } catch { /* ignore */ }
+    }
+
+    // Build result
+    const conventions: Record<string, unknown> = {};
+    const provenance: Provenance[] = [];
+
+    // Auth dep: high if exactly 1, low if >1 (still emit first), none if 0.
+    if (authDeps.size === 1) {
+      const [name, { line, file }] = authDeps.entries().next().value as [string, { line: number; file: string }];
+      conventions.auth_dep = name;
+      provenance.push({ field: 'auth_dep', sourceFile: file, line, query: 'fastapi-auth-dep' });
+    } else if (authDeps.size >= 2) {
+      // Ambiguous — prefer the first-seen (stable order from input file list).
+      const [name, { line, file }] = authDeps.entries().next().value as [string, { line: number; file: string }];
+      conventions.auth_dep = name;
+      provenance.push({ field: 'auth_dep', sourceFile: file, line, query: 'fastapi-auth-dep' });
+    }
+
+    if (prefixBases.size >= 1) {
+      const [base, { line, file }] = prefixBases.entries().next().value as [string, { line: number; file: string }];
+      conventions.api_prefix_base = base;
+      provenance.push({ field: 'api_prefix_base', sourceFile: file, line, query: 'fastapi-api-prefix' });
+    }
+
+    if (testAsyncPatterns.size >= 1) {
+      const [pat, { line, file }] = testAsyncPatterns.entries().next().value as [string, { line: number; file: string }];
+      conventions.test_async_pattern = pat;
+      provenance.push({ field: 'test_async_pattern', sourceFile: file, line, query: 'fastapi-pytest-asyncio' });
+    }
+
+    let confidence: AdapterResult['confidence'];
+    if (Object.keys(conventions).length === 0) {
+      confidence = 'none';
+    } else if (authDeps.size === 1 || (authDeps.size === 0 && prefixBases.size > 0)) {
+      confidence = 'high';
+    } else if (authDeps.size >= 2) {
+      confidence = 'low';
+    } else {
+      confidence = 'medium';
+    }
+
+    return { conventions, provenance, confidence };
+  },
+};
+
+// ============================================================
+// Helpers
+// ============================================================
+
+function extractPrefixBase(prefix: string): string | null {
+  if (!prefix.startsWith('/')) return null;
+  const stripped = prefix.replace(/^\/+/, '');
+  const firstSeg = stripped.split('/')[0];
+  if (!firstSeg) return null;
+  return '/' + firstSeg;
+}

package/src/detect/adapters/query-helpers.ts

@@ -0,0 +1,170 @@
+// Copyright (c) 2026 Massu. All rights reserved.
+// Licensed under BSL 1.1 - see LICENSE file for details.
+
+/**
+ * Plan 3b — Phase 1: Tree-sitter query wrapper.
+ *
+ * Adapters consume the helpers in this file — never the raw `web-tree-sitter`
+ * API. This keeps the surface area minimal and testable.
+ *
+ * Design:
+ *   - `compileQuery` caches compiled `Query` instances per (language, source)
+ *     tuple. Compiling an S-expression is non-trivial; cache hit-rate is
+ *     critical when the same query runs across N sampled files.
+ *   - `runQuery` returns the captures as `{captures, file, line}` records so
+ *     adapters never need to touch raw `Node` objects.
+ *   - `InvalidQueryError` is the typed error thrown when an S-expression is
+ *     malformed; never let a raw `Error` reach the adapter (per audit-iter-5
+ *     fix HH test (b)).
+ */
+
+import { Query, type Language, type Node, type Parser, type QueryMatch } from 'web-tree-sitter';
+
+/**
+ * Thrown when an S-expression query string fails to compile against the
+ * supplied grammar. Carries the original message and the offending source
+ * so adapter authors can debug.
+ */
+export class InvalidQueryError extends Error {
+  public readonly queryName: string;
+  public readonly querySource: string;
+  public readonly cause?: unknown;
+  constructor(queryName: string, querySource: string, cause: unknown) {
+    const causeMsg = cause instanceof Error ? cause.message : String(cause);
+    super(
+      `[query-helpers] Invalid Tree-sitter query "${queryName}": ${causeMsg}\n` +
+        `Query source:\n${querySource}`,
+    );
+    this.name = 'InvalidQueryError';
+    this.queryName = queryName;
+    this.querySource = querySource;
+    this.cause = cause;
+  }
+}
+
+// ============================================================
+// Query compile cache
+// ============================================================
+
+// We key by Language identity (not by name) AND by source string. The Query
+// type from web-tree-sitter is opaque; we store it directly.
+const queryCache = new WeakMap<Language, Map<string, Query>>();
+
+/**
+ * Compile (and cache) an S-expression query against `language`.
+ *
+ * Throws `InvalidQueryError` (NOT raw Error) on malformed S-expressions —
+ * adapters can catch this without losing the typed boundary.
+ *
+ * Cache lookup is O(1) on the (Language, source) tuple via WeakMap+Map.
+ */
+export function compileQuery(
+  language: Language,
+  source: string,
+  queryName: string,
+): Query {
+  let perLang = queryCache.get(language);
+  if (!perLang) {
+    perLang = new Map();
+    queryCache.set(language, perLang);
+  }
+  const cached = perLang.get(source);
+  if (cached) return cached;
+
+  let q: Query;
+  try {
+    q = new Query(language, source);
+  } catch (e) {
+    throw new InvalidQueryError(queryName, source, e);
+  }
+
+  perLang.set(source, q);
+  return q;
+}
+
+// ============================================================
+// Capture extraction
+// ============================================================
+
+export interface RunQueryHit {
+  /**
+   * Capture name → captured text. If the same capture name appears multiple
+   * times in a single match, the LAST occurrence wins (callers usually want
+   * the most-specific one).
+   */
+  captures: Record<string, string>;
+  /** Absolute path to the file being parsed. */
+  file: string;
+  /** 1-based line number of the FIRST capture in the match. */
+  line: number;
+  /** Name of the query (used for provenance). */
+  queryName: string;
+}
+
+/**
+ * Run a compiled query against a parsed tree. Returns a flat list of hits.
+ *
+ * Each match becomes one `RunQueryHit`. The `line` is computed from the
+ * earliest-starting capture in the match (1-based). Note that this helper is
+ * intentionally narrow — it is NOT a general node-walker. Adapters that need
+ * tree traversal should compose multiple queries instead.
+ */
+export function runQuery(
+  parser: Parser,
+  source: string,
+  queryText: string,
+  queryName: string,
+  filePath: string,
+): RunQueryHit[] {
+  const language = parser.language;
+  if (!language) {
+    throw new InvalidQueryError(
+      queryName,
+      queryText,
+      new Error('Parser has no language assigned'),
+    );
+  }
+  const query = compileQuery(language, queryText, queryName);
+
+  const tree = parser.parse(source);
+  if (!tree) return [];
+
+  let matches: QueryMatch[];
+  try {
+    matches = query.matches(tree.rootNode);
+  } catch (e) {
+    // Match-time errors are unusual (compile-time catches most), but we still
+    // wrap to keep the typed-error contract.
+    throw new InvalidQueryError(queryName, queryText, e);
+  }
+
+  const out: RunQueryHit[] = [];
+  for (const match of matches) {
+    if (!match.captures || match.captures.length === 0) continue;
+    const captures: Record<string, string> = {};
+    let earliestLine = Number.POSITIVE_INFINITY;
+    for (const cap of match.captures) {
+      const node: Node = cap.node;
+      captures[cap.name] = node.text;
+      if (node.startPosition.row + 1 < earliestLine) {
+        earliestLine = node.startPosition.row + 1;
+      }
+    }
+    out.push({
+      captures,
+      file: filePath,
+      line: Number.isFinite(earliestLine) ? earliestLine : 1,
+      queryName,
+    });
+  }
+
+  // Per Tree-sitter docs: trees should be deleted to free WASM memory.
+  // Adapters call runQuery once per file so this cleanup is local.
+  try {
+    tree.delete();
+  } catch {
+    /* deletion is best-effort — some test mocks don't implement delete */
+  }
+
+  return out;
+}

package/src/detect/adapters/runner.ts

@@ -0,0 +1,252 @@
+// Copyright (c) 2026 Massu. All rights reserved.
+// Licensed under BSL 1.1 - see LICENSE file for details.
+
+/**
+ * Plan 3b — Phase 1: AST adapter runner.
+ *
+ * Orchestrates: filter adapters via `matches()`, run them, isolate failures
+ * via per-adapter try/catch (audit-iter-5 fix HH test (d)), and merge their
+ * results.
+ *
+ * Confidence merge rule (spec §5):
+ *   - 'high' / 'medium' / 'low' → field is written, with per-field provenance.
+ *   - 'none' → field DROPPED (introspect's regex fallback may then emit it).
+ *
+ * AST-wins rule:
+ *   - When the same conventions key appears in two adapters that BOTH return
+ *     non-'none', the FIRST adapter (by source-list order) wins. This is
+ *     deterministic — adapters are listed in `runner.ts`'s static array,
+ *     never user-provided.
+ */
+
+import { basename, relative } from 'path';
+import type {
+  AdapterResolved,
+  CodebaseAdapter,
+  DetectionSignals,
+  MergedAdapterOutput,
+  Provenance,
+  SourceFile,
+} from './types.ts';
+import { isParsableSource, MAX_AST_FILE_BYTES } from './parse-guard.ts';
+
+export interface RunAdaptersOptions {
+  /**
+   * Optional file sampler — given an adapter and the project root, returns
+   * the SourceFile[] the adapter should consume. If omitted, the runner
+   * passes an empty file list (useful in unit tests where the caller has
+   * already constructed adapters that don't need files).
+   */
+  sampleFiles?: (adapter: CodebaseAdapter, rootDir: string) => Promise<SourceFile[]> | SourceFile[];
+}
+
+/**
+ * Run a static list of adapters against a project root.
+ *
+ * Per-adapter try/catch isolation: a single adapter throwing MUST NOT crash
+ * the runner. The error is captured in `errored[]` and the runner continues.
+ *
+ * @param adapters - Static list of first-party adapters (no user-authored
+ *   adapters at v1 — Plan 3c will add discovery).
+ * @param rootDir - Absolute project root.
+ * @param signals - Pre-built `DetectionSignals` (manifest reads, present
+ *   dirs/files). Adapters consume these read-only.
+ * @param options - Hooks for testing.
+ */
+export async function runAdapters(
+  adapters: CodebaseAdapter[],
+  rootDir: string,
+  signals: DetectionSignals,
+  options: RunAdaptersOptions = {},
+): Promise<MergedAdapterOutput> {
+  const out: MergedAdapterOutput = {
+    byAdapter: {},
+    skipped: [],
+    errored: [],
+  };
+
+  // AST-wins / per-adapter merge:
+  // Each adapter writes to its own `detected.<adapter.id>` namespace, so
+  // global field collisions across adapters can't happen at the conventions
+  // level. The "AST-wins" rule in the spec applies at the introspector tier
+  // (regex fallback only fills fields the adapter returned 'none' for).
+  // Within a single adapter, if `conventions` repeats a key (shouldn't, but
+  // defensively), the first occurrence wins. For multiple adapters with the
+  // same id (shouldn't, but defensively), the first wins.
+
+  for (const adapter of adapters) {
+    if (out.byAdapter[adapter.id] || out.skipped.includes(adapter.id)) {
+      // Duplicate adapter id → skip the second one to preserve first-wins.
+      continue;
+    }
+    let matches: boolean;
+    try {
+      matches = adapter.matches(signals);
+    } catch (e) {
+      out.errored.push({
+        adapterId: adapter.id,
+        error: `matches() threw: ${e instanceof Error ? e.message : String(e)}`,
+      });
+      continue;
+    }
+    if (!matches) {
+      out.skipped.push(adapter.id);
+      continue;
+    }
+
+    let files: SourceFile[];
+    try {
+      files = options.sampleFiles
+        ? await options.sampleFiles(adapter, rootDir)
+        : [];
+    } catch (e) {
+      out.errored.push({
+        adapterId: adapter.id,
+        error: `sampleFiles threw: ${e instanceof Error ? e.message : String(e)}`,
+      });
+      continue;
+    }
+
+    // Phase 3.5 fix: size + depth + control-byte gate. Drop adversarial
+    // inputs BEFORE the adapter sees them — adapters trust this layer.
+    // Files dropped here are logged once per drop so operators see the
+    // signal; the adapter then runs against the surviving subset.
+    const safeFiles: SourceFile[] = [];
+    for (const f of files) {
+      const skip = isParsableSource(f.content, f.size);
+      if (skip) {
+        process.stderr.write(
+          `[massu/ast] WARN: skipping ${f.path} for adapter ${adapter.id}: ${skip.reason} (${skip.detail}). Cap=${MAX_AST_FILE_BYTES} bytes. (Phase 3.5 mitigation)\n`,
+        );
+        continue;
+      }
+      safeFiles.push(f);
+    }
+    files = safeFiles;
+
+    let result;
+    try {
+      result = await adapter.introspect(files, rootDir);
+    } catch (e) {
+      out.errored.push({
+        adapterId: adapter.id,
+        error: `introspect() threw: ${e instanceof Error ? e.message : String(e)}`,
+      });
+      continue;
+    }
+
+    // 'none' confidence drops the entire adapter result. The runner records
+    // that the adapter was attempted (in `byAdapter`) so callers can see it
+    // ran, but with empty conventions. introspect()'s regex fallback then
+    // takes over for the field.
+    if (result.confidence === 'none') {
+      out.byAdapter[adapter.id] = {
+        conventions: {},
+        _provenance: {},
+        confidence: 'none',
+      };
+      continue;
+    }
+
+    // Merge: keep first occurrence of each field (defensive against an
+    // adapter accidentally writing the same field twice).
+    const conventions: Record<string, unknown> = {};
+    const provenanceMap: Record<string, string> = {};
+    for (const [field, value] of Object.entries(result.conventions)) {
+      if (value === null || value === undefined) continue;
+      if (field in conventions) continue;
+      conventions[field] = value;
+    }
+    for (const p of result.provenance) {
+      if (p.field in provenanceMap) continue;
+      provenanceMap[p.field] = formatProvenance(p, rootDir);
+    }
+
+    const resolved: AdapterResolved = {
+      conventions,
+      _provenance: provenanceMap,
+      confidence: result.confidence,
+    };
+    out.byAdapter[adapter.id] = resolved;
+  }
+
+  return out;
+}
+
+function formatProvenance(p: Provenance, rootDir: string): string {
+  const rel = p.sourceFile.startsWith(rootDir + '/')
+    ? relative(rootDir, p.sourceFile)
+    : basename(p.sourceFile);
+  return `${rel}:${p.line} :: ${p.query}`;
+}
+
+// ============================================================
+// Signal builder — used by codebase-introspector to feed the runner
+// ============================================================
+
+import { existsSync, readdirSync, readFileSync, statSync } from 'fs';
+import { join } from 'path';
+
+/**
+ * Build a `DetectionSignals` bundle by reading manifest files at the project
+ * root. Cheap (one-level dir scan + a handful of file reads). Failures
+ * degrade gracefully — a missing manifest just means that field is undefined.
+ */
+export function buildDetectionSignals(rootDir: string): DetectionSignals {
+  const presentDirs = new Set<string>();
+  const presentFiles = new Set<string>();
+  try {
+    for (const entry of readdirSync(rootDir)) {
+      if (entry.startsWith('.')) continue;
+      try {
+        const st = statSync(join(rootDir, entry));
+        if (st.isDirectory()) presentDirs.add(entry);
+        else if (st.isFile()) presentFiles.add(entry);
+      } catch {
+        /* ignore */
+      }
+    }
+  } catch {
+    /* unreadable root → empty signals */
+  }
+
+  return {
+    packageJson: tryReadJson(join(rootDir, 'package.json')),
+    pyprojectToml: tryReadToml(join(rootDir, 'pyproject.toml')),
+    gemfile: tryReadString(join(rootDir, 'Gemfile')),
+    cargoToml: tryReadToml(join(rootDir, 'Cargo.toml')),
+    goMod: tryReadString(join(rootDir, 'go.mod')),
+    presentDirs,
+    presentFiles,
+  };
+}
+
+function tryReadString(path: string): string | undefined {
+  if (!existsSync(path)) return undefined;
+  try {
+    return readFileSync(path, 'utf-8');
+  } catch {
+    return undefined;
+  }
+}
+
+function tryReadJson(path: string): Record<string, unknown> | undefined {
+  const txt = tryReadString(path);
+  if (!txt) return undefined;
+  try {
+    const parsed = JSON.parse(txt);
+    return typeof parsed === 'object' && parsed !== null ? (parsed as Record<string, unknown>) : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+function tryReadToml(path: string): Record<string, unknown> | undefined {
+  const txt = tryReadString(path);
+  if (!txt) return undefined;
+  // Cheap signal-only parse: we just need top-level table presence + keys.
+  // Avoid pulling the full toml parser for this; check `[project]`/`[tool.x]`
+  // headers and treat `tool.poetry.dependencies` etc. as opaque text-search.
+  // Adapters that need structured data can grep `txt` themselves.
+  return { __raw: txt };
+}