@remnic/coding-graph 9.3.759

package/src/engine/fixtures.ts

@@ -0,0 +1,364 @@
+/**
+ * Synthetic fixture source code for each tier-1 language.
+ *
+ * Public-repo policy (#1551): every fixture is synthetic — never real user
+ * code. Each fixture is small enough that the expected IR can be reasoned
+ * about and asserted against without magic numbers.
+ */
+import type { CodingGraphLanguage } from "@remnic/core";
+
+export interface Fixture {
+  /** Repository-relative path (used for language sniffing + IR output). */
+  readonly path: string;
+  /** Synthetic source code (UTF-8). */
+  readonly code: string;
+}
+
+export const FIXTURES: Record<CodingGraphLanguage, Fixture> = {
+  typescript: {
+    path: "src/app.ts",
+    code: [
+      'import { Router } from "express";',
+      "import type { Config } from './types';",
+      "",
+      "export interface AppOptions {",
+      "  port: number;",
+      "}",
+      "",
+      "export type Mode = 'dev' | 'prod';",
+      "",
+      "export class Server {",
+      "  private router: Router;",
+      "",
+      "  constructor(router: Router) {",
+      "    this.router = router;",
+      "  }",
+      "",
+      "  public start(): void {",
+      "    this.router.get('/', () => {});",
+      "  }",
+      "}",
+      "",
+      "export function createServer(opts: AppOptions): Server {",
+      "  const r = new Router();",
+      "  return new Server(r);",
+      "}",
+      "",
+      "const app = createServer({ port: 3000 });",
+    ].join("\n"),
+  },
+
+  tsx: {
+    path: "src/component.tsx",
+    code: [
+      'import React, { useState } from "react";',
+      "",
+      "export interface ButtonProps {",
+      "  label: string;",
+      "  onClick: () => void;",
+      "}",
+      "",
+      "export function Button(props: ButtonProps) {",
+      "  const [clicked, setClicked] = useState(false);",
+      "  return null;",
+      "}",
+      "",
+      "export const Container = () => {",
+      "  return null;",
+      "};",
+    ].join("\n"),
+  },
+
+  javascript: {
+    path: "lib/router.js",
+    code: [
+      'const express = require("express");',
+      "",
+      "function createRouter() {",
+      "  const router = express.Router();",
+      "  router.get('/health', function healthCheck(req, res) {});",
+      "  return router;",
+      "}",
+      "",
+      "class App {",
+      "  constructor() {",
+      "    this.router = createRouter();",
+      "  }",
+      "}",
+      "",
+      "module.exports = { App, createRouter };",
+    ].join("\n"),
+  },
+
+  python: {
+    path: "app/models.py",
+    code: [
+      "from typing import List",
+      "from dataclasses import dataclass",
+      "",
+      "class User:",
+      "    def __init__(self, name):",
+      "        self.name = name",
+      "",
+      "    def greet(self):",
+      "        return format_name(self.name)",
+      "",
+      "def format_name(name):",
+      "    return name.strip()",
+      "",
+      "def main():",
+      "    user = User('alice')",
+      "    print(user.greet())",
+    ].join("\n"),
+  },
+
+  go: {
+    path: "cmd/server/main.go",
+    code: [
+      "package main",
+      "",
+      'import "fmt"',
+      "",
+      "type Server struct {",
+      "    Addr string",
+      "}",
+      "",
+      "func main() {",
+      '    s := Server{Addr: ":8080"}',
+      "    fmt.Println(s.Addr)",
+      "}",
+      "",
+      "func (s *Server) Start() error {",
+      "    return nil",
+      "}",
+    ].join("\n"),
+  },
+
+  rust: {
+    path: "src/lib.rs",
+    code: [
+      "use std::collections::HashMap;",
+      "",
+      "pub fn add(a: i32, b: i32) -> i32 {",
+      "    a + b",
+      "}",
+      "",
+      "pub struct Config {",
+      "    pub port: u16,",
+      "}",
+      "",
+      "impl Config {",
+      "    pub fn new() -> Self {",
+      "        Self { port: 8080 }",
+      "    }",
+      "}",
+      "",
+      "pub trait Service {",
+      "    fn run(&self) -> bool;",
+      "}",
+    ].join("\n"),
+  },
+
+  java: {
+    path: "src/main/Server.java",
+    code: [
+      "package com.example;",
+      "",
+      "import java.util.List;",
+      "",
+      "public class Server {",
+      "    private String name;",
+      "",
+      "    public void start() {",
+      "        System.out.println(name);",
+      "    }",
+      "",
+      "    protected static int compute() {",
+      "        return 0;",
+      "    }",
+      "}",
+      "",
+      "interface Handler {",
+      "    void handle();",
+      "}",
+    ].join("\n"),
+  },
+
+  c: {
+    path: "src/main.c",
+    code: [
+      "#include <stdio.h>",
+      "#include <string.h>",
+      "",
+      "typedef struct {",
+      "    int x;",
+      "    int y;",
+      "} Point;",
+      "",
+      "int add(int a, int b) {",
+      "    return a + b;",
+      "}",
+      "",
+      "void main_loop(void) {",
+      "    int sum = add(1, 2);",
+      "    printf(\"%d\\n\", sum);",
+      "}",
+    ].join("\n"),
+  },
+
+  cpp: {
+    path: "src/engine.cpp",
+    code: [
+      "#include <vector>",
+      "#include <string>",
+      "",
+      "namespace engine {",
+      "",
+      "class Widget {",
+      "public:",
+      "    void render();",
+      "    int getWidth() { return width_; }",
+      "private:",
+      "    int width_;",
+      "};",
+      "",
+      "int compute_total(int count) {",
+      "    return count * 2;",
+      "}",
+      "",
+      "}  // namespace engine",
+    ].join("\n"),
+  },
+
+  csharp: {
+    path: "src/Server.cs",
+    code: [
+      "using System;",
+      "using System.Collections.Generic;",
+      "",
+      "namespace MyApp {",
+      "    public class Server {",
+      "        public void Start() {",
+      "            Console.WriteLine(\"started\");",
+      "        }",
+      "        private static int Compute() {",
+      "            return 42;",
+      "        }",
+      "    }",
+      "",
+      "    interface IService {",
+      "        void Run();",
+      "    }",
+      "}",
+    ].join("\n"),
+  },
+
+  ruby: {
+    path: "lib/app.rb",
+    code: [
+      'require "json"',
+      "",
+      "class User",
+      "  def initialize(name)",
+      "    @name = name",
+      "  end",
+      "",
+      "  def greet",
+      "    format_name(@name)",
+      "  end",
+      "end",
+      "",
+      "module Auth",
+      "  def login",
+      "    true",
+      "  end",
+      "end",
+    ].join("\n"),
+  },
+
+  php: {
+    path: "src/Order.php",
+    code: [
+      "<?php",
+      "namespace App;",
+      "",
+      "use Lib\\Client;",
+      "",
+      "class Order {",
+      "    public function total(): float {",
+      "        return 0.0;",
+      "    }",
+      "    private function validate(): bool {",
+      "        return true;",
+      "    }",
+      "}",
+      "",
+      "function helper(): void {",
+      "    $order = new Order();",
+      "    $order->total();",
+      "}",
+    ].join("\n"),
+  },
+
+  kotlin: {
+    path: "src/main/kotlin/Server.kt",
+    code: [
+      "package com.example",
+      "",
+      "import kotlin.collections.*",
+      "",
+      "class Server(val port: Int) {",
+      "    fun start() {",
+      "        println(port)",
+      "    }",
+      "}",
+      "",
+      "object Singleton {",
+      "    val count = 0",
+      "}",
+      "",
+      "fun create(): Server {",
+      "    return Server(8080)",
+      "}",
+    ].join("\n"),
+  },
+
+  swift: {
+    path: "Sources/App.swift",
+    code: [
+      "import Foundation",
+      "",
+      "class View {",
+      "    func render() {",
+      "        print(\"rendering\")",
+      "    }",
+      "}",
+      "",
+      "protocol Drawable {",
+      "    func draw()",
+      "}",
+      "",
+      "func main() {",
+      "    let view = View()",
+      "    view.render()",
+      "}",
+    ].join("\n"),
+  },
+
+  bash: {
+    path: "scripts/deploy.sh",
+    code: [
+      "#!/bin/bash",
+      "",
+      "function deploy() {",
+      "    echo \"deploying\"",
+      "    rsync -avz dist/ server:/app/",
+      "}",
+      "",
+      "function rollback() {",
+      "    echo \"rolling back\"",
+      "    return 0",
+      "}",
+    ].join("\n"),
+  },
+};

package/src/engine/language-sniff.ts

@@ -0,0 +1,56 @@
+/**
+ * Language sniffing — map a file path to a tier-1 CodingGraphLanguage by
+ * its extension. Used when `ParseFileInput.language` is omitted.
+ *
+ * If the path doesn't match any tier-1 extension, returns `null` and the
+ * engine emits a `parse_failed` result (rule 44 — no silent skip).
+ */
+import { TIER_1_LANGUAGES, type CodingGraphLanguage } from "@remnic/core";
+
+/** Extension (lowercase, dot-prefixed) → language. Static table. */
+const EXTENSION_MAP: Record<string, CodingGraphLanguage> = {
+  ".ts": "typescript",
+  ".tsx": "tsx",
+  ".mts": "typescript",
+  ".cts": "typescript",
+  ".js": "javascript",
+  ".jsx": "javascript",
+  ".mjs": "javascript",
+  ".cjs": "javascript",
+  ".py": "python",
+  ".pyi": "python",
+  ".go": "go",
+  ".rs": "rust",
+  ".java": "java",
+  ".c": "c",
+  ".h": "c",
+  ".cpp": "cpp",
+  ".cc": "cpp",
+  ".cxx": "cpp",
+  ".hpp": "cpp",
+  ".hxx": "cpp",
+  ".cs": "csharp",
+  ".rb": "ruby",
+  ".php": "php",
+  ".kt": "kotlin",
+  ".kts": "kotlin",
+  ".swift": "swift",
+  ".sh": "bash",
+  ".bash": "bash",
+};
+
+/**
+ * Sniff the tier-1 language from a file path's extension.
+ * Returns `null` if the extension is not recognized.
+ */
+export function sniffLanguage(filePath: string): CodingGraphLanguage | null {
+  const dot = filePath.lastIndexOf(".");
+  if (dot < 0) return null;
+  const ext = filePath.slice(dot).toLowerCase();
+  return EXTENSION_MAP[ext] ?? null;
+}
+
+/** Return true if `lang` is a valid tier-1 language. */
+export function isTier1Language(lang: string): lang is CodingGraphLanguage {
+  return (TIER_1_LANGUAGES as readonly string[]).includes(lang);
+}

package/src/engine/parser-backend.ts

@@ -0,0 +1,206 @@
+/**
+ * ParserBackend — the abstraction layer between the tree-sitter parsing
+ * engine and the per-language extractors.
+ *
+ * Design rationale (issue #1551): everything goes through this interface so
+ * that a native `node-tree-sitter` backend (or the external C binary via the
+ * subprocess provider) can slot in later without touching extractors. The WASM
+ * backend (`WasmTreeSitterBackend`) is the default; it costs ~2–3× parse speed
+ * vs native but has no per-platform build toolchain — the class of pain
+ * documented in #1518 and #1538.
+ *
+ * Rule 11 (no module-level caches): every parser instance, grammar cache, and
+ * initialization flag lives on the *engine instance*, never at module scope.
+ * This means two engines in the same process have fully isolated state.
+ */
+import { Parser, Language, type Node as TSNode, type Tree as TSTree, type Language as TSLanguage } from "web-tree-sitter";
+import { fileURLToPath } from "node:url";
+import path from "node:path";
+import { statSync } from "node:fs";
+import type { CodingGraphLanguage } from "@remnic/core";
+
+/** Re-export tree-sitter types so extractors don't depend on web-tree-sitter directly. */
+export type { TSNode, TSTree, TSLanguage };
+
+/**
+ * The opaque interface a backend exposes to extractors. Each method is keyed
+ * per-instance so the engine lifecycle (`dispose()`) can clean everything up.
+ */
+export interface ParserBackend {
+  /** Initialize the backend runtime (e.g. load the WASM core). Idempotent per instance. */
+  init(): Promise<void>;
+  /** Ensure the grammar for `lang` is loaded and a parser is ready. Idempotent per instance. */
+  ensureLanguage(lang: CodingGraphLanguage): Promise<void>;
+  /**
+   * Parse `content` (a UTF-8 string) using the grammar for `lang`. Returns the
+   * tree-sitter Tree on success, or `null` if the grammar is not loaded / the
+   * parser cannot produce a tree. Node offsets are UTF-16 code-unit offsets;
+   * callers must convert to UTF-8 byte offsets for multibyte content (#1659).
+   */
+  parse(lang: CodingGraphLanguage, content: string): TSTree | null;
+  /** Return the loaded Language for `lang`, or null if not loaded. */
+  getLanguage(lang: CodingGraphLanguage): TSLanguage | null;
+  /** Release all parsers, grammars, and the WASM runtime. Safe to call multiple times. */
+  dispose(): Promise<void>;
+}
+
+// ---------------------------------------------------------------------------
+// Grammar directory resolution.
+//
+// The .wasm grammar files ship inside the package at `grammars/`. Their
+// location relative to the running module varies by context:
+//   - tsx source run (src/engine/):  ../../grammars
+//   - tsup dist (dist/):              ../grammars
+//   - published package (dist/):      ../grammars
+// We probe candidates and pick the first that exists. This avoids hard-coding
+// a depth that breaks under different bundler layouts.
+// ---------------------------------------------------------------------------
+
+function resolveGrammarDir(): string {
+  const here = path.dirname(fileURLToPath(import.meta.url));
+  const candidates = [
+    path.join(here, "..", "grammars"), // dist/ → ../grammars
+    path.join(here, "..", "..", "grammars"), // src/engine/ → ../../grammars
+    path.join(here, "grammars"), // flat layout fallback
+  ];
+  for (const candidate of candidates) {
+    try {
+      if (statSync(candidate).isDirectory()) return candidate;
+    } catch {
+      // try next candidate
+    }
+  }
+  throw new Error(
+    `coding-graph: could not locate the grammars/ directory from ${here}. ` +
+      `Expected one of: ${candidates.join(", ")}`,
+  );
+}
+
+/**
+ * web-tree-sitter 0.25 WASM backend.
+ *
+ * Per rule 11, all mutable state (init flag, parser instance, grammar cache)
+ * lives on the instance — never at module scope. The backend is constructed by
+ * `createCodingGraphEngine` and disposed alongside the engine.
+ */
+export class WasmTreeSitterBackend implements ParserBackend {
+  private initialized = false;
+  private initializing: Promise<void> | null = null;
+  private parser: Parser | null = null;
+  private readonly languages = new Map<CodingGraphLanguage, Language>();
+  private readonly loadingLanguages = new Map<CodingGraphLanguage, Promise<void>>();
+  private grammarDir = "";
+  private grammarDirResolved = false;
+  private readonly grammarDirHint: string | null;
+
+  constructor(grammarDir?: string) {
+    // Defer resolveGrammarDir() to first use so that a missing grammars/
+    // directory surfaces as a per-file parse_failed (rule 44) rather than
+    // a constructor exception that bricks createCodingGraphEngine.
+    this.grammarDirHint = grammarDir ?? null;
+  }
+
+  private getGrammarDir(): string {
+    if (this.grammarDirResolved) return this.grammarDir;
+    this.grammarDir = this.grammarDirHint ?? resolveGrammarDir();
+    this.grammarDirResolved = true;
+    return this.grammarDir;
+  }
+
+  async init(): Promise<void> {
+    if (this.initialized) return;
+    if (this.initializing) return this.initializing;
+    this.initializing = (async () => {
+      try {
+        await Parser.init();
+        this.parser = new Parser();
+        this.initialized = true;
+      } finally {
+        // Clear the in-flight promise on BOTH success and failure. Without
+        // this, a transient Parser.init() rejection leaves the rejected
+        // promise cached in `initializing`, so every subsequent call
+        // re-awaits the same failure instead of retrying.
+        this.initializing = null;
+      }
+    })();
+    return this.initializing;
+  }
+
+  async ensureLanguage(lang: CodingGraphLanguage): Promise<void> {
+    await this.init();
+    if (this.languages.has(lang)) return;
+    if (this.loadingLanguages.has(lang)) return this.loadingLanguages.get(lang)!;
+    const p = (async () => {
+      try {
+        const wasmPath = path.join(this.getGrammarDir(), grammarFileName(lang));
+        const language = await Language.load(wasmPath);
+        this.languages.set(lang, language);
+      } finally {
+        // Drop the in-flight promise on BOTH success and failure. Without
+        // this, a transient Language.load() rejection leaves the rejected
+        // promise cached in `loadingLanguages`, permanently bricking that
+        // language until dispose() — the next call would re-await the same
+        // failure instead of retrying the load.
+        this.loadingLanguages.delete(lang);
+      }
+    })();
+    this.loadingLanguages.set(lang, p);
+    return p;
+  }
+
+  parse(lang: CodingGraphLanguage, content: string): TSTree | null {
+    if (!this.parser) return null;
+    const language = this.languages.get(lang);
+    if (!language) return null;
+    this.parser.setLanguage(language);
+    return this.parser.parse(content);
+  }
+
+  /** Return the loaded Language object for `lang`, or null if not loaded. */
+  getLanguage(lang: CodingGraphLanguage): Language | null {
+    return this.languages.get(lang) ?? null;
+  }
+
+  async dispose(): Promise<void> {
+    if (this.parser) {
+      try {
+        this.parser.delete();
+      } catch {
+        // already deleted — ignore
+      }
+      this.parser = null;
+    }
+    this.languages.clear();
+    this.loadingLanguages.clear();
+    this.initialized = false;
+    this.initializing = null;
+    this.grammarDirResolved = false;
+    this.grammarDir = "";
+  }
+}
+
+/**
+ * Map a tier-1 language to its .wasm grammar filename. The names match the
+ * files shipped by `tree-sitter-wasms` (0.1.13) and the grammars/ directory.
+ * C# uses `c_sharp` because the upstream grammar names C# with an underscore.
+ */
+export function grammarFileName(lang: CodingGraphLanguage): string {
+  const map: Record<CodingGraphLanguage, string> = {
+    typescript: "tree-sitter-typescript.wasm",
+    tsx: "tree-sitter-tsx.wasm",
+    javascript: "tree-sitter-javascript.wasm",
+    python: "tree-sitter-python.wasm",
+    go: "tree-sitter-go.wasm",
+    rust: "tree-sitter-rust.wasm",
+    java: "tree-sitter-java.wasm",
+    c: "tree-sitter-c.wasm",
+    cpp: "tree-sitter-cpp.wasm",
+    csharp: "tree-sitter-c_sharp.wasm",
+    ruby: "tree-sitter-ruby.wasm",
+    php: "tree-sitter-php.wasm",
+    kotlin: "tree-sitter-kotlin.wasm",
+    swift: "tree-sitter-swift.wasm",
+    bash: "tree-sitter-bash.wasm",
+  };
+  return map[lang];
+}

package/src/engine/utf16-offsets.ts

@@ -0,0 +1,68 @@
+/**
+ * UTF-16 code-unit offset → UTF-8 byte offset conversion (issue #1659 item 3).
+ *
+ * web-tree-sitter 0.25 parses strings internally by converting them to UTF-16
+ * (`stringToUTF16` in the WASM glue), so every node's `startIndex`/`endIndex`
+ * is a UTF-16 code-unit offset, NOT a UTF-8 byte offset. For pure-ASCII
+ * content these are identical, but multibyte content (comments, strings, or
+ * identifiers containing non-ASCII characters) produces incorrect spans:
+ * dead-code analysis, navigation, and body extraction all rely on byte
+ * offsets matching the on-disk file.
+ *
+ * This module builds a single lookup array per file (O(n) build, O(1)
+ * lookup) so all offsets in a FileIR can be converted in one pass.
+ */
+
+/**
+ * Build a map from UTF-16 code-unit offset → UTF-8 byte offset for the given
+ * string. `map[i]` is the byte offset of the i-th UTF-16 code unit.
+ * `map[content.length]` is the total byte length.
+ *
+ * Surrogate pairs (U+10000–U+10FFFF) occupy 2 UTF-16 code units but encode to
+ * 4 UTF-8 bytes. The high surrogate's entry points to the start of the pair;
+ * the low surrogate's entry also points there (its byte delta is 0, added by
+ * the high surrogate).
+ */
+export function buildUtf16ToByteOffsetMap(content: string): Uint32Array {
+  const map = new Uint32Array(content.length + 1);
+  let byteOffset = 0;
+  for (let i = 0; i < content.length; i++) {
+    map[i] = byteOffset;
+    const code = content.charCodeAt(i);
+    if (code < 0x80) {
+      byteOffset += 1;
+    } else if (code < 0x800) {
+      byteOffset += 2;
+    } else if (code >= 0xd800 && code <= 0xdbff) {
+      // High surrogate of a surrogate pair. The full code point encodes to
+      // 4 UTF-8 bytes; we charge all 4 here and skip the low surrogate.
+      byteOffset += 4;
+      // Both code units of the pair represent the SAME character, which
+      // starts at map[i] (set at the top of the loop, before the increment).
+      // The low surrogate must map to that SAME start byte — not to
+      // byteOffset (which is now past the pair) — or a span whose start
+      // falls on the low code unit gets an inflated startByte (cursor
+      // #1659 review: 'Surrogate UTF-16 index maps wrong').
+      map[i + 1] = map[i]!;
+      i++;
+    } else if (code >= 0xdc00 && code <= 0xdfff) {
+      // Lone low surrogate (unpaired) — treat as 3-byte UTF-8 replacement.
+      // This path is only hit for malformed input.
+      byteOffset += 3;
+    } else {
+      byteOffset += 3;
+    }
+  }
+  map[content.length] = byteOffset;
+  return map;
+}
+
+/**
+ * Convert a single UTF-16 code-unit offset to a UTF-8 byte offset using a
+ * pre-built map. Returns 0 for negative offsets; clamps to the map length.
+ */
+export function utf16ToByte(map: Uint32Array, utf16Offset: number): number {
+  if (utf16Offset <= 0) return 0;
+  if (utf16Offset >= map.length) return map[map.length - 1];
+  return map[utf16Offset];
+}