@opencodehub/ingestion 0.1.0

package/dist/parse/wasm-fallback.d.ts

@@ -0,0 +1,88 @@
+/**
+ * WASM parser opener (default runtime) + native-availability probe.
+ *
+ * WASM is the default parse runtime as of Node 24 / M5. The native
+ * `tree-sitter` N-API addon is still fully supported and is opt-in via
+ * `OCH_NATIVE_PARSER=1` (or `--native-parser` on the CLI) — useful on
+ * Node 22 developer boxes where native parsing is measurably faster.
+ * Exotic environments (musl-libc Alpine, Cloudflare Workers, sandboxed
+ * Electron renderers, AWS Lambda ARM64 custom runtimes, restricted CI)
+ * can't load `.node` addons at all; on those hosts the default WASM
+ * path Just Works and `isNativeAvailable()` returns false.
+ *
+ * `openWasmParser(lang)` lazily initializes the web-tree-sitter runtime
+ * once per process and resolves the grammar WASM from the installed
+ * `tree-sitter-<lang>` package. Per-grammar cache means repeated calls
+ * are O(1).
+ *
+ * Query execution uses the same unified S-expression bodies from
+ * `unified-queries.ts`; the parse-phase consumer receives byte-
+ * identical ParseCapture output whether the runtime was native or WASM
+ * (asserted by the parity test in `wasm-parity.test.ts`).
+ */
+import type { LanguageId } from "./types.js";
+/**
+ * Returns true when `require('tree-sitter')` succeeds in the current process.
+ * Result is cached — subsequent calls are O(1).
+ *
+ * Call this at worker startup rather than on every parse.
+ */
+export declare function isNativeAvailable(): boolean;
+/**
+ * For tests and diagnostics: reset the cached detection result.
+ */
+export declare function resetNativeAvailabilityCache(): void;
+/**
+ * Minimal shape of what `openWasmParser` returns — enough to run the
+ * same capture loop the native worker implements. Intentionally
+ * decoupled from the `web-tree-sitter` types so test code can stub it.
+ */
+export interface WasmParserHandle {
+    readonly language: LanguageId;
+    /** Parse a source string and return the underlying tree. */
+    parse(source: string): WasmTree;
+    /**
+     * Execute the unified query and return the flat capture list. Callers
+     * translate into `ParseCapture` via the normal coordinate remapping
+     * (1-indexed lines, 0-indexed columns).
+     */
+    runQuery(queryText: string, source: string): readonly WasmCapture[];
+}
+export interface WasmTree {
+    readonly rootNode: WasmNode;
+}
+export interface WasmNode {
+    readonly type: string;
+    readonly text: string;
+    readonly startPosition: {
+        readonly row: number;
+        readonly column: number;
+    };
+    readonly endPosition: {
+        readonly row: number;
+        readonly column: number;
+    };
+}
+export interface WasmCapture {
+    readonly name: string;
+    readonly node: WasmNode;
+}
+/**
+ * Attempt to open a WASM-backed parser for `lang`. Returns `null` when
+ * either the `web-tree-sitter` runtime or the grammar's bundled `.wasm`
+ * could not be resolved — callers log a debug note and skip that file.
+ */
+export declare function openWasmParser(lang: LanguageId): Promise<WasmParserHandle | null>;
+/**
+ * Test hook: clear the per-process WASM parser cache. Never call in
+ * production paths — it would force a re-init of every grammar.
+ */
+export declare function _resetWasmCacheForTests(): void;
+/**
+ * Test hook: expose the grammar-path resolver so unit tests can assert
+ * the two-stage cascade (per-grammar package → tree-sitter-wasms
+ * catalog) resolves kotlin/swift/dart correctly. Not part of the public
+ * API — callers in production paths must go through `openWasmParser`.
+ */
+export declare function _resolveGrammarWasmPathForTests(lang: LanguageId): string | undefined;
+//# sourceMappingURL=wasm-fallback.d.ts.map

package/dist/parse/wasm-fallback.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"wasm-fallback.d.ts","sourceRoot":"","sources":["../../src/parse/wasm-fallback.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAKH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAgB7C;;;;;GAKG;AACH,wBAAgB,iBAAiB,IAAI,OAAO,CAW3C;AAED;;GAEG;AACH,wBAAgB,4BAA4B,IAAI,IAAI,CAEnD;AAMD;;;;GAIG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;IAC9B,4DAA4D;IAC5D,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,QAAQ,CAAC;IAChC;;;;OAIG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,SAAS,WAAW,EAAE,CAAC;CACrE;AAED,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;CAC7B;AAED,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,aAAa,EAAE;QAAE,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAC1E,QAAQ,CAAC,WAAW,EAAE;QAAE,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CACzE;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;CACzB;AAsCD;;;;GAIG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAwCvF;AA+HD;;;GAGG;AACH,wBAAgB,uBAAuB,IAAI,IAAI,CAG9C;AAED;;;;;GAKG;AACH,wBAAgB,+BAA+B,CAAC,IAAI,EAAE,UAAU,GAAG,MAAM,GAAG,SAAS,CAEpF"}

package/dist/parse/wasm-fallback.js

@@ -0,0 +1,258 @@
+/**
+ * WASM parser opener (default runtime) + native-availability probe.
+ *
+ * WASM is the default parse runtime as of Node 24 / M5. The native
+ * `tree-sitter` N-API addon is still fully supported and is opt-in via
+ * `OCH_NATIVE_PARSER=1` (or `--native-parser` on the CLI) — useful on
+ * Node 22 developer boxes where native parsing is measurably faster.
+ * Exotic environments (musl-libc Alpine, Cloudflare Workers, sandboxed
+ * Electron renderers, AWS Lambda ARM64 custom runtimes, restricted CI)
+ * can't load `.node` addons at all; on those hosts the default WASM
+ * path Just Works and `isNativeAvailable()` returns false.
+ *
+ * `openWasmParser(lang)` lazily initializes the web-tree-sitter runtime
+ * once per process and resolves the grammar WASM from the installed
+ * `tree-sitter-<lang>` package. Per-grammar cache means repeated calls
+ * are O(1).
+ *
+ * Query execution uses the same unified S-expression bodies from
+ * `unified-queries.ts`; the parse-phase consumer receives byte-
+ * identical ParseCapture output whether the runtime was native or WASM
+ * (asserted by the parity test in `wasm-parity.test.ts`).
+ */
+import { createRequire } from "node:module";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+const requireFn = createRequire(import.meta.url);
+// Resolve packages/ingestion/vendor/wasms/ relative to this module regardless
+// of whether we're running from src/ (ts-node-style) or dist/ (compiled).
+// `vendor/` lives at the package root, so we walk up from the file's dirname
+// until we find it. Computed once at module load.
+const VENDOR_WASMS_DIR = (() => {
+    const here = path.dirname(fileURLToPath(import.meta.url));
+    // src → <pkg>/src/parse; dist → <pkg>/dist/parse — both 2 levels up
+    return path.resolve(here, "..", "..", "vendor", "wasms");
+})();
+let cached;
+/**
+ * Returns true when `require('tree-sitter')` succeeds in the current process.
+ * Result is cached — subsequent calls are O(1).
+ *
+ * Call this at worker startup rather than on every parse.
+ */
+export function isNativeAvailable() {
+    if (cached !== undefined) {
+        return cached;
+    }
+    try {
+        requireFn("tree-sitter");
+        cached = true;
+    }
+    catch {
+        cached = false;
+    }
+    return cached;
+}
+/**
+ * For tests and diagnostics: reset the cached detection result.
+ */
+export function resetNativeAvailabilityCache() {
+    cached = undefined;
+}
+/**
+ * Per-LanguageId cache of WASM grammar handles. Populated lazily.
+ * `null` entries mean "tried and failed" — we don't retry forever.
+ */
+const wasmCache = new Map();
+let wasmRuntime;
+/**
+ * Attempt to open a WASM-backed parser for `lang`. Returns `null` when
+ * either the `web-tree-sitter` runtime or the grammar's bundled `.wasm`
+ * could not be resolved — callers log a debug note and skip that file.
+ */
+export async function openWasmParser(lang) {
+    const cached = wasmCache.get(lang);
+    if (cached !== undefined)
+        return cached;
+    try {
+        const runtime = await ensureWasmRuntime();
+        if (runtime === undefined) {
+            wasmCache.set(lang, null);
+            return null;
+        }
+        const wasmPath = resolveGrammarWasmPath(lang);
+        if (wasmPath === undefined) {
+            wasmCache.set(lang, null);
+            return null;
+        }
+        const tsLanguage = await runtime.Language.load(wasmPath);
+        const Parser = runtime.Parser;
+        const parser = new Parser();
+        parser.setLanguage(tsLanguage);
+        const handle = {
+            language: lang,
+            parse: (source) => parser.parse(source),
+            runQuery: (queryText, source) => {
+                // Fresh Query per call so state stays clean between bodies.
+                // Query construction is cheap relative to the parse itself.
+                const q = new runtime.Query(tsLanguage, queryText);
+                const tree = parser.parse(source);
+                const out = [];
+                for (const m of q.matches(tree.rootNode)) {
+                    for (const cap of m.captures)
+                        out.push(cap);
+                }
+                return out;
+            },
+        };
+        wasmCache.set(lang, handle);
+        return handle;
+    }
+    catch {
+        wasmCache.set(lang, null);
+        return null;
+    }
+}
+/**
+ * Load the web-tree-sitter runtime on demand and initialize it. Returns
+ * `undefined` when the package isn't installed or the runtime refuses to
+ * init (sandboxed, missing WebAssembly, etc.).
+ */
+async function ensureWasmRuntime() {
+    if (wasmRuntime?.initialized === true)
+        return wasmRuntime;
+    let mod;
+    try {
+        mod = requireFn("web-tree-sitter");
+    }
+    catch {
+        return undefined;
+    }
+    try {
+        if (typeof mod.Parser.init === "function") {
+            await mod.Parser.init();
+        }
+    }
+    catch {
+        return undefined;
+    }
+    wasmRuntime = {
+        Parser: mod.Parser,
+        Query: mod.Query,
+        Language: mod.Language,
+        initialized: true,
+    };
+    return wasmRuntime;
+}
+/**
+ * Resolve the `.wasm` grammar asset for `lang`. Two-stage cascade:
+ *
+ * 1. Per-grammar-package lookup — for the 11 languages whose
+ *    `tree-sitter-<lang>` npm package ships its own `.wasm` alongside
+ *    the `.node` addon (typescript, tsx, javascript, python, go, rust,
+ *    java, csharp, c, cpp, ruby, php).
+ * 2. Vendored-WASM fallback — for kotlin, swift, and dart, whose
+ *    per-grammar packages do NOT ship a `.wasm`. We build these once
+ *    from the same grammar sources npm pins (zero drift) and commit
+ *    them to `packages/ingestion/vendor/wasms/`. See
+ *    `scripts/build-vendor-wasms.sh` and `vendor/wasms/README.md`.
+ *
+ * Returns `undefined` when neither stage resolves (package not
+ * installed, or language not in either table).
+ */
+function resolveGrammarWasmPath(lang) {
+    const direct = tryPerGrammarPackage(lang);
+    if (direct !== undefined)
+        return direct;
+    return tryVendoredWasm(lang);
+}
+/**
+ * Stage 1: resolve a `.wasm` that ships inside the per-grammar
+ * `tree-sitter-<lang>` npm package. Returns `undefined` when the
+ * language has no entry in this table or the package is not installed.
+ */
+function tryPerGrammarPackage(lang) {
+    // `tree-sitter-typescript` ships two wasms in one package — select by
+    // language variant.
+    if (lang === "typescript" || lang === "tsx") {
+        const pkgDir = resolvePackageDir("tree-sitter-typescript");
+        if (pkgDir === undefined)
+            return undefined;
+        const fname = lang === "typescript" ? "tree-sitter-typescript.wasm" : "tree-sitter-tsx.wasm";
+        return path.join(pkgDir, fname);
+    }
+    const mapping = {
+        javascript: { pkg: "tree-sitter-javascript", file: "tree-sitter-javascript.wasm" },
+        python: { pkg: "tree-sitter-python", file: "tree-sitter-python.wasm" },
+        go: { pkg: "tree-sitter-go", file: "tree-sitter-go.wasm" },
+        rust: { pkg: "tree-sitter-rust", file: "tree-sitter-rust.wasm" },
+        java: { pkg: "tree-sitter-java", file: "tree-sitter-java.wasm" },
+        // c-sharp publishes `tree-sitter-c_sharp.wasm` (underscore, not hyphen).
+        csharp: { pkg: "tree-sitter-c-sharp", file: "tree-sitter-c_sharp.wasm" },
+        c: { pkg: "tree-sitter-c", file: "tree-sitter-c.wasm" },
+        cpp: { pkg: "tree-sitter-cpp", file: "tree-sitter-cpp.wasm" },
+        ruby: { pkg: "tree-sitter-ruby", file: "tree-sitter-ruby.wasm" },
+        // Use php_only (pure PHP, no HTML template injection) to match native loader (grammar-registry.ts:244-254).
+        php: { pkg: "tree-sitter-php", file: "tree-sitter-php_only.wasm" },
+    };
+    const entry = mapping[lang];
+    if (entry === undefined)
+        return undefined;
+    const pkgDir = resolvePackageDir(entry.pkg);
+    if (pkgDir === undefined)
+        return undefined;
+    return path.join(pkgDir, entry.file);
+}
+/**
+ * Stage 2: resolve from the vendored WASM directory at
+ * `packages/ingestion/vendor/wasms/`. Only opted-in for languages whose
+ * per-grammar npm package does NOT ship a `.wasm` — kotlin, swift, dart.
+ *
+ * These are built once from the same grammar sources our package.json
+ * pins (zero version drift vs native) and committed to the repo. The
+ * upstream `tree-sitter-wasms` catalog can't be used because its 0.1.13
+ * artifacts were built with tree-sitter-cli 0.20.x and ship the legacy
+ * `dylink` section, which web-tree-sitter 0.26+ refuses to load (it
+ * requires the standardized `dylink.0` section).
+ *
+ * Keep this table minimal — adding a language here is a deliberate
+ * architectural choice. See `scripts/build-vendor-wasms.sh`.
+ */
+function tryVendoredWasm(lang) {
+    const catalog = {
+        kotlin: "tree-sitter-kotlin.wasm",
+        swift: "tree-sitter-swift.wasm",
+        dart: "tree-sitter-dart.wasm",
+    };
+    const fname = catalog[lang];
+    if (fname === undefined)
+        return undefined;
+    return path.join(VENDOR_WASMS_DIR, fname);
+}
+function resolvePackageDir(pkgName) {
+    try {
+        const manifestPath = requireFn.resolve(`${pkgName}/package.json`);
+        return path.dirname(manifestPath);
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * Test hook: clear the per-process WASM parser cache. Never call in
+ * production paths — it would force a re-init of every grammar.
+ */
+export function _resetWasmCacheForTests() {
+    wasmCache.clear();
+    wasmRuntime = undefined;
+}
+/**
+ * Test hook: expose the grammar-path resolver so unit tests can assert
+ * the two-stage cascade (per-grammar package → tree-sitter-wasms
+ * catalog) resolves kotlin/swift/dart correctly. Not part of the public
+ * API — callers in production paths must go through `openWasmParser`.
+ */
+export function _resolveGrammarWasmPathForTests(lang) {
+    return resolveGrammarWasmPath(lang);
+}
+//# sourceMappingURL=wasm-fallback.js.map

package/dist/parse/wasm-fallback.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"wasm-fallback.js","sourceRoot":"","sources":["../../src/parse/wasm-fallback.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAGzC,MAAM,SAAS,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAEjD,8EAA8E;AAC9E,0EAA0E;AAC1E,6EAA6E;AAC7E,kDAAkD;AAClD,MAAM,gBAAgB,GAAG,CAAC,GAAG,EAAE;IAC7B,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1D,oEAAoE;IACpE,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;AAC3D,CAAC,CAAC,EAAE,CAAC;AAEL,IAAI,MAA2B,CAAC;AAEhC;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB;IAC/B,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,IAAI,CAAC;QACH,SAAS,CAAC,aAAa,CAAC,CAAC;QACzB,MAAM,GAAG,IAAI,CAAC;IAChB,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,GAAG,KAAK,CAAC;IACjB,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,4BAA4B;IAC1C,MAAM,GAAG,SAAS,CAAC;AACrB,CAAC;AAuCD;;;GAGG;AACH,MAAM,SAAS,GAAG,IAAI,GAAG,EAAuC,CAAC;AACjE,IAAI,WAAoC,CAAC;AA+BzC;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,IAAgB;IACnD,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACnC,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC;IACxC,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,iBAAiB,EAAE,CAAC;QAC1C,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC1B,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YAC1B,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,QAAQ,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC;QAC9C,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YAC1B,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACzD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAiD,CAAC;QACzE,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QAC5B,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QAE/B,MAAM,MAAM,GAAqB;YAC/B,QAAQ,EAAE,IAAI;YACd,KAAK,EAAE,CAAC,MAAc,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;YAC/C,QAAQ,EAAE,CAAC,SAAiB,EAAE,MAAc,EAAE,EAAE;gBAC9C,4DAA4D;gBAC5D,4DAA4D;gBAC5D,MAAM,CAAC,GAAG,IAAI,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;gBACnD,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAClC,MAAM,GAAG,GAAkB,EAAE,CAAC;gBAC9B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;oBACzC,KAAK,MAAM,GAAG,IAAI,CAAC,CAAC,QAAQ;wBAAE,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC9C,CAAC;gBACD,OAAO,GAAG,CAAC;YACb,CAAC;SACF,CAAC;QACF,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAC5B,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,MAAM,CAAC;QACP,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC1B,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,iBAAiB;IAC9B,IAAI,WAAW,EAAE,WAAW,KAAK,IAAI;QAAE,OAAO,WAAW,CAAC;IAC1D,IAAI,GAAmF,CAAC;IACxF,IAAI,CAAC;QACH,GAAG,GAAG,SAAS,CAAC,iBAAiB,CAIhC,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,CAAC;QACH,IAAI,OAAO,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YAC1C,MAAM,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,WAAW,GAAG;QACZ,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,WAAW,EAAE,IAAI;KAClB,CAAC;IACF,OAAO,WAAW,CAAC;AACrB,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,SAAS,sBAAsB,CAAC,IAAgB;IAC9C,MAAM,MAAM,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAC;IAC1C,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC;IACxC,OAAO,eAAe,CAAC,IAAI,CAAC,CAAC;AAC/B,CAAC;AAED;;;;GAIG;AACH,SAAS,oBAAoB,CAAC,IAAgB;IAC5C,sEAAsE;IACtE,oBAAoB;IACpB,IAAI,IAAI,KAAK,YAAY,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;QAC5C,MAAM,MAAM,GAAG,iBAAiB,CAAC,wBAAwB,CAAC,CAAC;QAC3D,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO,SAAS,CAAC;QAC3C,MAAM,KAAK,GAAG,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,6BAA6B,CAAC,CAAC,CAAC,sBAAsB,CAAC;QAC7F,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAClC,CAAC;IACD,MAAM,OAAO,GAA+D;QAC1E,UAAU,EAAE,EAAE,GAAG,EAAE,wBAAwB,EAAE,IAAI,EAAE,6BAA6B,EAAE;QAClF,MAAM,EAAE,EAAE,GAAG,EAAE,oBAAoB,EAAE,IAAI,EAAE,yBAAyB,EAAE;QACtE,EAAE,EAAE,EAAE,GAAG,EAAE,gBAAgB,EAAE,IAAI,EAAE,qBAAqB,EAAE;QAC1D,IAAI,EAAE,EAAE,GAAG,EAAE,kBAAkB,EAAE,IAAI,EAAE,uBAAuB,EAAE;QAChE,IAAI,EAAE,EAAE,GAAG,EAAE,kBAAkB,EAAE,IAAI,EAAE,uBAAuB,EAAE;QAChE,yEAAyE;QACzE,MAAM,EAAE,EAAE,GAAG,EAAE,qBAAqB,EAAE,IAAI,EAAE,0BAA0B,EAAE;QACxE,CAAC,EAAE,EAAE,GAAG,EAAE,eAAe,EAAE,IAAI,EAAE,oBAAoB,EAAE;QACvD,GAAG,EAAE,EAAE,GAAG,EAAE,iBAAiB,EAAE,IAAI,EAAE,sBAAsB,EAAE;QAC7D,IAAI,EAAE,EAAE,GAAG,EAAE,kBAAkB,EAAE,IAAI,EAAE,uBAAuB,EAAE;QAChE,4GAA4G;QAC5G,GAAG,EAAE,EAAE,GAAG,EAAE,iBAAiB,EAAE,IAAI,EAAE,2BAA2B,EAAE;KACnE,CAAC;IACF,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5B,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC1C,MAAM,MAAM,GAAG,iBAAiB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC5C,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC3C,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;AACvC,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,SAAS,eAAe,CAAC,IAAgB;IACvC,MAAM,OAAO,GAAwC;QACnD,MAAM,EAAE,yBAAyB;QACjC,KAAK,EAAE,wBAAwB;QAC/B,IAAI,EAAE,uBAAuB;KAC9B,CAAC;IACF,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5B,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC1C,OAAO,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,KAAK,CAAC,CAAC;AAC5C,CAAC;AAED,SAAS,iBAAiB,CAAC,OAAe;IACxC,IAAI,CAAC;QACH,MAAM,YAAY,GAAG,SAAS,CAAC,OAAO,CAAC,GAAG,OAAO,eAAe,CAAC,CAAC;QAClE,OAAO,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;IACpC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,uBAAuB;IACrC,SAAS,CAAC,KAAK,EAAE,CAAC;IAClB,WAAW,GAAG,SAAS,CAAC;AAC1B,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,+BAA+B,CAAC,IAAgB;IAC9D,OAAO,sBAAsB,CAAC,IAAI,CAAC,CAAC;AACtC,CAAC"}

package/dist/parse/worker-pool.d.ts

@@ -0,0 +1,48 @@
+/**
+ * Piscina-backed worker pool wrapper with byte-budget chunking.
+ *
+ * Callers pass a list of tasks; the pool chunks them into batches respecting
+ * a byte budget (default 20 MB) and file cap (default 200 files), dispatches
+ * each batch to one worker, and collects the results. Output is sorted by
+ * `filePath` so two runs with identical inputs produce byte-identical output
+ * regardless of worker completion order.
+ *
+ * Sizing: by default `maxThreads = clamp(os.availableParallelism(), 2, 8)`.
+ * Beyond 8 the bottleneck shifts to main-thread GC pressure and file I/O, so
+ * we cap there. Callers can override.
+ */
+import type { ParseResult, ParseTask } from "./types.js";
+/** Options for a single dispatch() call. */
+export interface DispatchOptions {
+    /** Max cumulative bytes per worker batch. Default 20 MB. */
+    readonly byteBudget?: number;
+    /** Max files per worker batch (guards against many-tiny-files). Default 200. */
+    readonly fileCap?: number;
+}
+/** Options passed at pool construction. */
+export interface ParsePoolOptions {
+    readonly minThreads?: number;
+    readonly maxThreads?: number;
+}
+/**
+ * Main-thread wrapper around the piscina pool.
+ */
+export declare class ParsePool {
+    #private;
+    constructor(opts?: ParsePoolOptions);
+    /**
+     * Dispatch a set of parse tasks across the pool.
+     *
+     * Results are always sorted by `filePath` so output is deterministic
+     * regardless of worker completion order.
+     */
+    dispatch(tasks: readonly ParseTask[], opts?: DispatchOptions): Promise<readonly ParseResult[]>;
+    /** Shut down the pool; pending tasks are aborted. */
+    destroy(): Promise<void>;
+}
+/**
+ * Walk tasks in input order, accumulating into batches until either the
+ * byte budget or file cap is reached. Exported for testing.
+ */
+export declare function chunkTasks(tasks: readonly ParseTask[], byteBudget: number, fileCap: number): ParseTask[][];
+//# sourceMappingURL=worker-pool.d.ts.map

package/dist/parse/worker-pool.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker-pool.d.ts","sourceRoot":"","sources":["../../src/parse/worker-pool.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAKH,OAAO,KAAK,EAAc,WAAW,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAMrE,4CAA4C;AAC5C,MAAM,WAAW,eAAe;IAC9B,4DAA4D;IAC5D,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,gFAAgF;IAChF,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,2CAA2C;AAC3C,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAC9B;AAUD;;GAEG;AACH,qBAAa,SAAS;;gBAGR,IAAI,GAAE,gBAAqB;IAUvC;;;;;OAKG;IACG,QAAQ,CACZ,KAAK,EAAE,SAAS,SAAS,EAAE,EAC3B,IAAI,GAAE,eAAoB,GACzB,OAAO,CAAC,SAAS,WAAW,EAAE,CAAC;IAsBlC,qDAAqD;IAC/C,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAG/B;AAED;;;GAGG;AACH,wBAAgB,UAAU,CACxB,KAAK,EAAE,SAAS,SAAS,EAAE,EAC3B,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,MAAM,GACd,SAAS,EAAE,EAAE,CAwBf"}

package/dist/parse/worker-pool.js

@@ -0,0 +1,97 @@
+/**
+ * Piscina-backed worker pool wrapper with byte-budget chunking.
+ *
+ * Callers pass a list of tasks; the pool chunks them into batches respecting
+ * a byte budget (default 20 MB) and file cap (default 200 files), dispatches
+ * each batch to one worker, and collects the results. Output is sorted by
+ * `filePath` so two runs with identical inputs produce byte-identical output
+ * regardless of worker completion order.
+ *
+ * Sizing: by default `maxThreads = clamp(os.availableParallelism(), 2, 8)`.
+ * Beyond 8 the bottleneck shifts to main-thread GC pressure and file I/O, so
+ * we cap there. Callers can override.
+ */
+import { availableParallelism } from "node:os";
+import { fileURLToPath } from "node:url";
+import { Piscina } from "piscina";
+// When we compile with tsc, __filename resolution is via import.meta.url.
+// The worker file sits next to this one as `parse-worker.js` in dist/.
+const WORKER_FILENAME = fileURLToPath(new URL("./parse-worker.js", import.meta.url));
+const DEFAULT_BYTE_BUDGET = 20 * 1024 * 1024;
+const DEFAULT_FILE_CAP = 200;
+function defaultMaxThreads() {
+    const cpus = availableParallelism();
+    return Math.max(2, Math.min(cpus, 8));
+}
+/**
+ * Main-thread wrapper around the piscina pool.
+ */
+export class ParsePool {
+    #pool;
+    constructor(opts = {}) {
+        const minThreads = opts.minThreads ?? 2;
+        const maxThreads = opts.maxThreads ?? defaultMaxThreads();
+        this.#pool = new Piscina({
+            filename: WORKER_FILENAME,
+            minThreads,
+            maxThreads,
+        });
+    }
+    /**
+     * Dispatch a set of parse tasks across the pool.
+     *
+     * Results are always sorted by `filePath` so output is deterministic
+     * regardless of worker completion order.
+     */
+    async dispatch(tasks, opts = {}) {
+        if (tasks.length === 0) {
+            return [];
+        }
+        const byteBudget = opts.byteBudget ?? DEFAULT_BYTE_BUDGET;
+        const fileCap = opts.fileCap ?? DEFAULT_FILE_CAP;
+        const batches = chunkTasks(tasks, byteBudget, fileCap);
+        const batchResults = await Promise.all(batches.map((batch) => this.#pool.run({ tasks: batch })));
+        const flat = [];
+        for (const br of batchResults) {
+            for (const r of br) {
+                flat.push(r);
+            }
+        }
+        flat.sort((a, b) => (a.filePath < b.filePath ? -1 : a.filePath > b.filePath ? 1 : 0));
+        return flat;
+    }
+    /** Shut down the pool; pending tasks are aborted. */
+    async destroy() {
+        await this.#pool.destroy();
+    }
+}
+/**
+ * Walk tasks in input order, accumulating into batches until either the
+ * byte budget or file cap is reached. Exported for testing.
+ */
+export function chunkTasks(tasks, byteBudget, fileCap) {
+    const batches = [];
+    let current = [];
+    let currentBytes = 0;
+    for (const t of tasks) {
+        const size = t.content.byteLength;
+        // Flush the current batch BEFORE appending if either:
+        //   (a) adding this task would meet or exceed the byte budget, or
+        //   (b) the batch is already full by file count.
+        // The `>=` on bytes ensures that when a task's size hits the budget exactly
+        // it starts a new batch — matching the documented behavior on borderline
+        // inputs such as three 5MB files with a 10MB budget → three batches.
+        if (current.length > 0 && (currentBytes + size >= byteBudget || current.length >= fileCap)) {
+            batches.push(current);
+            current = [];
+            currentBytes = 0;
+        }
+        current.push(t);
+        currentBytes += size;
+    }
+    if (current.length > 0) {
+        batches.push(current);
+    }
+    return batches;
+}
+//# sourceMappingURL=worker-pool.js.map

package/dist/parse/worker-pool.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker-pool.js","sourceRoot":"","sources":["../../src/parse/worker-pool.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAGlC,0EAA0E;AAC1E,uEAAuE;AACvE,MAAM,eAAe,GAAG,aAAa,CAAC,IAAI,GAAG,CAAC,mBAAmB,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAgBrF,MAAM,mBAAmB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAC7C,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAE7B,SAAS,iBAAiB;IACxB,MAAM,IAAI,GAAG,oBAAoB,EAAE,CAAC;IACpC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AACxC,CAAC;AAED;;GAEG;AACH,MAAM,OAAO,SAAS;IACX,KAAK,CAAqC;IAEnD,YAAY,OAAyB,EAAE;QACrC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QACxC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,iBAAiB,EAAE,CAAC;QAC1D,IAAI,CAAC,KAAK,GAAG,IAAI,OAAO,CAA4B;YAClD,QAAQ,EAAE,eAAe;YACzB,UAAU;YACV,UAAU;SACX,CAAC,CAAC;IACL,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,QAAQ,CACZ,KAA2B,EAC3B,OAAwB,EAAE;QAE1B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,mBAAmB,CAAC;QAC1D,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,gBAAgB,CAAC;QACjD,MAAM,OAAO,GAAG,UAAU,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;QAEvD,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CACpC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,CACzD,CAAC;QAEF,MAAM,IAAI,GAAkB,EAAE,CAAC;QAC/B,KAAK,MAAM,EAAE,IAAI,YAAY,EAAE,CAAC;YAC9B,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;gBACnB,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACf,CAAC;QACH,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACtF,OAAO,IAAI,CAAC;IACd,CAAC;IAED,qDAAqD;IACrD,KAAK,CAAC,OAAO;QACX,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;IAC7B,CAAC;CACF;AAED;;;GAGG;AACH,MAAM,UAAU,UAAU,CACxB,KAA2B,EAC3B,UAAkB,EAClB,OAAe;IAEf,MAAM,OAAO,GAAkB,EAAE,CAAC;IAClC,IAAI,OAAO,GAAgB,EAAE,CAAC;IAC9B,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC;QAClC,sDAAsD;QACtD,kEAAkE;QAClE,iDAAiD;QACjD,4EAA4E;QAC5E,yEAAyE;QACzE,qEAAqE;QACrE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,UAAU,IAAI,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,EAAE,CAAC;YAC3F,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtB,OAAO,GAAG,EAAE,CAAC;YACb,YAAY,GAAG,CAAC,CAAC;QACnB,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChB,YAAY,IAAI,IAAI,CAAC;IACvB,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACxB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/pipeline/dep-parsers/go.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Go ecosystem manifest parser.
+ *
+ * Supported inputs:
+ *   - `go.sum` — one module per line, format
+ *       `<module> <version>[/go.mod] <h1:hash>`
+ *     We consume only the 2-column (module, version) projection and skip
+ *     the `/go.mod` discriminator so each module is emitted once.
+ *   - `go.mod` — uses the text format specified at https://go.dev/ref/mod.
+ *     We parse the `require` block (single- or multi-line form), capture
+ *     `<module> <version>` pairs, and skip `replace` / `exclude` directives.
+ *
+ * The version is the raw Go module pseudo-version string (e.g.
+ * `v1.2.3`, or `v0.0.0-20231201123456-abcdef012345`) exactly as it appears
+ * in the source file.
+ *
+ * License detection: neither go.mod nor go.sum carries license metadata.
+ * The canonical source is `pkg.go.dev` + the module's `LICENSE` file,
+ * both of which live behind a network boundary this phase forbids.
+ * Per-dep `license` therefore stays undefined and the dependencies
+ * phase maps that to the `"UNKNOWN"` sentinel.
+ */
+import type { ParseDepsFn } from "./types.js";
+export declare const parseGoDeps: ParseDepsFn;
+//# sourceMappingURL=go.d.ts.map

package/dist/pipeline/dep-parsers/go.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"go.d.ts","sourceRoot":"","sources":["../../../src/pipeline/dep-parsers/go.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAIH,OAAO,KAAK,EAAE,WAAW,EAAoB,MAAM,YAAY,CAAC;AAIhE,eAAO,MAAM,WAAW,EAAE,WAgBzB,CAAC"}