gitnexus 1.6.8-rc.7 → 1.6.8-rc.9

package/dist/core/embeddings/embedder.js

@@ -21,6 +21,7 @@ import { isHttpMode, getHttpDimensions, httpEmbed } from './http-client.js';
 import { resolveEmbeddingConfig } from './config.js';
 import { applyHfEnvOverrides, isHfDownloadFailure, withHfDownloadRetry } from './hf-env.js';
 import { getLocalEmbeddingRuntimeBlocker } from './runtime-support.js';
+import { ensureOnnxRuntimeCommonResolvable } from './onnxruntime-common-resolver.js';
 import { logger } from '../logger.js';
 /**
  * Check whether the onnxruntime-node package that @huggingface/transformers
@@ -147,6 +148,9 @@ export const initEmbedder = async (onProgress, config = {}, forceDevice) => {
         try {
             // Lazy-load transformers.js only after the runtime guard has passed, so
             // unsupported platforms never reach the native ONNX import (#1515).
+            // Under pnpm-strict / `pnpm dlx`, transformers' phantom `onnxruntime-common`
+            // import is unresolvable; register the fallback resolver first (#307).
+            ensureOnnxRuntimeCommonResolvable();
             const { pipeline, env } = await import('@huggingface/transformers');
             // Configure transformers.js environment
             env.allowLocalModels = false;

package/dist/core/embeddings/onnxruntime-common-resolver.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Idempotently install the onnxruntime-common resolution fallback. Call once
+ * immediately before the dynamic `import('@huggingface/transformers')` on the
+ * local-embedding path.
+ */
+export declare const ensureOnnxRuntimeCommonResolvable: () => void;

package/dist/core/embeddings/onnxruntime-common-resolver.js

@@ -0,0 +1,130 @@
+/**
+ * Make `@huggingface/transformers`' phantom `onnxruntime-common` import
+ * resolvable under strict package-manager layouts (#307, #2069).
+ *
+ * ## Why
+ * transformers' shipped `dist/transformers.node.mjs` does a bare
+ * `import 'onnxruntime-common'`, but transformers' `package.json` never declares
+ * onnxruntime-common (it lists onnxruntime-node / onnxruntime-web / sharp). With
+ * npm's flat `node_modules` — or pnpm with hoisting — the package is hoisted to
+ * a directory on transformers' resolution path and the import resolves by
+ * accident. Under pnpm's isolated store (and therefore `pnpm dlx` / `pnpx`), a
+ * package only sees its *declared* deps, so the import dies with
+ * `ERR_MODULE_NOT_FOUND` before `analyze --embeddings` can run.
+ *
+ * Declaring onnxruntime-common in gitnexus' own dependencies (#2074) does NOT
+ * fix this under pnpm: Node resolves the bare specifier from *transformers'*
+ * module scope, not ours, and overrides/resolutions can only re-version an
+ * existing edge, never add the missing one.
+ *
+ * ## What this does
+ * Install a synchronous, in-thread ESM resolution hook (`module.registerHooks`,
+ * Node >= 22.15) that redirects `onnxruntime-common` to a copy gitnexus can
+ * resolve — but only when the default resolver fails. The redirect target is
+ * preferentially the `onnxruntime-common` that `onnxruntime-node` (the native
+ * binding transformers actually loads) itself depends on, so the redirected copy
+ * is version-matched to that binding even under `pnpm dlx` — where gitnexus'
+ * npm-style `overrides` block does NOT apply, because it is honoured only from a
+ * root manifest and gitnexus is a transitive dependency there. It falls back to
+ * gitnexus' own direct `onnxruntime-common` dependency when that chain can't be
+ * walked. onnxruntime-common is a stable, pure-JS package whose `Tensor` surface
+ * is unchanged across 1.24–1.26, so either target is API-compatible. On working
+ * layouts the default resolver succeeds first and the hook never fires, so
+ * behaviour is unchanged.
+ *
+ * `registerHooks` (synchronous, in-thread) is preferred over the older
+ * `module.register` (async, off-thread, now deprecated — DEP0205, removed in
+ * Node 26): the redirect is a one-line conditional that needs no worker thread,
+ * no separate hook module, and no `data` marshalling.
+ *
+ * ## Safety
+ * Best-effort and idempotent. The hook is installed lazily, only on the
+ * local-embedding code path (after parsing), so it is never registered during
+ * analysis, in the parse workers, or in HTTP embedding mode. Once installed it
+ * is process-global: its resolve closure runs for every subsequent module
+ * resolution, but it passes all of them through untouched and only substitutes a
+ * result for the exact `onnxruntime-common` specifier when that specifier is
+ * genuinely absent — so it cannot mask an unrelated resolution error, and the
+ * per-resolution cost is a single string comparison.
+ *
+ * `module.registerHooks` is marked `@experimental` and requires Node >= 22.15
+ * (the gitnexus engines floor is >= 22.0.0). On older runtimes it is absent and
+ * this is a graceful no-op: embeddings then resolve onnxruntime-common exactly
+ * as before — fine on hoisted layouts. Any failure during installation is
+ * swallowed.
+ */
+import { registerHooks, createRequire } from 'node:module';
+import { pathToFileURL } from 'node:url';
+import { logger } from '../logger.js';
+let attempted = false;
+/**
+ * Compute the file: URL the hook redirects `onnxruntime-common` to.
+ *
+ * Prefer the copy `onnxruntime-node` (the native binding transformers loads)
+ * depends on, so the redirected module is version-matched to the binding even
+ * under `pnpm dlx`, where transformers keeps its own pinned onnxruntime-node.
+ * The walk resolves transformers' MAIN entry — NOT `@huggingface/transformers/
+ * package.json`, which transformers' `exports` map blocks
+ * (`ERR_PACKAGE_PATH_NOT_EXPORTED`) — then onnxruntime-node, then its
+ * onnxruntime-common. Falls back to gitnexus' own direct dependency (always
+ * resolvable from our scope) when any step fails.
+ */
+const resolveOnnxRuntimeCommonUrl = () => {
+    const require = createRequire(import.meta.url);
+    try {
+        const transformersMain = require.resolve('@huggingface/transformers');
+        const ortNodePkg = createRequire(transformersMain).resolve('onnxruntime-node/package.json');
+        const common = createRequire(ortNodePkg).resolve('onnxruntime-common');
+        return pathToFileURL(common).href;
+    }
+    catch {
+        return pathToFileURL(require.resolve('onnxruntime-common')).href;
+    }
+};
+/**
+ * Idempotently install the onnxruntime-common resolution fallback. Call once
+ * immediately before the dynamic `import('@huggingface/transformers')` on the
+ * local-embedding path.
+ */
+export const ensureOnnxRuntimeCommonResolvable = () => {
+    if (attempted)
+        return;
+    // Mark attempted up-front: a failed attempt must not retry on every
+    // initEmbedder() call, and the hook is process-global — once is enough.
+    attempted = true;
+    try {
+        // Node < 22.15 (the gitnexus engines floor is >= 22.0.0): no synchronous
+        // hooks API. Degrade gracefully — the import still works on hoisted layouts.
+        if (typeof registerHooks !== 'function')
+            return;
+        const redirectUrl = resolveOnnxRuntimeCommonUrl();
+        registerHooks({
+            resolve(specifier, context, nextResolve) {
+                if (specifier !== 'onnxruntime-common')
+                    return nextResolve(specifier, context);
+                // Honour a real, package-manager-provided copy when one is on the path
+                // (npm / hoisted pnpm); only substitute ours when the specifier is
+                // genuinely absent.
+                try {
+                    return nextResolve(specifier, context);
+                }
+                catch (err) {
+                    // The phantom import surfaces as ERR_MODULE_NOT_FOUND (or, for a
+                    // present-but-exports-broken copy, ERR_PACKAGE_PATH_NOT_EXPORTED).
+                    // Rethrow anything else so a genuinely broken install is not masked.
+                    const code = err?.code;
+                    if (code === 'ERR_MODULE_NOT_FOUND' || code === 'ERR_PACKAGE_PATH_NOT_EXPORTED') {
+                        return { url: redirectUrl, shortCircuit: true };
+                    }
+                    throw err;
+                }
+            },
+        });
+        logger.debug({ redirectUrl }, 'Installed onnxruntime-common resolution fallback (#307)');
+    }
+    catch (err) {
+        // Never block embeddings on the fallback. On layouts where the package
+        // manager already resolves onnxruntime-common this is unnecessary anyway.
+        logger.debug({ err: err instanceof Error ? err.message : String(err) }, 'onnxruntime-common resolution fallback not installed');
+    }
+};

package/dist/core/ingestion/language-provider.d.ts

@@ -147,6 +147,12 @@ interface LanguageProviderConfig {
      * `undefined` when no constraints exist / the node isn't a templated
      * function. Languages without SFINAE / concept semantics leave this
      * undefined and the disambiguation is a pass-through.
+     *
+     * Cloneability contract: the returned payload crosses the worker boundary
+     * via structured clone, so it MUST be structured-clone-safe (no functions,
+     * symbols, or tree-sitter `SyntaxNode`s — only plain data). Wrap the return
+     * with `assertCloneable` from `workers/clone-safety.ts` so a future leak is a
+     * compile error at the source instead of a runtime DataCloneError (#2143).
      */
     readonly extractTemplateConstraints?: (definitionNode: SyntaxNode) => unknown;
     /** Override the default node label for definition.function captures.
@@ -268,8 +274,12 @@ interface LanguageProviderConfig {
      * disk store WITHOUT a main-thread re-parse. The main thread restores them
      * via the matching `ScopeResolver.applyCaptureSideChannel` hook.
      *
-     * MUST return plain data (objects / arrays / primitives) so it round-trips
-     * through `JSON.stringify` + the parsedfile-store interning reviver.
+     * Cloneability contract: MUST return plain data (objects / arrays /
+     * primitives — no functions, symbols, or tree-sitter `SyntaxNode`s) so it
+     * survives BOTH the worker→main structured clone AND `JSON.stringify` + the
+     * parsedfile-store interning reviver. Wrap the return with `assertCloneable`
+     * from `workers/clone-safety.ts` so a future non-serializable leak is a
+     * compile error at the source instead of a runtime DataCloneError (#2143).
      *
      * Default: undefined (provider has no capture-time module-level side effects).
      */

package/dist/core/ingestion/languages/c-cpp.js

@@ -38,7 +38,8 @@ import { cCallConfig, cppCallConfig } from '../call-extractors/configs/c-cpp.js'
 import { stripUeMacros } from '../cpp-ue-preprocessor.js';
 import { emitCScopeCaptures, interpretCImport, interpretCTypeBinding, cArityCompatibility, cBindingScopeFor, cImportOwningScope, cReceiverBinding, collectCStaticLinkageSideChannel, } from './c/index.js';
 import { emitCppScopeCaptures, interpretCppImport, interpretCppTypeBinding, cppArityCompatibility, cppBindingScopeFor, cppImportOwningScope, cppReceiverBinding, collectCppCaptureSideChannel, } from './cpp/index.js';
-import { extractCppTemplateConstraints } from './cpp/constraint-extractor.js';
+import { extractCppTemplateConstraints, } from './cpp/constraint-extractor.js';
+import { assertCloneable } from '../workers/clone-safety.js';
 const C_BUILT_INS = new Set([
     'printf',
     'fprintf',
@@ -358,7 +359,10 @@ export const cProvider = defineLanguage({
     // `static` functions look non-file-local on the main thread and leak into
     // cross-file global free-call resolution / wildcard imports. See
     // `c/capture-side-channel.ts`.
-    collectCaptureSideChannel: collectCStaticLinkageSideChannel,
+    // `assertCloneable` is a runtime identity; it makes a future non-serializable
+    // value in the side-channel payload a compile error here, at the source, rather
+    // than a DataCloneError at the worker boundary (#2143).
+    collectCaptureSideChannel: (filePath) => assertCloneable(collectCStaticLinkageSideChannel(filePath)),
     interpretImport: interpretCImport,
     interpretTypeBinding: interpretCTypeBinding,
     bindingScopeFor: cBindingScopeFor,
@@ -431,7 +435,7 @@ export const cppProvider = defineLanguage({
     // just populated for this file into plain data on `ParsedFile.captureSideChannel`,
     // so the main thread can restore them via `applyCaptureSideChannel` WITHOUT a
     // re-parse (#1983). See `cpp/capture-side-channel.ts`.
-    collectCaptureSideChannel: collectCppCaptureSideChannel,
+    collectCaptureSideChannel: (filePath) => assertCloneable(collectCppCaptureSideChannel(filePath)),
     interpretImport: interpretCppImport,
     interpretTypeBinding: interpretCppTypeBinding,
     bindingScopeFor: cppBindingScopeFor,
@@ -482,5 +486,8 @@ function extractCppTemplateConstraintsForProvider(definitionNode) {
         }
         break;
     }
-    return extractCppTemplateConstraints(templateDecl, declarator);
+    // Guard the boundary at the source: a future non-cloneable member of the
+    // constraint payload becomes a compile error here, not a runtime
+    // DataCloneError at the worker post (#2143).
+    return assertCloneable(extractCppTemplateConstraints(templateDecl, declarator));
 }

package/dist/core/ingestion/languages/kotlin.js

@@ -10,6 +10,7 @@ import { SupportedLanguages } from '../../../_shared/index.js';
 import { createClassExtractor } from '../class-extractors/generic.js';
 import { kotlinClassConfig } from '../class-extractors/configs/jvm.js';
 import { defineLanguage } from '../language-provider.js';
+import { assertCloneable } from '../workers/clone-safety.js';
 import { kotlinTypeConfig } from '../type-extractors/jvm.js';
 import { kotlinExportChecker } from '../export-detection.js';
 import { createImportResolver } from '../import-resolvers/resolver-factory.js';
@@ -166,7 +167,10 @@ export const kotlinProvider = defineLanguage({
     // so the main thread can restore them via `applyCaptureSideChannel` WITHOUT a
     // re-parse (#1983). Without this, companion/static dispatch emits no CALLS
     // edges on the worker path. See `kotlin/capture-side-channel.ts`.
-    collectCaptureSideChannel: collectKotlinCaptureSideChannel,
+    // `assertCloneable` is a runtime identity; it makes a future non-serializable
+    // value in the side-channel payload a compile error here, at the source, rather
+    // than a DataCloneError at the worker boundary (#2143).
+    collectCaptureSideChannel: (filePath) => assertCloneable(collectKotlinCaptureSideChannel(filePath)),
     interpretImport: interpretKotlinImport,
     interpretTypeBinding: interpretKotlinTypeBinding,
     bindingScopeFor: kotlinBindingScopeFor,

package/dist/core/ingestion/parsing-processor.js

@@ -144,6 +144,27 @@ chunkHash) => {
             .join(', ');
         logger.warn(`  Skipped unsupported languages: ${summary}`);
     }
+    // Clone-safety telemetry (#2112): files whose parse output carried a value
+    // the structured-clone algorithm couldn't serialize across the worker
+    // boundary. The worker sanitized/dropped the offending value so the run
+    // could complete; surface the (rare) data loss so it's visible and the
+    // offending extractor can be fixed at source.
+    const skippedPaths = [];
+    for (const result of chunkResults) {
+        for (const entry of result.skippedPaths ?? [])
+            skippedPaths.push(entry);
+    }
+    if (skippedPaths.length > 0) {
+        // Keep the per-file reason ("stripped N value(s) from nodes" /
+        // "dropped non-serializable parsedFiles entry") — it distinguishes a
+        // recoverable strip from a whole-record drop, which a path-only line loses.
+        const shown = skippedPaths
+            .slice(0, 10)
+            .map((e) => `${e.path} (${e.reason})`)
+            .join(', ');
+        const more = skippedPaths.length > 10 ? ` …and ${skippedPaths.length - 10} more` : '';
+        logger.warn(`  Sanitized ${skippedPaths.length} file(s) with non-serializable parse output: ${shown}${more}`);
+    }
     onFileProgress?.(total, total, 'done');
     return chunkResults;
 };

package/dist/core/ingestion/workers/clone-safety.d.ts

@@ -0,0 +1,109 @@
+/**
+ * Structured-clone safety for the worker result boundary (#2112).
+ *
+ * A parse worker delivers its accumulated result to the main thread via
+ * `parentPort.postMessage(...)`. Node serializes that payload with the
+ * structured-clone algorithm SYNCHRONOUSLY on the worker thread, and it
+ * THROWS a `DataCloneError` the instant it meets a value it can't serialize —
+ * a function, a symbol, a Promise, a WeakMap, etc. The reporter of #2112 hit
+ * exactly this: a node record whose `properties` carried an own-enumerable
+ * value pointing at a native function (`function toString() { [native code] }
+ * could not be cloned`). One such value aborted the entire parse phase,
+ * because the worker re-posts the throw as `{type:'error'}` which the pool
+ * counts as a worker death — and under `GITNEXUS_WORKER_POOL_SIZE=1` the same
+ * graph re-throws on every respawn until the slot's budget is exhausted.
+ *
+ * This module is the safety net. It runs ONLY after a real clone failure on
+ * the fast-path post (zero overhead on healthy runs), and rewrites the
+ * boundary-crossing arrays so the result becomes cloneable: a non-cloneable
+ * value inside a plain extraction record is dropped (the record is otherwise
+ * kept — strictly-missing data, never wrong), and a `ParsedFile` that can't be
+ * made cloneable is dropped whole so scope-resolution re-derives it on the
+ * main thread (where there is no clone boundary) with intact edge data.
+ *
+ * Language-neutral by construction: it keys on value shape and field name
+ * only, never on a language (AGENTS.md shared-pipeline rule). The strip
+ * semantics mirror what the store path's `JSON.stringify` already silently
+ * drops, so store / no-store / cold / warm runs converge on the same graph.
+ */
+/** A file whose parse result was sanitized or dropped at the clone boundary. */
+export interface SkippedPath {
+    /** Best-effort source path of the offending record (or `(unknown)`). */
+    path: string;
+    /** Human-readable reason, e.g. "dropped 1 non-serializable value from nodes". */
+    reason: string;
+}
+/**
+ * True iff `value` survives Node's structured-clone algorithm (the same
+ * algorithm `postMessage` uses). This is the authoritative probe — it matches
+ * the real failure exactly, including Map/Set/Date/RegExp/TypedArray support,
+ * so it never false-positives on the `Scope` Maps that clone fine.
+ */
+export declare function isStructuredCloneable(value: unknown): boolean;
+/** The leaf values the structured-clone algorithm copies verbatim. */
+type CloneablePrimitive = undefined | null | boolean | number | bigint | string;
+/**
+ * Maps `T` to itself when every value reachable from it is structured-clone
+ * safe, and to a type containing `never` at the first offending property
+ * otherwise. A function or symbol — the values `postMessage` rejects — becomes
+ * `never`, so a struct carrying one is no longer assignable to its own
+ * `Cloneable<T>` and `assertCloneable` rejects it, naming the bad key.
+ *
+ * Implemented as a homomorphic mapped type (`{ [K in keyof T]: … }`) so it
+ * preserves `interface` shapes and `readonly` modifiers and works WITHOUT
+ * requiring the payload types to carry an index signature — sidestepping the
+ * "closed interface is not assignable to a recursive index-signature type" wall
+ * that blocked the value-typed-`Cloneable` approach (#2143). `Map`/`Set`/array
+ * containers recurse into their element types; `Date`/`RegExp` are clone-safe
+ * leaves.
+ */
+/** True iff `T` is `any` (the canonical `IsAny` probe: only `any` satisfies `0 extends 1 & T`). */
+type IsAny<T> = 0 extends 1 & T ? true : false;
+export type Cloneable<T> = IsAny<T> extends true ? never : T extends CloneablePrimitive | Date | RegExp ? T : T extends (...args: never[]) => unknown ? never : T extends symbol ? never : T extends ReadonlyMap<infer K, infer V> ? ReadonlyMap<Cloneable<K>, Cloneable<V>> : T extends ReadonlySet<infer U> ? ReadonlySet<Cloneable<U>> : T extends readonly (infer U)[] ? T extends unknown[] ? Cloneable<U>[] : readonly Cloneable<U>[] : T extends object ? {
+    [K in keyof T]: Cloneable<T[K]>;
+} : never;
+/**
+ * Identity at runtime (zero cost — returns its argument unchanged); a
+ * compile-time assertion that `value` is structured-clone safe. Wrap a
+ * producer that feeds an `unknown` worker-result sink:
+ *
+ *   collectCaptureSideChannel: (filePath) => assertCloneable(collectFoo(filePath))
+ *
+ * If `collectFoo`'s return type ever gains a non-cloneable member (a function, a
+ * `SyntaxNode`, …) the call fails to compile, pointing at the offending key.
+ *
+ * The parameter is a conditional type rather than an `extends Cloneable<T>`
+ * constraint because a self-referential constraint (`T extends Cloneable<T>`)
+ * is a "circular constraint" error in TypeScript. For a clone-safe `T` the
+ * parameter resolves to `T` (call type-checks as a plain identity); for an
+ * unsafe `T` it resolves to `Cloneable<T>` (which has `never` at the bad key),
+ * so the argument is rejected.
+ */
+export declare function assertCloneable<T>(value: T extends Cloneable<T> ? T : Cloneable<T>): T;
+export interface MakeCloneSafeOptions {
+    /**
+     * Array field names whose offending elements are DROPPED whole rather than
+     * stripped in place (e.g. `parsedFiles` — its `captureSideChannel` drives
+     * edge resolution, so a stripped-and-delivered file would ship WRONG edges;
+     * dropping it lets scope-resolution re-derive it on the main thread).
+     */
+    dropWholeElement: ReadonlySet<string>;
+    /** Field names to skip entirely (e.g. the `skippedPaths` field itself). */
+    skipFields?: ReadonlySet<string>;
+    /** Keys to probe for a file path when attributing a skip. */
+    pathKeys?: readonly string[];
+}
+/**
+ * Make a worker result's boundary-crossing array fields structured-cloneable,
+ * mutating `result` in place. Only arrays that actually contain a
+ * non-cloneable value are rewritten; everything else keeps referential
+ * identity. Returns the list of affected file paths for reporting.
+ *
+ * Call this after ANY failure of the fast-path post — a `DataCloneError`, OR a
+ * throwing getter's own error surfaced by structuredClone (the caller in
+ * `post-result.ts` recovers on any throw, not only `DataCloneError`).
+ */
+export declare function makeWorkerResultCloneSafe(result: Record<string, unknown>, options: MakeCloneSafeOptions): {
+    skipped: SkippedPath[];
+};
+export {};