@maxgfr/codeindex 2.10.0 → 2.11.1

package/src/extract/code.ts

@@ -1,6 +1,7 @@
 import type { CodeSymbol, RawRef } from "../types.js";
 import { extractSymbols } from "../lang/registry.js";
 import { extractAst } from "../ast/extract.js";
+import { extractReexports } from "../lang/common.js";
 
 export interface CodeInfo {
   symbols: CodeSymbol[];
@@ -253,50 +254,6 @@ function extractImports(ext: string, content: string): RawRef[] {
   return [...specs].map((spec) => ({ kind: "import" as const, spec }));
 }
 
-// Barrel re-exports (`export { A, B as C } from './x'`, `export * from './y'`).
-// The line-based lang extractor can't capture multi-name lists, but these ARE
-// the public facade of a module — so list them as exported symbols here.
-function extractReexports(rel: string, content: string): CodeSymbol[] {
-  if (!JS_TS.has(rel.slice(rel.lastIndexOf(".")))) return [];
-  const lang = /\.(ts|tsx|mts|cts)$/.test(rel) ? "typescript" : "javascript";
-  const out: CodeSymbol[] = [];
-  const seen = new Set<string>();
-  const lineAt = (idx: number): number => content.slice(0, idx).split(/\r?\n/).length;
-
-  const named = /export\s*\{([\s\S]*?)\}\s*(?:from\s*['"]([^'"]+)['"])?\s*;?/g;
-  let m: RegExpExecArray | null;
-  while ((m = named.exec(content)) && out.length < 60) {
-    const from = m[2];
-    for (const part of m[1]!.split(",")) {
-      const p = part.trim().replace(/^type\s+/, "");
-      const as = /^(\S+)\s+as\s+([A-Za-z_$][\w$]*)$/.exec(p);
-      const name = as ? as[2]! : p;
-      if (!/^[A-Za-z_$][\w$]*$/.test(name) || name === "default" || seen.has(name)) continue;
-      seen.add(name);
-      out.push({
-        name, kind: "reexport", file: rel, line: lineAt(m.index),
-        signature: from ? `export { ${name} } from "${from}"` : `export { ${name} }`,
-        exported: true, lang,
-      });
-    }
-  }
-
-  const star = /export\s*\*\s*(?:as\s+([A-Za-z_$][\w$]*)\s+)?from\s*['"]([^'"]+)['"]/g;
-  while ((m = star.exec(content)) && out.length < 60) {
-    const ns = m[1];
-    const from = m[2]!;
-    const key = "*" + (ns ?? from);
-    if (seen.has(key)) continue;
-    seen.add(key);
-    out.push({
-      name: ns ?? `* (${from})`, kind: ns ? "reexport" : "reexport-all", file: rel,
-      line: lineAt(m.index), signature: `export * ${ns ? `as ${ns} ` : ""}from "${from}"`,
-      exported: true, lang,
-    });
-  }
-  return out;
-}
-
 // Control-flow and declaration keywords that syntactically precede `(` but are
 // never call targets — the union across supported languages. Deliberately does
 // NOT list real builtins (python's `print`, go's `make`…): a false call to a
@@ -319,8 +276,35 @@ const DEF_INTRODUCERS = /(?:\bfunction|\bdef|\bfunc|\bfun|\bfn|\bclass|\bsub|\bm
 // immediate `receiver.` prefix is captured too (`axios.get(` → receiver
 // "axios"; `a.b.c(` → receiver "b" — the group anchors to the segment right
 // before the called name); bare calls carry no receiver.
-function collectCallsRegex(content: string): { name: string; line: number; receiver?: string }[] {
+//
+// `symbols` is the file's OWN regex-extracted symbols (name + definition
+// line). DEF_INTRODUCERS already excludes definitions that read `function
+// foo(`/`def foo(`/etc. (per OCCURRENCE, wherever on the line it sits), but
+// C/C++ function definitions have no such introducer (`void load(void) {`) —
+// the bare name reads exactly like a call to itself on its own definition
+// line. For those, a call candidate whose (name, line) exactly matches one of
+// `symbols` is excluded too — but ONLY its first (leftmost) occurrence on that
+// line, never every same-named occurrence: dense/minified one-liners can pack
+// a genuine call to the same name on the same physical line as its own
+// definition (`function aa(){}function bb(){return aa()+cc()}function cc(){}`
+// — bb's calls to aa() and cc() must survive), and even a bodyless
+// single-line recursive definition (`function foo(){foo();}`) has a real self
+// -call to keep. Two-tier: if ANY occurrence of a def'd name on this line is
+// already caught by DEF_INTRODUCERS (JS/Python/…), that occurrence alone is
+// excluded (existing per-occurrence check below) and no further exclusion is
+// applied — every OTHER occurrence is a genuine call. Only when NO occurrence
+// carries an introducer (C/C++) does this fall back to excluding just the
+// first occurrence: C/C++'s own definition regex requires column 0, so on a
+// line where it matches at all, the first occurrence IS that definition.
+// Exported for direct testing (extraction-v8.test.ts): once a wasm sidecar/
+// grammar is loaded, extractCode never reaches this path for C/C++, so tests
+// exercise it directly rather than through extractCode.
+export function collectCallsRegex(
+  content: string,
+  symbols: Pick<CodeSymbol, "name" | "line">[] = [],
+): { name: string; line: number; receiver?: string }[] {
   const out = new Map<string, { name: string; line: number; receiver?: string }>();
+  const ownDefLines = new Set(symbols.map((s) => `${s.name} ${s.line}`));
   const lines = content.split("\n");
   const CALL_RE = /(?:\bnew\s+)?(?:([A-Za-z_$][\w$]*)\s*\.\s*)?([A-Za-z_$][\w$]*)\s*\(/g;
   for (let i = 0; i < lines.length && out.size < 512; i++) {
@@ -330,14 +314,36 @@ function collectCallsRegex(content: string): { name: string; line: number; recei
     // regexes — noise resolves to nothing in the global call pass).
     const trimmed = line.trimStart();
     if (trimmed.startsWith("//") || trimmed.startsWith("#") || trimmed.startsWith("*")) continue;
+
+    // Pass 1: which own-def keys on this line have at least one occurrence
+    // DEF_INTRODUCERS already catches? Those are fully handled per-occurrence
+    // below — no fallback exclusion needed (or wanted) for them.
+    CALL_RE.lastIndex = 0;
+    let probe: RegExpExecArray | null;
+    const introducerCaught = new Set<string>();
+    while ((probe = CALL_RE.exec(line)) !== null) {
+      const name = probe[2]!;
+      const key = `${name} ${i + 1}`;
+      if (ownDefLines.has(key) && DEF_INTRODUCERS.test(line.slice(0, probe.index))) introducerCaught.add(key);
+    }
+
+    // Pass 2: the real collection. Own-def keys with no introducer occurrence
+    // fall back to excluding just their first occurrence on the line.
     CALL_RE.lastIndex = 0;
     let m: RegExpExecArray | null;
+    const fallbackExcluded = new Set<string>();
     while ((m = CALL_RE.exec(line)) !== null && out.size < 512) {
       const receiver = m[1];
       const name = m[2]!;
       if (name.length < 2 || CALL_KEYWORDS.has(name)) continue;
       if (DEF_INTRODUCERS.test(line.slice(0, m.index))) continue;
       const key = `${name} ${i + 1}`;
+      if (ownDefLines.has(key) && !introducerCaught.has(key)) {
+        if (!fallbackExcluded.has(key)) {
+          fallbackExcluded.add(key);
+          continue;
+        }
+      }
       if (!out.has(key)) out.set(key, receiver ? { name, line: i + 1, receiver } : { name, line: i + 1 });
     }
   }
@@ -354,7 +360,7 @@ export function extractCode(rel: string, ext: string, content: string): CodeInfo
   const symbols = (ast ? ast.symbols : extractSymbols(rel, ext, content)).slice(0, 400);
   // Add barrel re-exports the local def didn't already cover.
   const known = new Set(symbols.map((s) => s.name));
-  const reexports = extractReexports(rel, content).filter((s) => !known.has(s.name));
+  const reexports = extractReexports(rel, content, symbols).filter((s) => !known.has(s.name));
   return {
     symbols: [...symbols, ...reexports],
     summary: topDocComment(content),
@@ -370,7 +376,9 @@ export function extractCode(rel: string, ext: string, content: string): CodeInfo
     idents: ast?.idents,
     // AST call sites when a grammar parsed the file; the conservative regex
     // collector otherwise, so caller indexes exist without the wasm sidecar.
-    calls: ast ? ast.calls : collectCallsRegex(content),
+    // `symbols` (this file's own regex-extracted defs) lets the collector
+    // exclude a definition's own name+line from its call candidates.
+    calls: ast ? ast.calls : collectCallsRegex(content, symbols),
     importedNames: ast?.importedNames,
   };
 }

package/src/lang/common.ts

@@ -66,3 +66,67 @@ const EXT_LANG: Record<string, string> = {
 export function extToLang(ext: string): string {
   return EXT_LANG[ext] ?? "other";
 }
+
+const REEXPORT_EXTS = new Set([".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs"]);
+
+// Barrel re-exports (`export { A, B as C } from './x'`, `export * from './y'`).
+// The line-based lang extractor can't capture multi-name lists, but these ARE
+// the public facade of a module — so list them as exported symbols here.
+//
+// An ALIAS with no `from` clause (`export { b as c }`) renames an in-file
+// declaration — `localSymbols` (already extracted by the AST or regex tier)
+// lets us resolve `b` and mirror ITS kind onto `c` (e.g. "function"), so the
+// alias reads as the real symbol it is rather than the generic "reexport".
+// A true cross-module re-export (`export { b as c } from "./mod"`) has no
+// local `b` to resolve — and an alias the local pass genuinely can't see
+// (destructured/ambient/etc.) falls back the same way — both keep "reexport".
+//
+// Shared by extractCode (extract/code.ts) AND the standalone extractSymbols
+// (lang/registry.ts) — ultradoc and other direct extractSymbols consumers hit
+// the same barrels a repo scan does, so both entry points must agree; this is
+// the one place the alias-mirroring logic lives, reused rather than
+// reimplemented on either side.
+export function extractReexports(rel: string, content: string, localSymbols: CodeSymbol[]): CodeSymbol[] {
+  if (!REEXPORT_EXTS.has(rel.slice(rel.lastIndexOf(".")))) return [];
+  const lang = /\.(ts|tsx|mts|cts)$/.test(rel) ? "typescript" : "javascript";
+  const out: CodeSymbol[] = [];
+  const seen = new Set<string>();
+  const lineAt = (idx: number): number => content.slice(0, idx).split(/\r?\n/).length;
+  const localKindOf = new Map<string, string>();
+  for (const s of localSymbols) if (!localKindOf.has(s.name)) localKindOf.set(s.name, s.kind);
+
+  const named = /export\s*\{([\s\S]*?)\}\s*(?:from\s*['"]([^'"]+)['"])?\s*;?/g;
+  let m: RegExpExecArray | null;
+  while ((m = named.exec(content)) && out.length < 60) {
+    const from = m[2];
+    for (const part of m[1]!.split(",")) {
+      const p = part.trim().replace(/^type\s+/, "");
+      const as = /^(\S+)\s+as\s+([A-Za-z_$][\w$]*)$/.exec(p);
+      const orig = as ? as[1]! : p;
+      const name = as ? as[2]! : p;
+      if (!/^[A-Za-z_$][\w$]*$/.test(name) || name === "default" || seen.has(name)) continue;
+      seen.add(name);
+      const mirroredKind = !from ? localKindOf.get(orig) : undefined;
+      out.push({
+        name, kind: mirroredKind ?? "reexport", file: rel, line: lineAt(m.index),
+        signature: from ? `export { ${name} } from "${from}"` : `export { ${name} }`,
+        exported: true, lang,
+      });
+    }
+  }
+
+  const star = /export\s*\*\s*(?:as\s+([A-Za-z_$][\w$]*)\s+)?from\s*['"]([^'"]+)['"]/g;
+  while ((m = star.exec(content)) && out.length < 60) {
+    const ns = m[1];
+    const from = m[2]!;
+    const key = "*" + (ns ?? from);
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push({
+      name: ns ?? `* (${from})`, kind: ns ? "reexport" : "reexport-all", file: rel,
+      line: lineAt(m.index), signature: `export * ${ns ? `as ${ns} ` : ""}from "${from}"`,
+      exported: true, lang,
+    });
+  }
+  return out;
+}

package/src/lang/registry.ts

@@ -1,5 +1,5 @@
 import type { CodeSymbol } from "../types.js";
-import { extToLang } from "./common.js";
+import { extToLang, extractReexports } from "./common.js";
 import { jsTs } from "./js-ts.js";
 import { python } from "./python.js";
 import { go } from "./go.js";
@@ -35,14 +35,29 @@ for (const e of EXTRACTORS) for (const ext of e.exts) BY_EXT.set(ext, e);
 
 // Extract declared symbols from one file. Returns [] for languages without a
 // dedicated extractor (their content is still fully searchable via ripgrep).
+//
+// Also mirrors extractCode's barrel re-export/alias pass (extractReexports):
+// a direct extractSymbols consumer (ultradoc, or any tool that doesn't go
+// through extractCode) hits the same `export { a, b as c }` barrels a repo
+// scan does, and should see the same alias-mirrors-b's-own-kind symbols —
+// not the bare "reexport" kind extractReexports falls back to when there's
+// no local declaration to mirror. extractCode already runs extractReexports
+// itself and filters by name, so this can't double-add: whichever entry point
+// adds "c" first wins, the other's re-run is a no-op duplicate filtered out.
 export function extractSymbols(rel: string, ext: string, content: string): CodeSymbol[] {
   const extractor = BY_EXT.get(ext);
-  if (!extractor) return [];
-  try {
-    return extractor.extract(rel, content);
-  } catch {
-    return [];
+  let symbols: CodeSymbol[];
+  if (!extractor) symbols = [];
+  else {
+    try {
+      symbols = extractor.extract(rel, content);
+    } catch {
+      symbols = [];
+    }
   }
+  const known = new Set(symbols.map((s) => s.name));
+  const reexports = extractReexports(rel, content, symbols).filter((s) => !known.has(s.name));
+  return reexports.length ? [...symbols, ...reexports] : symbols;
 }
 
 // Human-readable language label for an extension (used for the language

package/src/mcp.ts

@@ -28,6 +28,7 @@ import { checkRules, parseRules } from "./rules.js";
 import { EMBED_VERSION, resolveEmbedModelDir, loadEmbedModel } from "./embed/model.js";
 import { buildEmbeddingIndex } from "./embed/index.js";
 import { searchSemantic } from "./embed/search.js";
+import { resolveEmbedEndpoint, buildEndpointIndex, encodeQueryViaEndpoint, probeEndpoint } from "./embed/endpoint.js";
 
 interface RpcRequest {
   jsonrpc: "2.0";
@@ -285,7 +286,7 @@ const TOOLS = [
         semantic: {
           type: "boolean",
           description:
-            "RRF-fuse the deterministic static-embedding tier with lexical when a model asset is present (default false; silently lexical-only when no model)",
+            "RRF-fuse an embedding tier with lexical (default false). Precedence: the HTTP endpoint (CODEINDEX_EMBED_ENDPOINT) if set, else a local static model. Degrades silently to lexical-only when neither is available/reachable — see embed_status.",
         },
       },
       required: ["repo", "query"],
@@ -294,7 +295,7 @@ const TOOLS = [
   {
     name: "embed_status",
     description:
-      "Report the deterministic static-embedding tier: whether a model asset is resolved (opt-in, never shipped in the package), its modelId/dim, EMBED_VERSION, and any configured HTTP endpoint. Use to check whether `search` with semantic:true will actually fuse embeddings or degrade to lexical.",
+      "Report the embedding tier: the effective mode (none/static/endpoint; endpoint > static model), the resolved model (opt-in, never shipped in the package) with its modelId/dim, EMBED_VERSION, and the configured HTTP endpoint with its reachability. Use to check whether `search` with semantic:true will fuse embeddings or degrade to lexical.",
     inputSchema: { type: "object", properties: { ...repoProp }, required: ["repo"] },
   },
   {
@@ -320,7 +321,7 @@ function strArray(v: unknown): string[] | undefined {
   return Array.isArray(v) && v.every((x) => typeof x === "string") && v.length ? (v as string[]) : undefined;
 }
 
-function callTool(name: string, args: Record<string, unknown>): string {
+async function callTool(name: string, args: Record<string, unknown>): Promise<string> {
   const repo = str(args.repo);
   if (!repo) throw new Error("`repo` is required (absolute path to the repository root)");
   const scanOpts = { scope: str(args.scope), include: strArray(args.include), exclude: strArray(args.exclude) };
@@ -458,6 +459,18 @@ function callTool(name: string, args: Record<string, unknown>): string {
     const limit = typeof args.limit === "number" ? args.limit : undefined;
     const fuzzy = typeof args.fuzzy === "boolean" ? args.fuzzy : undefined;
     if (args.semantic === true) {
+      const endpoint = resolveEmbedEndpoint();
+      if (endpoint) {
+        // Rich tier — endpoint takes PRECEDENCE over a local static model. An
+        // unreachable/malformed endpoint degrades to lexical (no throw).
+        try {
+          const index = await buildEndpointIndex(scan);
+          const queryVec = await encodeQueryViaEndpoint(query);
+          return JSON.stringify(searchSemantic(scan, query, index, { queryVec, limit, fuzzy }), null, 2);
+        } catch {
+          return JSON.stringify(searchIndex(scan, query, { limit, fuzzy }), null, 2);
+        }
+      }
       const modelDir = resolveEmbedModelDir(repo);
       const model = modelDir ? loadEmbedModel(modelDir) : undefined;
       if (model) {
@@ -471,17 +484,18 @@ function callTool(name: string, args: Record<string, unknown>): string {
   if (name === "embed_status") {
     const modelDir = resolveEmbedModelDir(repo);
     const model = modelDir ? loadEmbedModel(modelDir) : undefined;
-    return JSON.stringify(
-      {
-        embedVersion: EMBED_VERSION,
-        model: model
-          ? { present: true, dir: modelDir, modelId: model.modelId, dim: model.dim, vocabSize: model.vocabSize }
-          : { present: false },
-        endpoint: process.env.CODEINDEX_EMBED_ENDPOINT ?? null,
-      },
-      null,
-      2,
-    );
+    const endpoint = resolveEmbedEndpoint();
+    const mode: "none" | "static" | "endpoint" = endpoint ? "endpoint" : model ? "static" : "none";
+    const status: Record<string, unknown> = {
+      embedVersion: EMBED_VERSION,
+      mode,
+      model: model
+        ? { present: true, dir: modelDir, modelId: model.modelId, dim: model.dim, vocabSize: model.vocabSize }
+        : { present: false },
+      endpoint: endpoint ?? null,
+    };
+    if (endpoint) status.endpointReachable = await probeEndpoint(endpoint);
+    return JSON.stringify(status, null, 2);
   }
   if (name === "check_rules") {
     const rules = parseRules(args.rules); // throws a descriptive error on a malformed payload
@@ -512,10 +526,10 @@ export async function runMcpServer(): Promise<void> {
     // JSON-RPC 2.0 batch: answer each member (a batching client would
     // otherwise hang forever on a silently dropped array).
     const requests = Array.isArray(parsed) ? (parsed as RpcRequest[]) : [parsed as RpcRequest];
-    for (const req of requests) handle(req);
+    for (const req of requests) await handle(req);
   }
 
-  function handle(req: RpcRequest): void {
+  async function handle(req: RpcRequest): Promise<void> {
     if (req.id === undefined || req.id === null) return; // notification — no response
 
     try {
@@ -537,7 +551,7 @@ export async function runMcpServer(): Promise<void> {
         const name = str(params.name) ?? "";
         const args = (params.arguments ?? {}) as Record<string, unknown>;
         try {
-          const text = callTool(name, args);
+          const text = await callTool(name, args);
           send({ id: req.id, result: { content: [{ type: "text", text }] } });
         } catch (e) {
           send({

package/src/types.ts

@@ -1,7 +1,7 @@
 // Single source of truth for the engine version the bundle reports. Kept in
 // lockstep with package.json by the release pipeline. Do not edit by hand
 // outside a release.
-export const ENGINE_VERSION = "2.10.0";
+export const ENGINE_VERSION = "2.11.1";
 
 // Bumped whenever the on-disk artifact shape changes, so a consumer can reject
 // an index written by an incompatible engine instead of misreading it. The
@@ -24,8 +24,15 @@ export const SCHEMA_VERSION = 4;
 // (the immediate receiver of a qualified call, both tiers) and JS/TS export
 // parity with ultradoc (CJS `exports.foo =` / `module.exports = {…}` named
 // exports, `export { a, b as c }` local marking, anonymous `export default`
-// named after the file stem, `export default Foo` marking the declaration).
-export const EXTRACTOR_VERSION = 6;
+// named after the file stem, `export default Foo` marking the declaration);
+// v7 makes an export-alias symbol (`export { b as c }`) mirror the aliased
+// local declaration's own kind (e.g. "function") instead of the generic
+// "reexport" when it resolves in-file. v8 fixes the C/C++ regex tier
+// reporting a function DEFINITION as a call to itself (`void load(void) {`
+// yielding a spurious call `load@<defline>`, found during the ultrasec
+// consumer migration) by excluding call candidates whose name+line match one
+// of the file's own extracted symbols.
+export const EXTRACTOR_VERSION = 8;
 
 // How a file is classified. `code` gets symbol/import extraction; `doc` gets
 // link/heading extraction; the rest are catalogued but not deeply parsed.