@functional-systems/srcx 0.1.0-dev.19 → 0.1.0-dev.21

package/runtime/qwen.py

@@ -4,6 +4,7 @@ import json
 import math
 import os
 import sys
+from pathlib import Path
 
 MODEL = "Qwen/Qwen3-Reranker-0.6B"
 REVISION = "e61197ed45024b0ed8a2d74b80b4d909f1255473"
@@ -14,6 +15,19 @@ PREFIX = (
 )
 SUFFIX = "<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n"
 INSTRUCTION = "Given a code search query, retrieve code passages that answer the query."
+CACHE_FILES = ["config.json", "generation_config.json", "model.safetensors",
+               "tokenizer.json", "tokenizer_config.json", "vocab.json", "merges.txt"]
+
+
+class WorkerFailure(ValueError):
+    """Exit codes: 20 imports, 21 cache, 22 device, 23 pair tokens, 24 request bytes.
+
+    The Node caller maps codes to messages; exception text never crosses the boundary.
+    """
+
+    def __init__(self, code):
+        super().__init__("Qwen worker failure")
+        self.code = code
 
 
 def require(condition, message):
@@ -25,7 +39,8 @@ def encode(tokenizer, query, text):
     body = f"<Instruct>: {INSTRUCTION}\n<Query>: {query}\n<Document>: {text}"
     ids = sum((tokenizer.encode(p, add_special_tokens=False)
                for p in (PREFIX, body, SUFFIX)), [])
-    require(len(ids) <= 8192, "Pair exceeds 8192 tokens; no truncation")
+    if len(ids) > 8192:
+        raise WorkerFailure(23)
     return ids
 
 
@@ -45,22 +60,33 @@ def validate(body):
 
 def score(body):
     os.environ["HF_HUB_OFFLINE"] = "1"
-    from huggingface_hub import snapshot_download
-    from transformers import AutoTokenizer, AutoModelForCausalLM
-    import torch
-
-    path = snapshot_download(MODEL, revision=REVISION, local_files_only=True,
-                             allow_patterns=["config.json", "generation_config.json", "model.safetensors",
-                                             "tokenizer.json", "tokenizer_config.json", "vocab.json", "merges.txt"])
+    device = os.environ.get("SRCX_RERANK_DEVICE", "auto")
+    if device not in ("auto", "cpu", "mps", "cuda"):
+        raise WorkerFailure(22)
+    try:
+        from huggingface_hub import snapshot_download
+        from huggingface_hub.errors import LocalEntryNotFoundError
+        from transformers import AutoTokenizer, AutoModelForCausalLM
+        import torch
+    except ImportError:
+        raise WorkerFailure(20) from None
+    if device == "auto":
+        device = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu"
+    elif ((device == "cuda" and not torch.cuda.is_available()) or
+          (device == "mps" and not torch.backends.mps.is_available())):
+        raise WorkerFailure(22)
+    try:
+        path = snapshot_download(MODEL, revision=REVISION, local_files_only=True,
+                                 allow_patterns=CACHE_FILES)
+    except LocalEntryNotFoundError:
+        raise WorkerFailure(21) from None
+    if not all((Path(path) / name).is_file() for name in CACHE_FILES):
+        raise WorkerFailure(21)
     tokenizer = AutoTokenizer.from_pretrained(path, local_files_only=True, trust_remote_code=False)
     # Reuse exact duplicate code within this query, retaining every candidate ID.
     encoded = {c["text"]: None for c in body["candidates"]}
     for text in encoded:
         encoded[text] = encode(tokenizer, body["query"], text)
-    device = os.environ.get("SRCX_RERANK_DEVICE", "auto")
-    require(device in ("auto", "cpu", "mps", "cuda"), "Invalid device")
-    if device == "auto":
-        device = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu"
     model = AutoModelForCausalLM.from_pretrained(
         path, local_files_only=True, trust_remote_code=False,
         dtype=torch.float32, attn_implementation="sdpa",
@@ -83,15 +109,24 @@ def score(body):
 
 def main():
     raw = sys.stdin.buffer.read(4 * 1024 * 1024 + 1)
-    require(len(raw) <= 4 * 1024 * 1024, "Request exceeds 4 MiB")
+    if len(raw) > 4 * 1024 * 1024:
+        raise WorkerFailure(24)
     body = validate(json.loads(raw))
     print(json.dumps(score(body), allow_nan=False))
 
 
-if __name__ == "__main__":
+def run():
     try:
         main()
+    except WorkerFailure as error:
+        # The Node caller maps these codes to fixed, actionable messages.
+        return error.code
     except Exception:
         # Libraries may include source text or environment values in exceptions.
         print("Qwen worker failed; verify setup, device and input size.", file=sys.stderr)
-        sys.exit(1)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(run())

package/src/build.ts

@@ -46,7 +46,7 @@ export const PRESET: Preset = {
   maxFileBytes: 1024 * 1024,
   embedding: null,
 };
-/** Opt-in managed preset; PRESET stays byte-for-byte compatible with lexical corpora. */
+/** Embedding variants share the same chunking behavior. */
 export const MANAGED_PRESET: Preset = {
   ...PRESET,
   embedding: {

package/src/chunk.ts

@@ -1,3 +1,4 @@
+import { fileURLToPath } from "node:url";
 import { createRequire } from "node:module";
 import { Parser, Language, type Node } from "web-tree-sitter";
 import { getEncoding } from "js-tiktoken";
@@ -8,9 +9,11 @@ export function tokens(text: string): number {
   return encoding.encode(text, [], []).length;
 }
 export const CHUNKER = {
-  version: 1,
+  version: 2,
   parser: "web-tree-sitter@0.25.10",
   grammars: "tree-sitter-wasms@0.1.13",
+  sqlGrammar:
+    "tree-sitter-wasm@2.0.2/sql/b77530893b1dd6d1d4814eacf34d25bcbe4a12ec9a25a8c45b320d447265f42b",
   tokenizer: "js-tiktoken@1.0.21/cl100k_base",
   targetTokens: 800,
   maxTokens: 1500,
@@ -46,17 +49,42 @@ async function language(name: string): Promise<Language> {
   let l = languages.get(name);
   if (!l) {
     l = await Language.load(
-      require.resolve(`tree-sitter-wasms/out/tree-sitter-${name}.wasm`),
+      name === "sql"
+        ? fileURLToPath(
+            new URL(
+              "../runtime/grammars/tree-sitter-sql.wasm",
+              import.meta.url,
+            ),
+          )
+        : require.resolve(
+            `tree-sitter-wasms/out/tree-sitter-${name === "shell" ? "bash" : name}.wasm`,
+          ),
     );
     languages.set(name, l);
   }
   return l;
 }
 export function detectLanguage(path: string): string {
+  if (path.endsWith(".C")) return "cpp";
   const ext = path.split(".").at(-1)?.toLowerCase();
   return (
     (
       {
+        py: "python",
+        pyi: "python",
+        go: "go",
+        rs: "rust",
+        c: "c",
+        h: "c",
+        cc: "cpp",
+        cpp: "cpp",
+        cxx: "cpp",
+        hh: "cpp",
+        hpp: "cpp",
+        hxx: "cpp",
+        sh: "shell",
+        bash: "shell",
+        sql: "sql",
         java: "java",
         ts: "typescript",
         tsx: "tsx",
@@ -102,6 +130,278 @@ function scopeName(n: Node): string | undefined {
   }
   return names.length ? names.join(".") : undefined;
 }
+
+/** Keep complete functions (including decorators) before token-bound splitting. */
+function pythonUnits(root: Node): Unit[] {
+  const units: Unit[] = [];
+  function visit(n: Node, scope?: string): void {
+    const definition = n.childForFieldName("definition") ?? n;
+    const body = definition.childForFieldName("body");
+    const symbol = definition.childForFieldName("name")?.text.slice(0, 200);
+    const unit: Unit = {
+      start: n.startIndex,
+      end: n.endIndex,
+      kind: classify(definition.type),
+      scope,
+      symbol,
+    };
+    if (definition.type === "class_definition" && body) {
+      units.push({ ...unit, end: body.startIndex });
+      const nested = [scope, symbol].filter(Boolean).join(".");
+      for (const child of body.namedChildren) if (child) visit(child, nested);
+    } else {
+      if (definition.type === "function_definition" && body)
+        unit.signature = definition.text
+          .slice(0, body.startIndex - definition.startIndex)
+          .trim()
+          .slice(0, 240);
+      if (
+        definition.type === "expression_statement" &&
+        definition.firstNamedChild?.type === "string"
+      )
+        unit.kind = "documentation";
+      units.push(unit);
+    }
+  }
+  for (const n of root.namedChildren) if (n) visit(n);
+  return units;
+}
+
+function goUnits(root: Node): Unit[] {
+  const units: Unit[] = [];
+  for (const n of root.namedChildren) {
+    if (!n) continue;
+    if (n.type === "type_declaration") {
+      // Include the type keyword and grouped declaration delimiters in coverage.
+      const specs = n.namedChildren.filter(
+        (node): node is Node => node !== null,
+      );
+      for (const [i, spec] of specs.entries())
+        units.push({
+          start: i === 0 ? n.startIndex : spec.startIndex,
+          end: i === specs.length - 1 ? n.endIndex : spec.endIndex,
+          kind: classify(spec.type),
+          symbol: spec.childForFieldName("name")?.text.slice(0, 200),
+        });
+      continue;
+    }
+    const unit: Unit = {
+      start: n.startIndex,
+      end: n.endIndex,
+      kind: classify(n.type),
+      symbol: n.childForFieldName("name")?.text.slice(0, 200),
+    };
+    let receiver = n
+      .childForFieldName("receiver")
+      ?.firstNamedChild?.childForFieldName("type");
+    if (receiver?.type === "pointer_type") receiver = receiver.firstNamedChild;
+    if (receiver?.type === "generic_type")
+      receiver = receiver.childForFieldName("type");
+    if (receiver) unit.scope = receiver.text.slice(0, 160);
+    const body = n.childForFieldName("body");
+    if (unit.kind === "function")
+      unit.signature = n.text
+        .slice(0, body ? body.startIndex - n.startIndex : undefined)
+        .trim()
+        .slice(0, 240);
+    units.push(unit);
+  }
+  return units;
+}
+
+/** Walk declarators rather than parameter/type names (including pointer returns). */
+function declaratorName(n: Node | null): string | undefined {
+  if (!n) return undefined;
+  if (
+    /^(identifier|field_identifier|type_identifier|qualified_identifier|destructor_name|operator_name)$/.test(
+      n.type,
+    )
+  )
+    return n.text;
+  return declaratorName(
+    n.childForFieldName("declarator") ??
+      (n.type === "parenthesized_declarator" ? n.firstNamedChild : null),
+  );
+}
+
+function cUnits(root: Node): Unit[] {
+  const units: Unit[] = [];
+  function visit(n: Node, scope?: string, start = n.startIndex): void {
+    if (n.type === "template_declaration") {
+      const declaration = n.namedChildren
+        .filter((c): c is Node => !!c && c.type !== "template_parameter_list")
+        .at(-1);
+      if (declaration) {
+        visit(declaration, scope, start);
+        return;
+      }
+    }
+    const body = n.childForFieldName("body");
+    const symbol =
+      n.childForFieldName("name")?.text ??
+      declaratorName(n.childForFieldName("declarator"));
+    const unit: Unit = {
+      start,
+      end: n.endIndex,
+      kind: n.type === "preproc_include" ? "imports" : classify(n.type),
+      scope,
+      symbol: symbol?.slice(0, 200),
+    };
+    if (body && /^(declaration_list|field_declaration_list)$/.test(body.type)) {
+      const nested = [scope, symbol].filter(Boolean).join("::") || undefined;
+      units.push({ ...unit, end: body.startIndex + 1 });
+      for (const child of body.namedChildren) if (child) visit(child, nested);
+      units.push({
+        start: body.endIndex - 1,
+        end: n.endIndex,
+        kind: "structural",
+        scope: nested,
+      });
+    } else if (/^preproc_(if|ifdef|elif|else|elifdef)$/.test(n.type)) {
+      // Preserve directives and both branches without evaluating the build configuration.
+      const condition =
+        n.childForFieldName("condition") ?? n.childForFieldName("name");
+      const children = n.namedChildren.filter(
+        (c): c is Node => !!c && c.id !== condition?.id,
+      );
+      if (!children.length) {
+        units.push(unit);
+        return;
+      }
+      units.push({ ...unit, end: children[0]!.startIndex });
+      for (const child of children) visit(child, scope);
+      const end = children.at(-1)!.endIndex;
+      if (end < n.endIndex)
+        units.push({ start: end, end: n.endIndex, kind: "structural", scope });
+    } else {
+      if (n.type === "function_definition")
+        unit.signature = n.text
+          .slice(0, body ? body.startIndex - n.startIndex : undefined)
+          .trim()
+          .slice(0, 240);
+      units.push(unit);
+    }
+  }
+  for (const n of root.namedChildren) if (n) visit(n);
+  return units;
+}
+
+function shellUnits(root: Node): Unit[] {
+  return root.namedChildren
+    .filter((n): n is Node => !!n)
+    .map((n) => {
+      const body = n.childForFieldName("body");
+      return {
+        start: n.startIndex,
+        end: n.endIndex,
+        kind: classify(n.type),
+        symbol:
+          n.type === "function_definition"
+            ? n.childForFieldName("name")?.text.slice(0, 200)
+            : undefined,
+        signature:
+          n.type === "function_definition"
+            ? n.text
+                .slice(0, body ? body.startIndex - n.startIndex : undefined)
+                .trim()
+                .slice(0, 240)
+            : undefined,
+      };
+    });
+}
+
+function sqlUnits(root: Node): Unit[] {
+  return root.namedChildren
+    .filter((n): n is Node => !!n)
+    .map((n) => {
+      const declaration = n.firstNamedChild;
+      // Object names describe DDL declarations, not every table referenced by a query.
+      const object = declaration?.type.startsWith("create_")
+        ? declaration.namedChildren.find((c) => c?.type === "object_reference")
+        : undefined;
+      return {
+        start: n.startIndex,
+        end: n.nextSibling?.type === ";" ? n.nextSibling.endIndex : n.endIndex,
+        kind: n.type === "comment" ? "documentation" : "statement",
+        symbol: object?.childForFieldName("name")?.text.slice(0, 200),
+        scope: object?.childForFieldName("schema")?.text.slice(0, 160),
+      };
+    });
+}
+
+function rustUnits(root: Node): Unit[] {
+  const units: Unit[] = [];
+  function visit(container: Node, scope?: string): void {
+    let prefix: number | undefined;
+    for (const n of container.namedChildren) {
+      if (!n) continue;
+      // Outer attributes/doc comments belong to the following item, not a chunk
+      // containing only cfg/derive metadata. Inner module docs remain separate.
+      if (
+        n.type === "attribute_item" ||
+        /^(\/\/\/(?!\/)|\/\*\*(?!\*))/.test(n.text)
+      ) {
+        prefix ??= n.startIndex;
+        continue;
+      }
+      if (prefix !== undefined && n.type.endsWith("comment")) continue;
+      const body = n.childForFieldName("body");
+      const name = n.childForFieldName("name")?.text;
+      const type = n.childForFieldName("type")?.text;
+      const trait = n.childForFieldName("trait")?.text;
+      const unit: Unit = {
+        start: prefix ?? n.startIndex,
+        end: n.endIndex,
+        kind:
+          n.type === "use_declaration" || n.type === "extern_crate_declaration"
+            ? "imports"
+            : classify(n.type),
+        scope,
+        symbol: (
+          name ??
+          (n.type === "impl_item" ? type : n.childForFieldName("macro")?.text)
+        )?.slice(0, 200),
+      };
+      prefix = undefined;
+      if (body?.type === "declaration_list") {
+        const owner =
+          n.type === "impl_item"
+            ? [type, trait].filter(Boolean).join(" as ")
+            : name;
+        const nested =
+          [scope, owner].filter(Boolean).join("::").slice(0, 160) || undefined;
+        units.push({ ...unit, end: body.startIndex + 1 });
+        visit(body, nested);
+        // Keep closing delimiters with their container, including empty bodies.
+        units.push({
+          start: body.endIndex - 1,
+          end: n.endIndex,
+          kind: "structural",
+          scope: nested,
+        });
+      } else {
+        if (unit.kind === "function")
+          unit.signature = n.text
+            .slice(0, body ? body.startIndex - n.startIndex : undefined)
+            .trim()
+            .slice(0, 240);
+        units.push(unit);
+      }
+    }
+    if (prefix !== undefined)
+      units.push({
+        start: prefix,
+        end:
+          container.type === "declaration_list"
+            ? container.endIndex - 1
+            : container.endIndex,
+        kind: "documentation",
+        scope,
+      });
+  }
+  visit(root);
+  return units;
+}
 /** Bound strings without slicing a surrogate pair; prefer line ends where possible. */
 function splitUnit(
   source: string,
@@ -198,7 +498,21 @@ export async function chunk(
   if (mode === "window") {
     status = "window-baseline";
     units = [{ start: 0, end: source.length, kind: "fallback" }];
-  } else if (["java", "typescript", "tsx", "javascript"].includes(lang)) {
+  } else if (
+    [
+      "java",
+      "typescript",
+      "tsx",
+      "javascript",
+      "python",
+      "go",
+      "rust",
+      "c",
+      "cpp",
+      "shell",
+      "sql",
+    ].includes(lang)
+  ) {
     const grammar = await language(lang);
     const parser = new Parser();
     let tree: ReturnType<Parser["parse"]> = null;
@@ -233,7 +547,13 @@ export async function chunk(
             for (const child of n.namedChildren) if (child) visit(child);
           } else if (n.endIndex > n.startIndex) units.push(u);
         }
-        visit(tree.rootNode);
+        if (lang === "python") units = pythonUnits(tree.rootNode);
+        else if (lang === "go") units = goUnits(tree.rootNode);
+        else if (lang === "rust") units = rustUnits(tree.rootNode);
+        else if (lang === "c" || lang === "cpp") units = cUnits(tree.rootNode);
+        else if (lang === "shell") units = shellUnits(tree.rootNode);
+        else if (lang === "sql") units = sqlUnits(tree.rootNode);
+        else visit(tree.rootNode);
         // Attach every gap (punctuation, BOM, comments, whitespace) without altering source bytes.
         let cursor = 0;
         const covered: Unit[] = [];
@@ -272,6 +592,9 @@ export async function chunk(
     const prev = merged.at(-1);
     if (
       prev &&
+      lang !== "sql" &&
+      (!["python", "go", "rust", "c", "cpp", "shell"].includes(lang) ||
+        (!prev.symbol && !u.symbol)) &&
       prev.kind !== "function" &&
       u.kind === prev.kind &&
       u.scope === prev.scope &&

package/src/rerank.ts

@@ -4,6 +4,14 @@ import { invariant } from "./common.js";
 
 export const QWEN_MODEL = "Qwen/Qwen3-Reranker-0.6B";
 export const QWEN_REVISION = "e61197ed45024b0ed8a2d74b80b4d909f1255473";
+// Reserved worker exit codes; never interpret or forward library stderr.
+const WORKER_FAILURES: Record<number, string> = {
+  20: "Qwen Python dependencies are missing or incompatible. Install runtime/requirements.txt with the Python selected by SRCX_RERANK_PYTHON.",
+  21: "Qwen model cache is missing or incomplete. Download the pinned model revision from README and check HF_HUB_CACHE.",
+  22: "Qwen device is invalid or unavailable. Set SRCX_RERANK_DEVICE to auto or cpu, or a supported mps/cuda device.",
+  23: "Qwen query/chunk pair exceeds 8,192 tokens. Shorten the query or use search without --rerank; source is not truncated.",
+  24: "Qwen request exceeds 4 MiB. Reduce --candidates or shorten the query.",
+};
 export type Candidate = { id: string; text: string };
 export type RerankOptions = {
   rerank?: "qwen";
@@ -49,7 +57,7 @@ export async function qwenScores(query: string, candidates: Candidate[]) {
   const input = JSON.stringify({ query, candidates });
   invariant(
     Buffer.byteLength(input) <= 4 * 1024 * 1024,
-    "Reranker input exceeds 4 MiB.",
+    "Reranker input exceeds 4 MiB. Reduce --candidates or shorten the query; no fallback was applied.",
   );
   const python = process.env.SRCX_RERANK_PYTHON;
   invariant(
@@ -74,13 +82,21 @@ export async function qwenScores(query: string, candidates: Candidate[]) {
         },
       },
       (error, output) => {
-        if (error)
-          reject(
-            new Error(
-              "Qwen reranking failed or exceeded 120 seconds. Check Python dependencies, cached model and SRCX_RERANK_DEVICE; no fallback was applied.",
-            ),
-          );
-        else resolve(output);
+        if (!error) return resolve(output);
+        let message =
+          "Qwen reranking failed. Check Python dependencies, cached model and SRCX_RERANK_DEVICE.";
+        if (error.code === "ENOENT")
+          message =
+            "Qwen Python executable was not found. Set SRCX_RERANK_PYTHON to an existing Python executable.";
+        else if (error.code === "EACCES")
+          message =
+            "Qwen Python executable cannot be started. Check execute permissions for SRCX_RERANK_PYTHON.";
+        else if (error.killed && error.signal === "SIGKILL" && !error.code)
+          message =
+            "Qwen reranking exceeded 120 seconds. Reduce --candidates or use an available faster device via SRCX_RERANK_DEVICE.";
+        else if (typeof error.code === "number")
+          message = WORKER_FAILURES[error.code] ?? message;
+        reject(new Error(`${message} No fallback was applied.`));
       },
     );
     // An early worker exit can close stdin before Node finishes writing.