@cue-dev/retrieval-core 0.1.3

package/package.json

@@ -0,0 +1,37 @@
+{
+  "name": "@cue-dev/retrieval-core",
+  "version": "0.1.3",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "dependencies": {
+    "@anthropic-ai/claude-agent-sdk": "^0.2.42",
+    "@anthropic-ai/sdk": "^0.55.0",
+    "@cue-dev/contracts": "0.1.1",
+    "@cue-dev/data-plane": "0.1.2",
+    "@cue-dev/observability": "0.1.1",
+    "tree-sitter": "^0.22.4",
+    "tree-sitter-go": "^0.23.4",
+    "tree-sitter-javascript": "^0.25.0",
+    "tree-sitter-javascript-v023": "npm:tree-sitter-javascript@0.23.1",
+    "tree-sitter-java": "^0.23.5",
+    "tree-sitter-python": "^0.25.0",
+    "tree-sitter-python-v023": "npm:tree-sitter-python@0.23.6",
+    "tree-sitter-rust": "^0.23.3",
+    "tree-sitter-typescript": "^0.23.2"
+  },
+  "devDependencies": {
+    "pg-mem": "^3.0.5",
+    "tree-sitter-go-v025": "npm:tree-sitter-go@0.25.0",
+    "tree-sitter-javascript-v025": "npm:tree-sitter-javascript@0.25.0",
+    "tree-sitter-python-v025": "npm:tree-sitter-python@0.25.0",
+    "tree-sitter-v025": "npm:tree-sitter@0.25.0",
+    "web-tree-sitter": "^0.26.5"
+  }
+}

package/scripts/poc-node-parser-host.cjs

@@ -0,0 +1,105 @@
+"use strict";
+
+const readline = require("node:readline");
+const Parser = require("tree-sitter");
+const JavaScript = require("tree-sitter-javascript-v023");
+const Java = require("tree-sitter-java");
+const Python = require("tree-sitter-python-v023");
+const Rust = require("tree-sitter-rust");
+
+const parserByLanguage = new Map();
+
+function initParser(language, grammarModule) {
+  const parser = new Parser();
+  parser.setLanguage(grammarModule);
+  parserByLanguage.set(language, parser);
+}
+
+const initStart = performance.now();
+initParser("javascript", JavaScript);
+initParser("python", Python);
+initParser("rust", Rust);
+initParser("java", Java);
+const initMs = performance.now() - initStart;
+
+if (process.argv[2] === "--batch") {
+  const language = String(process.argv[3] ?? "");
+  const iterations = Number(process.argv[4] ?? "1");
+  const source = Buffer.from(String(process.argv[5] ?? ""), "base64").toString("utf8");
+  const parser = parserByLanguage.get(language);
+  if (!parser) {
+    process.stdout.write(`${JSON.stringify({ ok: false, error: `unsupported language: ${language}` })}\n`);
+    process.exit(0);
+  }
+
+  const parseStart = performance.now();
+  let root = null;
+  for (let i = 0; i < iterations; i += 1) {
+    const tree = parser.parse(source);
+    root = tree?.rootNode?.type ?? null;
+  }
+  const totalParseMs = performance.now() - parseStart;
+  process.stdout.write(
+    `${JSON.stringify({
+      ok: true,
+      language,
+      init_ms: Number(initMs.toFixed(3)),
+      parse_avg_ms: Number((totalParseMs / Math.max(1, iterations)).toFixed(3)),
+      parse_total_ms: Number(totalParseMs.toFixed(3)),
+      root
+    })}\n`
+  );
+  process.exit(0);
+}
+
+process.stdout.write(`${JSON.stringify({ type: "ready", init_ms: Number(initMs.toFixed(3)) })}\n`);
+
+const rl = readline.createInterface({
+  input: process.stdin,
+  terminal: false
+});
+
+rl.on("line", (line) => {
+  if (!line.trim()) {
+    return;
+  }
+  let payload;
+  try {
+    payload = JSON.parse(line);
+  } catch (error) {
+    process.stdout.write(
+      `${JSON.stringify({ ok: false, error: error instanceof Error ? error.message : String(error) })}\n`
+    );
+    return;
+  }
+
+  const parser = parserByLanguage.get(payload.language);
+  if (!parser) {
+    process.stdout.write(
+      `${JSON.stringify({ id: payload.id, ok: false, error: `unsupported language: ${String(payload.language)}` })}\n`
+    );
+    return;
+  }
+
+  try {
+    const start = performance.now();
+    const tree = parser.parse(String(payload.source ?? ""));
+    const parseMs = performance.now() - start;
+    process.stdout.write(
+      `${JSON.stringify({
+        id: payload.id,
+        ok: true,
+        parse_ms: Number(parseMs.toFixed(3)),
+        root: tree?.rootNode?.type ?? null
+      })}\n`
+    );
+  } catch (error) {
+    process.stdout.write(
+      `${JSON.stringify({
+        id: payload.id,
+        ok: false,
+        error: error instanceof Error ? error.message : String(error)
+      })}\n`
+    );
+  }
+});

package/scripts/poc-parser-availability-benchmark.ts

@@ -0,0 +1,338 @@
+import { spawnSync } from "node:child_process";
+import { createRequire } from "node:module";
+import { fileURLToPath } from "node:url";
+import Parser from "tree-sitter";
+import JavaScriptV023 from "tree-sitter-javascript-v023";
+import Java from "tree-sitter-java";
+import PythonV023 from "tree-sitter-python-v023";
+import Rust from "tree-sitter-rust";
+import { Language as WebLanguage, Parser as WebParser } from "web-tree-sitter";
+import {
+  __resetChunkingParserStateForTests,
+  buildChunksForFile,
+  getChunkingParserAvailabilitySnapshot
+} from "../src/chunking.js";
+
+type TargetLanguage = "javascript" | "python" | "rust" | "java";
+type BackendId = "baseline_chunking" | "native_v023_bun" | "wasm_web_tree_sitter" | "node_boundary_native";
+
+interface BenchRow {
+  backend: BackendId;
+  language: TargetLanguage;
+  available: boolean;
+  cold_start_ms: number;
+  per_file_ms: number;
+  parse_engine_ms?: number;
+  fallback_reason?: string;
+  notes?: string;
+}
+
+const ITERATIONS = 60;
+const require = createRequire(import.meta.url);
+
+const SAMPLES: Record<TargetLanguage, string> = {
+  javascript: [
+    "import { dep } from './dep';",
+    "",
+    "export function alpha(input) {",
+    "  const value = dep(input);",
+    "  return value + 1;",
+    "}",
+    "",
+    "export class Greeter {",
+    "  greet(name) {",
+    "    return `hello ${name}`;",
+    "  }",
+    "}",
+    "",
+    "export function beta(input) {",
+    "  return dep(input) * 2;",
+    "}"
+  ].join("\n"),
+  python: [
+    "def alpha(x):",
+    "    return x + 1",
+    "",
+    "class Greeter:",
+    "    def greet(self, name):",
+    "        return f\"hello {name}\"",
+    "",
+    "def beta(x):",
+    "    return x * 2"
+  ].join("\n"),
+  rust: [
+    "pub struct Counter {",
+    "  value: i32,",
+    "}",
+    "",
+    "impl Counter {",
+    "  pub fn increment(&mut self, amount: i32) {",
+    "    self.value += amount;",
+    "  }",
+    "}",
+    "",
+    "pub fn alpha(input: i32) -> i32 {",
+    "  input + 1",
+    "}"
+  ].join("\n"),
+  java: [
+    "public class Example {",
+    "  int alpha(int input) {",
+    "    return input + 1;",
+    "  }",
+    "",
+    "  int beta(int input) {",
+    "    return input * 2;",
+    "  }",
+    "}"
+  ].join("\n")
+};
+
+function nowMs(): number {
+  return performance.now();
+}
+
+function formatMs(value: number): number {
+  return Number(value.toFixed(3));
+}
+
+async function benchBaselineChunking(language: TargetLanguage): Promise<BenchRow> {
+  __resetChunkingParserStateForTests();
+  const file = {
+    path:
+      language === "javascript"
+        ? "src/example.js"
+        : language === "python"
+          ? "src/example.py"
+          : language === "rust"
+            ? "src/example.rs"
+            : "src/Example.java",
+    language,
+    content: SAMPLES[language]
+  };
+  const config = {
+    strategy: "language_aware" as const,
+    fallback_strategy: "sliding" as const,
+    target_chunk_tokens: 220,
+    chunk_overlap_tokens: 40,
+    budget_tokenizer: "ranking" as const,
+    boundary_strictness: "legacy" as const,
+    max_chunks_per_file: 300,
+    parse_timeout_ms: 80,
+    enabled_languages: ["javascript", "python", "rust", "java"]
+  };
+  const tokenize = (text: string): string[] => text.split(/\s+/).filter(Boolean);
+
+  let totalMs = 0;
+  let coldMs = 0;
+  let fallbackReason: string | undefined;
+  let available = false;
+
+  for (let i = 0; i < ITERATIONS; i += 1) {
+    const start = nowMs();
+    const result = buildChunksForFile({ file, config, tokenize });
+    const elapsed = nowMs() - start;
+    if (i === 0) {
+      coldMs = elapsed;
+    }
+    totalMs += elapsed;
+    fallbackReason = result.fallback_reason;
+    available = result.strategy === "language_aware" && !result.fallback_reason;
+  }
+
+  return {
+    backend: "baseline_chunking",
+    language,
+    available,
+    cold_start_ms: formatMs(coldMs),
+    per_file_ms: formatMs(totalMs / ITERATIONS),
+    fallback_reason: fallbackReason
+  };
+}
+
+async function benchNativeCompatible(language: TargetLanguage): Promise<BenchRow> {
+  const grammar =
+    language === "javascript"
+      ? JavaScriptV023
+      : language === "python"
+        ? PythonV023
+        : language === "rust"
+          ? Rust
+          : Java;
+  const parser = new Parser();
+  const coldStart = nowMs();
+  parser.setLanguage(grammar as unknown as Parser.Language);
+  const firstTree = parser.parse(SAMPLES[language]);
+  const coldMs = nowMs() - coldStart;
+
+  let parseTotalMs = 0;
+  for (let i = 0; i < ITERATIONS; i += 1) {
+    const start = nowMs();
+    parser.parse(SAMPLES[language]);
+    parseTotalMs += nowMs() - start;
+  }
+
+  return {
+    backend: "native_v023_bun",
+    language,
+    available: Boolean(firstTree?.rootNode),
+    cold_start_ms: formatMs(coldMs),
+    per_file_ms: formatMs(parseTotalMs / ITERATIONS),
+    parse_engine_ms: formatMs(parseTotalMs / ITERATIONS)
+  };
+}
+
+let webTreeSitterRuntimeReady = false;
+
+async function ensureWebTreeSitterRuntime(): Promise<void> {
+  if (webTreeSitterRuntimeReady) {
+    return;
+  }
+  const runtimeWasm = require.resolve("web-tree-sitter/web-tree-sitter.wasm");
+  await WebParser.init({
+    locateFile() {
+      return runtimeWasm;
+    }
+  });
+  webTreeSitterRuntimeReady = true;
+}
+
+async function benchWasm(language: TargetLanguage): Promise<BenchRow> {
+  await ensureWebTreeSitterRuntime();
+  const grammarWasm =
+    language === "javascript"
+      ? require.resolve("tree-sitter-javascript/tree-sitter-javascript.wasm")
+      : language === "python"
+        ? require.resolve("tree-sitter-python/tree-sitter-python.wasm")
+        : language === "rust"
+          ? require.resolve("tree-sitter-rust/tree-sitter-rust.wasm")
+          : require.resolve("tree-sitter-java/tree-sitter-java.wasm");
+
+  const coldStart = nowMs();
+  const webLanguage = await WebLanguage.load(grammarWasm);
+  const parser = new WebParser();
+  parser.setLanguage(webLanguage);
+  const firstTree = parser.parse(SAMPLES[language]);
+  const coldMs = nowMs() - coldStart;
+
+  let parseTotalMs = 0;
+  for (let i = 0; i < ITERATIONS; i += 1) {
+    const start = nowMs();
+    parser.parse(SAMPLES[language]);
+    parseTotalMs += nowMs() - start;
+  }
+
+  return {
+    backend: "wasm_web_tree_sitter",
+    language,
+    available: Boolean(firstTree?.rootNode),
+    cold_start_ms: formatMs(coldMs),
+    per_file_ms: formatMs(parseTotalMs / ITERATIONS),
+    parse_engine_ms: formatMs(parseTotalMs / ITERATIONS)
+  };
+}
+
+async function benchNodeBoundary(language: TargetLanguage): Promise<BenchRow> {
+  const hostPath = fileURLToPath(new URL("./poc-node-parser-host.cjs", import.meta.url));
+  const sourceBase64 = Buffer.from(SAMPLES[language], "utf8").toString("base64");
+  const start = nowMs();
+  const child = spawnSync("node", [hostPath, "--batch", language, String(ITERATIONS), sourceBase64], {
+    cwd: process.cwd(),
+    encoding: "utf8"
+  });
+  const elapsed = nowMs() - start;
+  const stdoutRaw = child.stdout ?? "";
+  const stderrRaw = child.stderr ?? "";
+
+  if (child.status !== 0) {
+    return {
+      backend: "node_boundary_native",
+      language,
+      available: false,
+      cold_start_ms: formatMs(elapsed),
+      per_file_ms: formatMs(elapsed / ITERATIONS),
+      notes: stderrRaw || `node exited with code ${String(child.status)}`
+    };
+  }
+
+  const stdout = stdoutRaw.trim();
+  const payload = JSON.parse(stdout) as {
+    ok: boolean;
+    error?: string;
+    init_ms?: number;
+    parse_avg_ms?: number;
+  };
+  if (!payload.ok) {
+    return {
+      backend: "node_boundary_native",
+      language,
+      available: false,
+      cold_start_ms: formatMs(elapsed),
+      per_file_ms: formatMs(elapsed / ITERATIONS),
+      notes: payload.error ?? "node boundary parse failed"
+    };
+  }
+
+  return {
+    backend: "node_boundary_native",
+    language,
+    available: true,
+    cold_start_ms: formatMs(payload.init_ms ?? elapsed),
+    per_file_ms: formatMs(elapsed / ITERATIONS),
+    parse_engine_ms: formatMs(payload.parse_avg_ms ?? 0),
+    notes: "includes process boundary overhead"
+  };
+}
+
+function renderTable(rows: BenchRow[]): string {
+  const header = [
+    "backend",
+    "language",
+    "available",
+    "cold_start_ms",
+    "per_file_ms",
+    "parse_engine_ms",
+    "fallback_reason",
+    "notes"
+  ];
+  const lines = [header.join("\t")];
+  for (const row of rows) {
+    lines.push(
+      [
+        row.backend,
+        row.language,
+        row.available ? "yes" : "no",
+        row.cold_start_ms.toFixed(3),
+        row.per_file_ms.toFixed(3),
+        row.parse_engine_ms?.toFixed(3) ?? "",
+        row.fallback_reason ?? "",
+        row.notes ?? ""
+      ].join("\t")
+    );
+  }
+  return lines.join("\n");
+}
+
+async function main(): Promise<void> {
+  const baselineSnapshot = getChunkingParserAvailabilitySnapshot({
+    enabled_languages: ["javascript", "python", "rust", "java", "typescript", "go"]
+  });
+  const rows: BenchRow[] = [];
+  for (const language of ["javascript", "python", "rust", "java"] as const) {
+    rows.push(await benchBaselineChunking(language));
+    rows.push(await benchNativeCompatible(language));
+    rows.push(await benchWasm(language));
+    rows.push(await benchNodeBoundary(language));
+  }
+
+  console.log("=== parser availability snapshot (current chunking path) ===");
+  console.log(JSON.stringify(baselineSnapshot, null, 2));
+  console.log("");
+  console.log("=== benchmark table (tsv) ===");
+  console.log(renderTable(rows));
+  console.log("");
+  console.log("=== benchmark rows (json) ===");
+  console.log(JSON.stringify(rows, null, 2));
+}
+
+await main();