@opengeni/jev 0.1.0-canary.36199476632001

package/src/code-search/session.ts

@@ -0,0 +1,89 @@
+/**
+ * session.ts - one search's view of the workspace: counts calls, records partial ripgrep output and
+ * makes every call honour the search's AbortSignal.
+ */
+import { CodeSearchWorkspaceError, type CodeSearchWorkspace } from "./workspace";
+
+/** Parallel reads per search. */
+export const READ_CONCURRENCY = 8;
+/** Parallel ripgrep calls for one pattern split under CODE_SEARCH_MAX_PATTERN_CHARS. */
+export const RIPGREP_SPLIT_CONCURRENCY = 4;
+
+export class WorkspaceSession {
+  calls = 0;
+  /** A ripgrep call returned output cut at the adapter's byte cap. */
+  truncated = false;
+  /** A ripgrep call hit its time limit (its output is partial). */
+  timedOut = false;
+
+  constructor(
+    private readonly workspace: CodeSearchWorkspace,
+    readonly signal: AbortSignal,
+    private readonly ripgrepTimeoutMs: number,
+  ) {}
+
+  get partial(): boolean {
+    return this.truncated || this.timedOut;
+  }
+
+  /**
+   * ripgrep stdout. Exit code 2 (error) with some output keeps the output (for example unreadable files);
+   * with no output it throws unless `allowFailure`, which then yields "".
+   */
+  async ripgrep(
+    args: readonly string[],
+    options: { allowFailure?: boolean } = {},
+  ): Promise<string> {
+    this.signal.throwIfAborted();
+    this.calls++;
+    const r = await this.workspace.ripgrep(args, {
+      signal: this.signal,
+      timeoutMs: this.ripgrepTimeoutMs,
+    });
+    this.signal.throwIfAborted();
+    if (r.truncated) this.truncated = true;
+    if (r.timedOut) this.timedOut = true;
+    if (r.exitCode === 2 && !r.stdout && !r.truncated && !r.timedOut && !options.allowFailure) {
+      throw new CodeSearchWorkspaceError("ripgrep failed (exit code 2) without output");
+    }
+    return r.stdout;
+  }
+
+  /** File text, or null when missing or binary (NUL bytes). */
+  async readText(path: string, maxBytes: number): Promise<string | null> {
+    this.signal.throwIfAborted();
+    this.calls++;
+    const r = await this.workspace.readText(path, { signal: this.signal, maxBytes });
+    this.signal.throwIfAborted();
+    if (!r || r.binary || r.text.includes("\u0000")) return null;
+    return r.text;
+  }
+
+  async pathKinds(
+    paths: readonly string[],
+  ): Promise<Record<string, "file" | "directory" | "missing">> {
+    this.signal.throwIfAborted();
+    this.calls++;
+    const r = await this.workspace.pathKinds(paths, { signal: this.signal });
+    this.signal.throwIfAborted();
+    return r;
+  }
+}
+
+/** Map with at most `limit` calls in flight; results keep the input order. */
+export async function mapLimit<T, R>(
+  items: readonly T[],
+  limit: number,
+  fn: (item: T) => Promise<R>,
+): Promise<R[]> {
+  const out = new Array<R>(items.length);
+  let next = 0;
+  const worker = async () => {
+    while (next < items.length) {
+      const i = next++;
+      out[i] = await fn(items[i]!);
+    }
+  };
+  await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
+  return out;
+}

package/src/code-search/text.ts

@@ -0,0 +1,159 @@
+/**
+ * text.ts - keyword variants, word splitting, stopwords and small text helpers.
+ */
+
+/** Split an identifier or phrase into lowercase words: fooBarBAZ_qux-v2 -> [foo, bar, baz, qux, v2]. */
+export function splitWords(s: string): string[] {
+  return s
+    .replace(/([a-z0-9])([A-Z])/g, "$1 $2")
+    .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2")
+    .split(/[^A-Za-z0-9]+/)
+    .filter(Boolean)
+    .map((w) => w.toLowerCase());
+}
+
+const cap = (w: string) => (w ? w[0]!.toUpperCase() + w.slice(1) : w);
+
+/**
+ * Identifier variants of a keyword. The search is case-insensitive, so only the
+ * separator forms matter: concatenated (camel/Pascal/flat), snake/SCREAMING, kebab, spaced phrase.
+ * Returned in display casing (camel, snake, kebab, spaced) for the trace; deduped case-insensitively.
+ */
+export function keywordVariants(kw: string): string[] {
+  const raw = kw.trim();
+  if (!raw) return [];
+  const words = splitWords(raw);
+  const out: string[] = [raw];
+  if (words.length >= 2) {
+    out.push(words[0] + words.slice(1).map(cap).join("")); // camelCase (== PascalCase / flat under -i)
+    out.push(words.join("_")); // snake_case (== SCREAMING_SNAKE under -i)
+    out.push(words.join("-")); // kebab-case
+    out.push(words.join(" ")); // spaced phrase (prose in docs)
+  }
+  const seen = new Set<string>();
+  return out.filter((v) => {
+    const k = v.toLowerCase();
+    if (seen.has(k) || k.length < 2) return false;
+    seen.add(k);
+    return true;
+  });
+}
+
+/** Contiguous 2-word fragments of a compound with >= 3 words (fallback when the full name has no hits). */
+export function compoundFragments(kw: string): string[] {
+  const words = splitWords(kw);
+  if (words.length < 3) return [];
+  const out: string[] = [];
+  for (let i = 0; i + 1 < words.length; i++) {
+    const a = words[i]!;
+    const b = words[i + 1]!;
+    if (a.length + b.length < 8) continue; // skip tiny fragments like "is" + "on"
+    out.push(a + cap(b));
+  }
+  return [...new Set(out)];
+}
+
+/** A single plain word (no case change, no separator) short enough to need a word boundary. */
+export function isShortPlainWord(kw: string, maxLen: number): boolean {
+  return /^[A-Za-z]+$/.test(kw) && kw.length <= maxLen && splitWords(kw).length === 1;
+}
+
+/** Escape for Rust regex (ripgrep). */
+export function escapeRegex(s: string): string {
+  return s.replace(/[\\.+*?()|[\]{}^$#&\-~]/g, (m) => `\\${m}`);
+}
+
+export const STOPWORDS = new Set(
+  (
+    "a an and are as at be been but by can could did do does doing done for from had has have how i if in into is it its " +
+    "just me my no not of on or our should so some such than that the their them then there these they this those to " +
+    "too was we were what when where which while who why will with would you your also any each else ever every much " +
+    "more most other same very about above after again against all am before being below between both during few " +
+    "further here him his her hers himself itself let may might must nor now off once only own over under until up " +
+    "use used using want way get got make makes happen happens happening still actually really even whether instead " +
+    "rather something someone thing things one two three kind already mean means"
+  ).split(" "),
+);
+
+/** Crude stemmer: good enough for overlap scoring (not for display). */
+export function stem(w: string): string {
+  let s = w.toLowerCase();
+  if (s.length > 5 && s.endsWith("ies")) s = s.slice(0, -3) + "y";
+  else if (s.length > 5 && (s.endsWith("ing") || s.endsWith("ers"))) s = s.slice(0, -3);
+  else if (s.length > 4 && (s.endsWith("ed") || s.endsWith("es") || s.endsWith("er")))
+    s = s.slice(0, -2);
+  else if (s.length > 3 && s.endsWith("s") && !s.endsWith("ss")) s = s.slice(0, -1);
+  return s;
+}
+
+/** Content terms of a text (identifiers split into words, stopwords dropped, stemmed, deduped). */
+export function contentTerms(text: string, minLen = 3): string[] {
+  const out = new Set<string>();
+  for (const tok of text.match(/[A-Za-z][A-Za-z0-9_]*/g) ?? []) {
+    for (const w of splitWords(tok)) {
+      if (w.length < minLen || STOPWORDS.has(w)) continue;
+      out.add(stem(w));
+    }
+  }
+  return [...out];
+}
+
+/** Fraction of `terms` present in `textTerms`. */
+export function overlap(terms: string[], textTerms: Set<string>): number {
+  if (!terms.length) return 0;
+  let n = 0;
+  for (const t of terms) if (textTerms.has(t)) n++;
+  return n / terms.length;
+}
+
+export function isTestPath(p: string): boolean {
+  return (
+    /(^|\/)(test|tests|__tests__|spec|e2e|fixtures?)\//.test(p) || /\.(test|spec)\.[a-z]+$/.test(p)
+  );
+}
+
+/** Release notes: CHANGELOG*, HISTORY*, RELEASE_NOTES*, .changeset/ entries (describe past changes, not current code). */
+export function isChangelogPath(p: string): boolean {
+  return (
+    /(^|\/)(CHANGELOG|HISTORY|RELEASE[-_]?NOTES)[^/]*$/i.test(p) || /(^|\/)\.changeset\//.test(p)
+  );
+}
+
+/** Is the question about history / releases (then release notes are not down-weighted)? */
+export function questionMentionsHistory(q: string): boolean {
+  return /\b(changelog|release notes?|released|history|historical|when (?:was|did)|which version|since version|changed in)\b/i.test(
+    q,
+  );
+}
+
+export function isDocPath(p: string): boolean {
+  return /\.(md|mdx|txt|rst)$/i.test(p);
+}
+
+export function questionMentionsTests(q: string): boolean {
+  return /\b(test|tests|testing|spec|specs|unit test|e2e|fixture)\b/i.test(q);
+}
+
+/** Trim a line to maxChars around the first occurrence of any needle (case-insensitive). */
+export function trimAround(line: string, needles: string[], maxChars: number): string {
+  const t = line.replace(/\t/g, "  ").trim();
+  if (t.length <= maxChars) return t;
+  const lower = t.toLowerCase();
+  let at = -1;
+  for (const n of needles) {
+    const i = lower.indexOf(n.toLowerCase());
+    if (i >= 0 && (at < 0 || i < at)) at = i;
+  }
+  if (at < 0) return t.slice(0, maxChars - 3) + "...";
+  const start = Math.max(0, Math.min(at - Math.floor(maxChars / 3), t.length - maxChars));
+  const s = t.slice(start, start + maxChars - 6);
+  return (start > 0 ? "..." : "") + s + (start + maxChars - 6 < t.length ? "..." : "");
+}
+
+export function estTokens(chars: number, charsPerToken: number): number {
+  return Math.ceil(chars / charsPerToken);
+}
+
+export function fmtK(n: number): string {
+  return n >= 1000 ? `${(n / 1000).toFixed(1)}k` : String(n);
+}

package/src/code-search/tool.ts

@@ -0,0 +1,209 @@
+/**
+ * tool.ts - the model-facing surface of code_search: name, description, input schema, argument
+ * parsing and short error texts. The worker wires these to runCodeSearch.
+ */
+import { JevRequestError, JevUnavailableError } from "../client";
+import { CodeSearchRipgrepMissingError, CodeSearchWorkspaceError } from "./workspace";
+
+export const CODE_SEARCH_TOOL_NAME = "code_search";
+
+export const CODE_SEARCH_TOOL_DESCRIPTION =
+  "Find where something is implemented, configured or decided in the code under the working directory, in one call " +
+  "instead of many separate searches and file reads. Give one precise question and 6-15 keywords: likely identifiers, " +
+  "file-name fragments, config keys, error strings and synonyms. Optional subQuestions split distinct parts (for " +
+  "'is X required?', add one for what could skip or override X); optional paths limit the search. It ranks files and " +
+  "passages with a fast relevance model, follows definitions one level, and returns the best passages verbatim with " +
+  "file paths and line numbers, plus an evidence rating for those passages. The rating cannot see what the search " +
+  "missed: use the passages instead of re-reading them, then check what they do not cover (other entry points, " +
+  "defaults, flags, exceptions) before concluding.";
+
+export const CODE_SEARCH_LIMITS = {
+  questionMinChars: 3,
+  questionMaxChars: 2000,
+  keywordsMax: 20,
+  keywordMaxChars: 120,
+  subQuestionsMax: 3,
+  subQuestionMaxChars: 1000,
+  pathsMax: 8,
+  pathMaxChars: 1000,
+} as const;
+
+const L = CODE_SEARCH_LIMITS;
+
+/** Plain JSON (mutable arrays), so it fits any JSON-schema slot of a tool definition. */
+export type CodeSearchJsonValue =
+  | string
+  | number
+  | boolean
+  | null
+  | CodeSearchJsonValue[]
+  | { [key: string]: CodeSearchJsonValue };
+
+/** JSON schema of the tool input; parseCodeSearchArguments enforces the same rules. */
+export const codeSearchInputSchema: { [key: string]: CodeSearchJsonValue } = {
+  type: "object",
+  properties: {
+    question: {
+      type: "string",
+      minLength: L.questionMinChars,
+      maxLength: L.questionMaxChars,
+      description:
+        "One precise question about the code, for example where a behavior is implemented or how a value is computed.",
+    },
+    keywords: {
+      type: "array",
+      minItems: 1,
+      maxItems: L.keywordsMax,
+      items: { type: "string", minLength: 1, maxLength: L.keywordMaxChars },
+      description:
+        "6-15 search keywords: likely identifiers (camelCase, snake_case, UPPER_CASE), file-name fragments, config or env keys, error strings, synonyms. Case and camel/snake/kebab variants are searched automatically.",
+    },
+    subQuestions: {
+      type: "array",
+      maxItems: L.subQuestionsMax,
+      items: { type: "string", minLength: 1, maxLength: L.subQuestionMaxChars },
+      description: "Optional: the distinct parts of a multi-part question, one per entry.",
+    },
+    paths: {
+      type: "array",
+      maxItems: L.pathsMax,
+      items: { type: "string", minLength: 1, maxLength: L.pathMaxChars },
+      description:
+        "Optional: workspace-relative directories or files to limit the search to. Default: the whole workspace.",
+    },
+  },
+  required: ["question", "keywords"],
+  additionalProperties: false,
+};
+
+export interface CodeSearchArguments {
+  question: string;
+  keywords: string[];
+  subQuestions: string[];
+  paths: string[];
+}
+
+/** Invalid tool arguments; the message is written for the model. */
+export class CodeSearchArgumentError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "CodeSearchArgumentError";
+  }
+}
+
+const KNOWN_ARGUMENTS = new Set(["question", "keywords", "subQuestions", "paths"]);
+
+/** Validate and normalize tool arguments (trim, drop empty entries, dedupe, workspace-relative paths). */
+export function parseCodeSearchArguments(args: Record<string, unknown>): CodeSearchArguments {
+  const unknown = Object.keys(args).filter((k) => !KNOWN_ARGUMENTS.has(k));
+  if (unknown.length) {
+    throw new CodeSearchArgumentError(
+      `unknown argument ${unknown.map((k) => `"${k}"`).join(", ")}; allowed: question, keywords, subQuestions, paths`,
+    );
+  }
+
+  if (typeof args.question !== "string")
+    throw new CodeSearchArgumentError("question is required and must be a string");
+  const question = args.question.trim();
+  if (question.length < L.questionMinChars)
+    throw new CodeSearchArgumentError(`question must be at least ${L.questionMinChars} characters`);
+  if (question.length > L.questionMaxChars)
+    throw new CodeSearchArgumentError(`question must be at most ${L.questionMaxChars} characters`);
+
+  const keywords = stringList(args.keywords, "keywords", true);
+  if (!keywords.length)
+    throw new CodeSearchArgumentError("keywords must contain at least one non-empty keyword");
+  if (keywords.length > L.keywordsMax)
+    throw new CodeSearchArgumentError(
+      `keywords accepts at most ${L.keywordsMax} entries; keep the ${L.keywordsMax} most specific`,
+    );
+  const longKeyword = keywords.find((k) => k.length > L.keywordMaxChars);
+  if (longKeyword)
+    throw new CodeSearchArgumentError(
+      `each keyword must be at most ${L.keywordMaxChars} characters; use short identifiers or phrases`,
+    );
+
+  const subQuestions = stringList(args.subQuestions, "subQuestions", false);
+  if (subQuestions.length > L.subQuestionsMax)
+    throw new CodeSearchArgumentError(`subQuestions accepts at most ${L.subQuestionsMax} entries`);
+  if (subQuestions.some((s) => s.length > L.subQuestionMaxChars)) {
+    throw new CodeSearchArgumentError(
+      `each sub-question must be at most ${L.subQuestionMaxChars} characters`,
+    );
+  }
+
+  const paths = [...new Set(stringList(args.paths, "paths", false).map(normalizePath))];
+  if (paths.length > L.pathsMax)
+    throw new CodeSearchArgumentError(`paths accepts at most ${L.pathsMax} entries`);
+
+  return { question, keywords, subQuestions, paths };
+}
+
+function stringList(value: unknown, name: string, required: boolean): string[] {
+  if (value === undefined || value === null) {
+    if (required)
+      throw new CodeSearchArgumentError(`${name} is required and must be an array of strings`);
+    return [];
+  }
+  if (!Array.isArray(value) || value.some((v) => typeof v !== "string")) {
+    throw new CodeSearchArgumentError(`${name} must be an array of strings`);
+  }
+  return [...new Set((value as string[]).map((v) => v.trim()).filter(Boolean))];
+}
+
+function normalizePath(raw: string): string {
+  if (raw.length > L.pathMaxChars)
+    throw new CodeSearchArgumentError(`each path must be at most ${L.pathMaxChars} characters`);
+  if (raw.startsWith("/") || raw.startsWith("~") || /^[A-Za-z]:[\\/]/.test(raw)) {
+    throw new CodeSearchArgumentError(
+      `path "${raw}" is absolute; give paths relative to the working directory`,
+    );
+  }
+  let p = raw;
+  while (p.startsWith("./")) p = p.slice(2);
+  p = p.replace(/\/+$/, "");
+  if (!p) p = ".";
+  if (p.split("/").some((seg) => seg === "..")) {
+    throw new CodeSearchArgumentError(
+      `path "${raw}" leaves the working directory; ".." is not allowed`,
+    );
+  }
+  if (p.startsWith("-")) throw new CodeSearchArgumentError(`path "${raw}" must not start with "-"`);
+  return p;
+}
+
+const FALLBACK = "Search with exec_command (rg, sed) instead.";
+
+/** Short model-facing text for a failed code_search call. */
+export function renderCodeSearchError(error: unknown): string {
+  const detail = (e: unknown) =>
+    (e instanceof Error ? e.message : String(e)).replace(/\s+/g, " ").trim().slice(0, 240);
+  if (error instanceof CodeSearchArgumentError)
+    return `code_search: invalid arguments: ${error.message}.`;
+  if (error instanceof JevUnavailableError)
+    return `code_search is unavailable right now (${detail(error)}). ${FALLBACK}`;
+  if (error instanceof JevRequestError)
+    return `code_search failed: the relevance model rejected the request (${detail(error)}). ${FALLBACK}`;
+  if (isRipgrepMissing(error)) {
+    return "code_search cannot run here: ripgrep (rg) is not installed in this workspace. Search with exec_command (grep, find, sed) instead.";
+  }
+  if (error instanceof CodeSearchWorkspaceError)
+    return `code_search could not search this workspace (${detail(error)}). ${FALLBACK}`;
+  if (isAbort(error)) return "code_search was cancelled.";
+  return `code_search failed unexpectedly (${detail(error)}). ${FALLBACK}`;
+}
+
+function isRipgrepMissing(error: unknown): boolean {
+  if (error instanceof CodeSearchRipgrepMissingError) return true;
+  if (!(error instanceof CodeSearchWorkspaceError)) return false;
+  const m = error.message;
+  return /ripgrep|\brg\b/i.test(m) && /not (installed|found)|missing|ENOENT|no such file/i.test(m);
+}
+
+function isAbort(error: unknown): boolean {
+  return (
+    typeof error === "object" &&
+    error !== null &&
+    (error as { name?: unknown }).name === "AbortError"
+  );
+}