ashlr-plugin 1.36.4

package/servers/_edit-match.ts

@@ -0,0 +1,236 @@
+/**
+ * _edit-match.ts — Shared helpers for fuzzy search/replace matching.
+ *
+ * Exports:
+ *   levenshtein    — used by both fuzzyTopLines (hint display) and findFuzzyMatch
+ *   fuzzyTopLines  — top-3 closest lines for error hints (unchanged behavior)
+ *   findFuzzyMatch — two-tier fuzzy matcher, returns start/end byte offsets
+ *
+ * Design principles:
+ *   - Conservative: only returns a match when confidence is HIGH and UNIQUE.
+ *   - Never returns a low-confidence or ambiguous match.
+ *   - Caps work on huge files (>2 MB) → returns null fast.
+ *   - All errors swallowed by callers; this module must never throw.
+ */
+
+/** Levenshtein distance (capped at maxDist for speed). */
+export function levenshtein(a: string, b: string, maxDist = 256): number {
+  if (a === b) return 0;
+  if (a.length === 0) return Math.min(b.length, maxDist);
+  if (b.length === 0) return Math.min(a.length, maxDist);
+  const A = a.slice(0, 200);
+  const B = b.slice(0, 200);
+  const m = A.length, n = B.length;
+  let prev = Array.from({ length: n + 1 }, (_, i) => i);
+  let curr = new Array<number>(n + 1);
+  for (let i = 1; i <= m; i++) {
+    curr[0] = i;
+    for (let j = 1; j <= n; j++) {
+      const cost = A[i - 1] === B[j - 1] ? 0 : 1;
+      curr[j] = Math.min(curr[j - 1] + 1, prev[j] + 1, prev[j - 1] + cost);
+    }
+    [prev, curr] = [curr, prev];
+  }
+  return prev[n];
+}
+
+/** Return top-3 closest lines to `search` from `content`, with similarity scores. */
+export function fuzzyTopLines(
+  search: string,
+  content: string,
+): Array<{ lineNo: number; text: string; sim: number }> {
+  const needle = search.split("\n")[0].trim().slice(0, 200);
+  const lines = content.split("\n");
+  const results: Array<{ lineNo: number; text: string; sim: number }> = [];
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i].trim();
+    if (!line) continue;
+    const dist = levenshtein(needle, line.slice(0, 200));
+    const maxLen = Math.max(needle.length, line.length, 1);
+    const sim = Math.round((1 - dist / maxLen) * 100) / 100;
+    results.push({ lineNo: i + 1, text: lines[i], sim });
+  }
+  results.sort((a, b) => b.sim - a.sim);
+  return results.slice(0, 3);
+}
+
+/** Result from a successful fuzzy match. */
+export interface FuzzyMatchResult {
+  /** Byte offset of match start in `content`. */
+  start: number;
+  /** Byte offset of match end (exclusive) in `content`. */
+  end: number;
+  /** Similarity score [0..1]. */
+  score: number;
+}
+
+const FUZZY_CONTENT_LIMIT = 2 * 1024 * 1024; // 2 MB
+
+/**
+ * Try to find a fuzzy match for `search` in `content`.
+ *
+ * Tier 1 — whitespace-normalized unique match:
+ *   Normalize each run of whitespace to a single space and trim; if exactly ONE
+ *   normalized occurrence maps to a real offset → return it (score 1.0).
+ *
+ * Tier 2 — block Levenshtein:
+ *   Slide a window of search's line count over content's lines; compute
+ *   per-window normalized similarity; accept ONLY IF best >= 0.90 AND the
+ *   margin over the 2nd-best is >= 0.05 (uniqueness guard).
+ *
+ * Returns null when no confident, unambiguous match is found.
+ * NEVER called for content > 2 MB (caller guards).
+ */
+export function findFuzzyMatch(
+  content: string,
+  search: string,
+): FuzzyMatchResult | null {
+  if (!search || !content) return null;
+  if (content.length > FUZZY_CONTENT_LIMIT) return null;
+
+  // -------------------------------------------------------------------------
+  // Tier 1: whitespace-normalized match
+  // -------------------------------------------------------------------------
+  const normalizeWS = (s: string) =>
+    s.replace(/[ \t]+/g, " ").replace(/^ /gm, "").replace(/ $/gm, "").trimEnd();
+
+  const normContent = normalizeWS(content);
+  const normSearch = normalizeWS(search);
+
+  if (normSearch.length > 0) {
+    // Find all occurrences in normalized content.
+    const occurrences: number[] = [];
+    let pos = 0;
+    while ((pos = normContent.indexOf(normSearch, pos)) !== -1) {
+      occurrences.push(pos);
+      pos += normSearch.length;
+    }
+
+    if (occurrences.length === 1) {
+      // Map normalized offset back to original content offset.
+      // We rebuild the mapping by walking both strings in tandem.
+      const normToOrig = buildNormToOrigMap(content, normContent);
+      const normStart = occurrences[0];
+      const normEnd = normStart + normSearch.length;
+
+      const origStart = normToOrig[normStart];
+      if (origStart === undefined) {
+        // Could not map back to original offset — skip Tier 1.
+      } else {
+        // For end, we use normEnd-1 to get the last char's orig position, then
+        // advance past it. Guard against out-of-bounds.
+        const origEnd =
+          normEnd > 0 && normToOrig[normEnd - 1] !== undefined
+            ? normToOrig[normEnd - 1]! + 1
+            : origStart + search.length;
+
+        if (origEnd > origStart) {
+          return { start: origStart, end: origEnd, score: 1.0 };
+        }
+      }
+    }
+  }
+
+  // -------------------------------------------------------------------------
+  // Tier 2: block Levenshtein over line windows
+  // -------------------------------------------------------------------------
+  const searchLines = search.split("\n");
+  const contentLines = content.split("\n");
+  const windowSize = searchLines.length;
+
+  if (windowSize === 0 || contentLines.length < windowSize) return null;
+
+  // Build a flat string per window for distance comparison.
+  const searchFlat = searchLines.map(l => l.trim()).join("\n");
+
+  interface WindowScore { lineIdx: number; score: number }
+  let best: WindowScore | null = null;
+  let secondBest: WindowScore | null = null;
+
+  for (let i = 0; i <= contentLines.length - windowSize; i++) {
+    const windowFlat = contentLines.slice(i, i + windowSize).map(l => l.trim()).join("\n");
+    const dist = levenshtein(searchFlat, windowFlat, 512);
+    const maxLen = Math.max(searchFlat.length, windowFlat.length, 1);
+    const score = 1 - dist / maxLen;
+
+    if (!best || score > best.score) {
+      secondBest = best;
+      best = { lineIdx: i, score };
+    } else if (!secondBest || score > secondBest.score) {
+      secondBest = { lineIdx: i, score };
+    }
+  }
+
+  if (!best) return null;
+
+  // Confidence gate: score >= 0.90 AND margin over 2nd-best >= 0.05.
+  const margin = secondBest ? best.score - secondBest.score : 1;
+  if (best.score < 0.90 || margin < 0.05) return null;
+
+  // Map best window back to byte offsets.
+  const start = lineToOffset(content, best.lineIdx);
+  const endLineIdx = best.lineIdx + windowSize;
+  const end =
+    endLineIdx >= contentLines.length
+      ? content.length
+      : lineToOffset(content, endLineIdx);
+
+  return { start, end, score: Math.round(best.score * 100) / 100 };
+}
+
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+
+/**
+ * Build a map: normContent index → original content index.
+ * Both strings must have been produced by the same normalizeWS logic above.
+ */
+function buildNormToOrigMap(orig: string, norm: string): Array<number | undefined> {
+  const map: Array<number | undefined> = new Array(norm.length + 1).fill(undefined);
+  let oi = 0; // index into orig
+  let ni = 0; // index into norm
+
+  while (ni < norm.length && oi < orig.length) {
+    // Skip whitespace runs in orig that were collapsed to a single space in norm.
+    // The normalizer collapses /[ \t]+/ → " " and trims line ends, but newlines
+    // are preserved. We handle char-by-char.
+    const nc = norm[ni];
+    const oc = orig[oi];
+
+    if (nc === oc) {
+      map[ni] = oi;
+      ni++;
+      oi++;
+    } else if (nc === " " && (oc === " " || oc === "\t")) {
+      // Collapsed whitespace: norm has one space, orig has one or more spaces/tabs.
+      map[ni] = oi;
+      ni++;
+      oi++;
+      // Skip remaining contiguous spaces/tabs in orig.
+      while (oi < orig.length && (orig[oi] === " " || orig[oi] === "\t")) {
+        oi++;
+      }
+    } else {
+      // Mismatch not explained by WS collapse — advance orig to resync.
+      // This handles trimmed leading/trailing spaces on lines.
+      oi++;
+    }
+  }
+  return map;
+}
+
+/**
+ * Return the byte offset of the start of line `lineIdx` (0-based) in `content`.
+ */
+function lineToOffset(content: string, lineIdx: number): number {
+  if (lineIdx === 0) return 0;
+  let count = 0;
+  for (let i = 0; i < content.length; i++) {
+    if (content[i] === "\n") {
+      count++;
+      if (count === lineIdx) return i + 1;
+    }
+  }
+  return content.length;
+}

package/servers/_edit-validate.ts

@@ -0,0 +1,196 @@
+/**
+ * _edit-validate.ts — Post-edit syntax validation.
+ *
+ * Wired languages: TypeScript (.ts/.tsx), JavaScript (.js/.mjs/.cjs/.jsx), JSON.
+ * YAML: no parser in deps → skipped.
+ * Everything else: skipped (returns {introducedError:false}).
+ *
+ * Design contract:
+ *   - NEVER throws or rejects — all errors are caught and return safe defaults.
+ *   - Only reports introducedError=true when the edit INTRODUCED the error
+ *     (original was clean, updated is broken).
+ *   - If the original already had parse errors → introducedError=false (don't blame the edit).
+ *   - _ast-languages is dynamically imported so the WASM binary is NOT parsed at
+ *     module load time. This keeps subprocess startup fast (~50ms vs ~1.5s with
+ *     static import), ensuring spawned MCP servers stay within test timeouts.
+ *   - The entire call is guarded by a 2s timeout. The timer is .unref()'d so it
+ *     never prevents subprocess exit.
+ */
+
+import { extname, basename } from "path";
+
+export interface ValidationResult {
+  introducedError: boolean;
+  detail: string;
+}
+
+const SAFE: ValidationResult = { introducedError: false, detail: "" };
+
+/**
+ * Timeout (ms) for the entire validation call.
+ * 2s: generous for a warm parser (~microseconds), safely under typical 5s test
+ * timeouts even when the subprocess has other startup overhead.
+ */
+const VALIDATE_TIMEOUT_MS = 2000;
+
+/**
+ * Validate that `updated` does not introduce a syntax error that was not
+ * already present in `original`.
+ *
+ * @param absPath  Absolute path of the file (used only for extension detection).
+ * @param original Content before the edit.
+ * @param updated  Content after the edit.
+ */
+export async function validateEdit(
+  absPath: string,
+  original: string,
+  updated: string,
+): Promise<ValidationResult> {
+  // Wrap everything in a timeout + catch so a hung parser never blocks a write.
+  try {
+    return await Promise.race([
+      _validateInner(absPath, original, updated),
+      new Promise<ValidationResult>((resolve) => {
+        const t = setTimeout(() => resolve(SAFE), VALIDATE_TIMEOUT_MS);
+        // unref so this timer never prevents subprocess exit
+        if (t && typeof t === "object" && "unref" in t) (t as NodeJS.Timeout).unref();
+      }),
+    ]);
+  } catch {
+    return SAFE;
+  }
+}
+
+async function _validateInner(
+  absPath: string,
+  original: string,
+  updated: string,
+): Promise<ValidationResult> {
+  const ext = extname(basename(absPath)).toLowerCase();
+
+  // -------------------------------------------------------------------------
+  // JSON
+  // -------------------------------------------------------------------------
+  if (ext === ".json") {
+    return validateJson(original, updated);
+  }
+
+  // -------------------------------------------------------------------------
+  // JS / TS via tree-sitter — dynamic import keeps WASM out of module load
+  // -------------------------------------------------------------------------
+  const isWiredTS =
+    ext === ".ts" || ext === ".tsx" || ext === ".js" ||
+    ext === ".mjs" || ext === ".cjs" || ext === ".jsx";
+
+  if (isWiredTS) {
+    return await validateAst(absPath, original, updated);
+  }
+
+  // All other extensions: skip.
+  return SAFE;
+}
+
+// ---------------------------------------------------------------------------
+// JSON validation
+// ---------------------------------------------------------------------------
+
+function validateJson(original: string, updated: string): ValidationResult {
+  let originalOk = true;
+  try {
+    JSON.parse(original);
+  } catch {
+    originalOk = false;
+  }
+
+  if (!originalOk) {
+    // Original was already broken — don't blame the edit.
+    return SAFE;
+  }
+
+  try {
+    JSON.parse(updated);
+    return SAFE;
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    return {
+      introducedError: true,
+      detail: `JSON parse error: ${msg.slice(0, 120)}`,
+    };
+  }
+}
+
+// ---------------------------------------------------------------------------
+// AST (tree-sitter) validation — dynamic import so WASM isn't loaded at startup
+// ---------------------------------------------------------------------------
+
+async function validateAst(
+  absPath: string,
+  original: string,
+  updated: string,
+): Promise<ValidationResult> {
+  // Dynamic import: WASM binary is parsed only on first call, not at module load.
+  let resolveLanguage: (f: string) => import("./_ast-languages").Language | null;
+  let getParser: (lang: import("./_ast-languages").Language) => Promise<import("./_ast-languages").WTSParser>;
+  try {
+    const mod = await import("./_ast-languages");
+    resolveLanguage = mod.resolveLanguage;
+    getParser = mod.getParser;
+  } catch {
+    return SAFE;
+  }
+
+  const lang = resolveLanguage(absPath);
+  if (!lang || (lang !== "typescript" && lang !== "tsx" && lang !== "javascript")) {
+    return SAFE;
+  }
+
+  let parser: import("./_ast-languages").WTSParser;
+  try {
+    parser = await getParser(lang);
+  } catch {
+    // Language not wired yet (Python/Go/Rust stub) — skip gracefully.
+    return SAFE;
+  }
+
+  // Parse both versions. tree-sitter is synchronous despite the async wrapper.
+  let origTree: ReturnType<typeof parser.parse>;
+  let updTree: ReturnType<typeof parser.parse>;
+  try {
+    origTree = parser.parse(original);
+    updTree = parser.parse(updated);
+  } catch {
+    return SAFE;
+  }
+
+  // If the original already had errors, don't blame the edit.
+  if (origTree.rootNode.hasError) {
+    return SAFE;
+  }
+
+  if (!updTree.rootNode.hasError) {
+    return SAFE;
+  }
+
+  // Find the first error node and report its start line.
+  const errLine = findFirstErrorLine(updTree.rootNode);
+  const lineRef = errLine >= 0 ? ` at line ${errLine + 1}` : "";
+  return {
+    introducedError: true,
+    detail: `${lang} parse error${lineRef}`,
+  };
+}
+
+/**
+ * Walk the tree to find the first ERROR or MISSING node.
+ * Returns 0-based line number, or -1 if not found.
+ */
+function findFirstErrorLine(node: { type: string; startPosition: { row: number }; children: unknown[] }): number {
+  if (node.type === "ERROR" || node.type === "MISSING") {
+    return node.startPosition.row;
+  }
+  for (const child of node.children) {
+    const result = findFirstErrorLine(child as { type: string; startPosition: { row: number }; children: unknown[] });
+    if (result >= 0) return result;
+  }
+  return -1;
+}

package/servers/_embed-calibration.ts

@@ -0,0 +1,123 @@
+/**
+ * _embed-calibration — embedding threshold constants and calibration helpers.
+ *
+ * Used by grep-server to gate the embedding cache and log A/B data so we
+ * can tune EMBED_HIT_THRESHOLD from real usage instead of guesses.
+ */
+
+import { createHash } from "crypto";
+
+/**
+ * Cosine similarity threshold for a cache hit to prepend results.
+ *
+ * Baseline 0.68 (down from 0.75 in v1.12) — calibrated against the 24-doc
+ * BM25 corpus where IDF weights are nearly flat and real similarity scores
+ * cluster lower than classic dense-embedding expectations. Override via
+ * `ASHLR_EMBED_THRESHOLD` (float in [0, 1]). A/B data flows to
+ * `~/.ashlr/embed-calibration.jsonl` (see recordEmbedCalibration below).
+ */
+export const EMBED_HIT_THRESHOLD = (() => {
+  const raw = process.env.ASHLR_EMBED_THRESHOLD;
+  if (!raw) return 0.68;
+  const n = Number.parseFloat(raw);
+  return Number.isFinite(n) && n >= 0 && n <= 1 ? n : 0.68;
+})();
+
+/**
+ * Minimum BM25 corpus size before the embedding cache is consulted. Below
+ * this threshold, IDF weights are dominated by noise and nearly any query
+ * produces a spurious hit — see v1.17 incident where a 6-doc corpus emitted
+ * 0.7+ cosine hits on unrelated patterns.
+ *
+ * v1.18 Trust Pass: if docCount < THIS, the cache is skipped outright.
+ * v1.24 Track E: lowered to 10 (warm-start). The warm tier (10–49) uses a
+ * stricter cosine threshold to compensate for the thinner corpus.
+ */
+export const BM25_CORPUS_MIN = 50;
+
+/**
+ * v1.24 Track E — Three-tier warm-start corpus classification.
+ *
+ * cold  (0–9 docs)  : skip cache entirely (too few docs, high noise).
+ * warm  (10–49 docs): consult cache with a stricter cosine threshold.
+ * hot   (50+ docs)  : full v1.23 behavior (EMBED_HIT_THRESHOLD = 0.68).
+ */
+export type CorpusTier = "cold" | "warm" | "hot";
+
+/** Threshold applied at N=10 (warm entry). */
+export const WARM_THRESHOLD_START = 0.80;
+/** Threshold applied at N≥50 (hot entry). */
+export const WARM_THRESHOLD_END = 0.68;
+/** Corpus size where warm tier begins. */
+export const WARM_CORPUS_MIN = 10;
+
+/**
+ * Classify corpus into cold / warm / hot based on document count.
+ */
+export function computeCorpusTier(corpusSize: number): CorpusTier {
+  if (corpusSize < WARM_CORPUS_MIN) return "cold";
+  if (corpusSize < BM25_CORPUS_MIN) return "warm";
+  return "hot";
+}
+
+/**
+ * Compute the cosine similarity threshold for the given corpus size.
+ *
+ * Warm tier uses a linear gradient:
+ *   threshold = max(WARM_THRESHOLD_END, WARM_THRESHOLD_START - (corpusSize - 10) * 0.003)
+ *
+ * At N=10  → 0.80 (strictest, smallest corpus)
+ * At N=50  → 0.68 (matches hot tier, smooth handoff)
+ * Hot tier → always 0.68 (EMBED_HIT_THRESHOLD)
+ * Cold tier → threshold is irrelevant (cache is not consulted)
+ */
+export function computeWarmThreshold(corpusSize: number): number {
+  const tier = computeCorpusTier(corpusSize);
+  if (tier === "cold") return WARM_THRESHOLD_START; // unused; cache skipped
+  if (tier === "hot") return WARM_THRESHOLD_END;
+  // warm: linear gradient
+  const raw = WARM_THRESHOLD_START - (corpusSize - WARM_CORPUS_MIN) * 0.003;
+  return Math.max(WARM_THRESHOLD_END, raw);
+}
+
+/** Stable 8-char project hash from absolute cwd path. */
+export function projectHash(cwd: string): string {
+  return createHash("sha256").update(cwd).digest("hex").slice(0, 8);
+}
+
+export function readCorpusDocCount(): number {
+  try {
+    const { readFileSync, existsSync } = require("fs") as typeof import("fs");
+    const { join } = require("path") as typeof import("path");
+    const { homedir } = require("os") as typeof import("os");
+    const p = join(process.env.HOME ?? homedir(), ".ashlr", "embed-corpus.json");
+    if (!existsSync(p)) return 0;
+    const raw = readFileSync(p, "utf-8");
+    const parsed = JSON.parse(raw) as { docCount?: number };
+    return typeof parsed.docCount === "number" && Number.isFinite(parsed.docCount) ? parsed.docCount : 0;
+  } catch {
+    return 0;
+  }
+}
+
+// Lightweight calibration log so we can tune EMBED_HIT_THRESHOLD from real
+// data instead of guesses. Appends one JSONL record per grep call; fire-and-forget.
+export async function recordEmbedCalibration(record: {
+  queryHashHex: string;
+  topSimilarity: number;
+  hit: boolean;
+  contentLength: number;
+  threshold: number;
+}): Promise<void> {
+  try {
+    const { homedir } = await import("os");
+    const { appendFile, mkdir } = await import("fs/promises");
+    const { dirname, join } = await import("path");
+    const path = join(process.env.HOME ?? homedir(), ".ashlr", "embed-calibration.jsonl");
+    await mkdir(dirname(path), { recursive: true }).catch(() => undefined);
+    const line = JSON.stringify({ ts: new Date().toISOString(), ...record }) + "\n";
+    await appendFile(path, line, "utf-8");
+  } catch {
+    // Calibration log is observability, not critical path.
+  }
+}