@mrsf/cli 0.4.2 → 0.4.3

package/dist/lib/fuzzy.js

@@ -110,6 +110,37 @@ export function normalizedMatch(lines, needle) {
 function tokenize(text) {
     return text.split(/\s+/).filter((t) => t.length > 0);
 }
+/**
+ * Build a frequency map for a token array. Used for fast multiset
+ * intersection during the fuzzySearch prefilter.
+ */
+function buildTokenFreq(tokens) {
+    const freq = new Map();
+    for (const tok of tokens) {
+        freq.set(tok, (freq.get(tok) ?? 0) + 1);
+    }
+    return freq;
+}
+/**
+ * Multiset intersection size: sum over needle tokens of
+ * min(needleFreq[token], windowFreq[token]). This is an upper bound on
+ * the longest common subsequence length, so it lets us cheaply skip
+ * windows that cannot possibly reach the score threshold.
+ */
+function multisetIntersection(needleFreq, windowFreq) {
+    let common = 0;
+    // Iterate the smaller map for speed.
+    const [smaller, larger] = needleFreq.size <= windowFreq.size
+        ? [needleFreq, windowFreq]
+        : [windowFreq, needleFreq];
+    for (const [tok, cnt] of smaller) {
+        const other = larger.get(tok);
+        if (other !== undefined) {
+            common += cnt < other ? cnt : other;
+        }
+    }
+    return common;
+}
 /**
  * Longest Common Subsequence length of two token arrays.
  */
@@ -199,38 +230,144 @@ export function fuzzySearch(lines, needle, threshold = 0.6, hintLine) {
     const needleLines = needle.split("\n");
     const needleLineCount = needleLines.length;
     const candidates = [];
+    // Pre-tokenize: do this once per call instead of per window. Per-line
+    // tokenization gives identical results to tokenize(lines.join("\n")) since
+    // tokenize splits on /\s+/ and filters empties.
+    const lineTokens = new Array(lines.length);
+    const lineTokenLengths = new Array(lines.length);
+    for (let i = 0; i < lines.length; i++) {
+        const toks = lines[i] ? tokenize(lines[i]) : [];
+        lineTokens[i] = toks;
+        lineTokenLengths[i] = toks.length;
+    }
+    const needleTokens = tokenize(needle);
+    const needleTokenCount = needleTokens.length;
+    const needleFreq = buildTokenFreq(needleTokens);
+    // The combined score is 0.6 * tScore + 0.4 * lScore. Since lScore <= 1.0,
+    // a window can only reach `threshold` if tScore >= (threshold - 0.4) / 0.6.
+    // Use this as a hard prefilter before doing the expensive LCS / Levenshtein.
+    const minTScore = Math.max(0, (threshold - 0.4) / 0.6);
     // Window sizes: ±30% of original line count, minimum 1
     const minWindow = Math.max(1, Math.floor(needleLineCount * 0.7));
     const maxWindow = Math.min(lines.length - 1, Math.ceil(needleLineCount * 1.3) + 1);
+    // Track total token count per window using a rolling sum so we don't
+    // re-walk the line-token arrays for every (winSize, startLine) pair.
     for (let winSize = minWindow; winSize <= maxWindow; winSize++) {
+        if (winSize < 1)
+            continue;
+        if (winSize > lines.length - 1)
+            continue;
+        // Initial window token sum for startLine = 1.
+        let runningTokenCount = 0;
+        for (let i = 1; i < 1 + winSize && i < lines.length; i++) {
+            runningTokenCount += lineTokenLengths[i];
+        }
         for (let startLine = 1; startLine + winSize - 1 < lines.length; startLine++) {
-            const windowLines = lines.slice(startLine, startLine + winSize);
-            const windowText = windowLines.join("\n");
-            const score = combinedScore(needle, windowText);
-            if (score >= threshold) {
-                candidates.push({
-                    text: windowText,
-                    line: startLine,
-                    endLine: startLine + winSize - 1,
-                    startColumn: 0,
-                    endColumn: windowLines[windowLines.length - 1].length,
-                    score,
-                });
+            const endLineExclusive = startLine + winSize;
+            const M = runningTokenCount;
+            // Cheap skip: empty windows or empty needle can't fuzzy-match.
+            if (needleTokenCount > 0 && M > 0) {
+                // Build a frequency map for the window tokens. Cheaper than LCS:
+                // O(M) instead of O(M * needleTokenCount). We allocate a fresh map
+                // per window; tried a rolling map and the bookkeeping was a wash
+                // with realistic windowSize but if profiling demands it can be
+                // converted.
+                const windowFreq = new Map();
+                for (let li = startLine; li < endLineExclusive; li++) {
+                    const arr = lineTokens[li];
+                    for (let ti = 0; ti < arr.length; ti++) {
+                        const tok = arr[ti];
+                        windowFreq.set(tok, (windowFreq.get(tok) ?? 0) + 1);
+                    }
+                }
+                // Multiset intersection is an upper bound on LCS length. If we
+                // can't reach minTScore even with a perfect LCS, skip the window.
+                const common = multisetIntersection(needleFreq, windowFreq);
+                const tScoreUpperBound = common / Math.max(needleTokenCount, M);
+                if (tScoreUpperBound >= minTScore) {
+                    // Build the flat window token array only when we actually need
+                    // to compute LCS.
+                    const windowTokens = new Array(M);
+                    let wi = 0;
+                    for (let li = startLine; li < endLineExclusive; li++) {
+                        const arr = lineTokens[li];
+                        for (let ti = 0; ti < arr.length; ti++) {
+                            windowTokens[wi++] = arr[ti];
+                        }
+                    }
+                    const lcs = lcsLength(needleTokens, windowTokens);
+                    const tScore = lcs / Math.max(needleTokenCount, M);
+                    if (tScore >= minTScore) {
+                        // Defer expensive string join + Levenshtein until we know the
+                        // tScore could plausibly reach threshold.
+                        const windowLines = lines.slice(startLine, endLineExclusive);
+                        const windowText = windowLines.join("\n");
+                        let lScore;
+                        if (needle.length < 500 && windowText.length < 500) {
+                            lScore = levenshteinScore(needle, windowText);
+                        }
+                        else {
+                            lScore = tScore; // fall back to token score only
+                        }
+                        const score = tScore * 0.6 + lScore * 0.4;
+                        if (score >= threshold) {
+                            candidates.push({
+                                text: windowText,
+                                line: startLine,
+                                endLine: startLine + winSize - 1,
+                                startColumn: 0,
+                                endColumn: windowLines[windowLines.length - 1].length,
+                                score,
+                            });
+                        }
+                    }
+                }
+            }
+            // Slide window by one: drop startLine token count, add endLine token count.
+            const nextStart = startLine + 1;
+            const nextEndExclusive = nextStart + winSize;
+            if (nextEndExclusive <= lines.length) {
+                runningTokenCount += lineTokenLengths[endLineExclusive] - lineTokenLengths[startLine];
             }
         }
     }
-    // For single-line needles, also try substring matching within each line
+    // For single-line needles, also try substring matching within each line.
+    // Prefilter each line by token overlap before running the inner sliding
+    // window — long lines without enough needle-token coverage are skipped
+    // wholesale, since no substring of such a line can exceed its parent
+    // line's token-overlap with the needle.
     if (needleLineCount === 1 && needle.length < 200) {
+        // The minimum number of needle tokens that must appear in a line for any
+        // substring of that line to plausibly reach minTScore. tScore =
+        // lcs / max(needleN, subN), so lcs >= minTScore * needleN is required,
+        // and lcs <= number of common tokens.
+        const requiredLineCommon = Math.max(0, Math.ceil(needleTokenCount * minTScore));
         for (let lineNum = 1; lineNum < lines.length; lineNum++) {
             const line = lines[lineNum];
             if (!line)
                 continue;
-            // Sliding window within line
+            if (needleTokenCount > 0 && requiredLineCommon > 0) {
+                const lineFreq = buildTokenFreq(lineTokens[lineNum]);
+                const lineCommon = multisetIntersection(needleFreq, lineFreq);
+                if (lineCommon < requiredLineCommon)
+                    continue;
+            }
+            // Sliding window within line. Step by 1 by default for accuracy, but
+            // for very long lines and long needles, we coarsen the column step to
+            // avoid quadratic-in-line-length behaviour. The natural `len = winLen`
+            // is always swept densely so we don't miss the most likely match.
             const winLen = needle.length;
             const minWinLen = Math.max(3, Math.floor(winLen * 0.7));
             const maxWinLen = Math.min(line.length, Math.ceil(winLen * 1.3));
             for (let len = minWinLen; len <= maxWinLen; len++) {
-                for (let col = 0; col + len <= line.length; col++) {
+                // Coarsen the column step proportionally to needle length once the
+                // sub-window is no longer the natural needle length. This trades a
+                // tiny bit of recall for an order-of-magnitude speedup on long
+                // lines with many candidate offsets.
+                const colStep = len === winLen
+                    ? 1
+                    : Math.max(1, Math.floor(winLen / 16));
+                for (let col = 0; col + len <= line.length; col += colStep) {
                     const sub = line.substring(col, col + len);
                     const score = combinedScore(needle, sub);
                     if (score >= threshold) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mrsf/cli",
-  "version": "0.4.2",
+  "version": "0.4.3",
   "description": "CLI and library for the Markdown Review Sidecar Format (MRSF)",
   "type": "module",
   "main": "./dist/index.js",