@cueloop/schema 0.1.0-alpha.42 → 0.1.0-alpha.43

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cueloop/schema",
-  "version": "0.1.0-alpha.42",
+  "version": "0.1.0-alpha.43",
   "description": "The ReviewSession primitive: types, markdown block model, quote-primary anchors, feedback serialization",
   "homepage": "https://github.com/mmurakaru/cueloop#readme",
   "bugs": {

package/src/anchor.test.ts

@@ -103,6 +103,67 @@ describe("anchor cascade", () => {
     expect(resolved.start).toBe(5);
   });
 
+  test("binds a quote that still carries its leading list marker", () => {
+    // Arrange
+    // an agent copied the source line verbatim, including the "- " bullet, but
+    // the parsed list block strips the marker from its text
+    const blocks = blocksOf("## Notes\n\n- This is a dummy plan - confirm the item");
+    const anchor = { quote: "- This is a dummy plan", prefix: "", suffix: "" };
+
+    // Act
+    const resolved = resolveAnchor(anchor, blocks)!;
+
+    // Assert
+    expect(resolved).not.toBeNull();
+    expect(resolved.strategy).toBe("normalized");
+    expect(blocks[resolved.blockIndex]!.text.slice(resolved.start, resolved.end)).toBe(
+      "This is a dummy plan",
+    );
+  });
+
+  test("binds a heading quote that still carries its leading hashes", () => {
+    // Arrange
+    const blocks = blocksOf(DOC);
+    const anchor = { quote: "## Storage", prefix: "", suffix: "" };
+
+    // Act
+    const resolved = resolveAnchor(anchor, blocks)!;
+
+    // Assert
+    expect(resolved.strategy).toBe("normalized");
+    expect(blocks[resolved.blockIndex]!.text).toBe("Storage");
+  });
+
+  test("fuzzy tier binds a lightly edited quote", () => {
+    // Arrange
+    const blocks = blocksOf(DOC);
+    const contextBlockIndex = blocks.findIndex((block) => block.text.startsWith("Review sessions"));
+    // a typo the reviewer never saw: "sesssions"
+    const anchor = { quote: "Review sesssions currently live", prefix: "", suffix: "" };
+
+    // Act
+    const resolved = resolveAnchor(anchor, blocks)!;
+
+    // Assert
+    expect(resolved.strategy).toBe("fuzzy");
+    expect(resolved.blockIndex).toBe(contextBlockIndex);
+    expect(resolved.approximate).toBe(true);
+  });
+
+  test("exact matches report the exact strategy", () => {
+    // Arrange
+    const blocks = blocksOf(DOC);
+    const contextBlockIndex = blocks.findIndex((block) => block.text.startsWith("Review sessions"));
+    const anchor = makeAnchor(blocks, contextBlockIndex, 0, 15);
+
+    // Act
+    const resolved = resolveAnchor(anchor, blocks)!;
+
+    // Assert
+    expect(resolved.strategy).toBe("exact");
+    expect(resolved.approximate).toBe(false);
+  });
+
   test("trimmed-quote fallback marks the resolution approximate", () => {
     // Arrange
     const anchor = {

package/src/anchor.ts

@@ -1,25 +1,38 @@
 /**
  * Anchor resolution: the quote-primary cascade. Anchors resolve against the
- * current block list:
- *   1. exact quote in a block (prefix/suffix selectors break ties,
- *      position hint breaks remaining ties)
- *   2. trimmed quote
- *   3. orphan - keep the quote for display, drop the highlight.
- * Never silently bind to the wrong text.
+ * current block list, each tier tried only when the one above it misses:
+ *   1. exact      - the quote appears verbatim in a block
+ *   2. trimmed    - the quote appears once surrounding whitespace is dropped
+ *   3. normalized - the quote appears once a leading markdown block marker
+ *                   (bullet, heading, ordered, quote) is stripped, matching how
+ *                   the parser strips block text
+ *   4. fuzzy      - the quote was lightly edited but is still recognizably there
+ *   5. orphan     - none matched; keep the quote for display, drop the highlight
+ * Prefix/suffix selectors and the recorded position break ties within a tier.
+ * Never silently bind to the wrong text: fuzzy needs a high similarity floor.
  */
 
 import type { Anchor } from "./types";
-import type { Block } from "./markdown";
+import { type Block, stripLeadingBlockMarker } from "./markdown";
+import { fuzzyFindBestMatch } from "./fuzzy";
 
 /** Characters of surrounding text captured as prefix/suffix selectors. */
 const ANCHOR_CONTEXT_CHARS = 24;
 
+/** Minimum similarity a fuzzy match needs before the resolver will trust it. */
+const FUZZY_MINIMUM_SIMILARITY = 0.75;
+
+/** How the quote bound to the block text, weakest binding named last. */
+export type AnchorMatchStrategy = "exact" | "trimmed" | "normalized" | "fuzzy";
+
 export interface ResolvedAnchor {
   blockIndex: number;
   start: number;
   end: number;
-  /** True when the quote had to be trimmed to bind. */
+  /** True whenever the binding was not an exact quote match. */
   approximate: boolean;
+  /** Which cascade tier bound the quote. */
+  strategy: AnchorMatchStrategy;
 }
 
 export function makeAnchor(
@@ -39,11 +52,33 @@ export function makeAnchor(
   };
 }
 
-interface Candidate extends ResolvedAnchor {
+interface Candidate {
+  blockIndex: number;
+  start: number;
+  end: number;
   score: number;
 }
 
-function findCandidates(anchor: Anchor, blocks: Block[], quote: string): Candidate[] {
+/** Score a candidate window by how well its context and position agree. */
+function contextScore(
+  anchor: Anchor,
+  blockIndex: number,
+  block: Block,
+  start: number,
+  end: number,
+): number {
+  const prefix = block.text.slice(Math.max(0, start - ANCHOR_CONTEXT_CHARS), start);
+  const suffix = block.text.slice(end, end + ANCHOR_CONTEXT_CHARS);
+  let score = 0;
+  if (prefix === anchor.prefix) score += 2;
+  if (suffix === anchor.suffix) score += 2;
+  if (anchor.blockIndex === blockIndex) score += 1;
+  if (anchor.blockIndex === blockIndex && anchor.start === start) score += 1;
+  return score;
+}
+
+/** Every verbatim occurrence of `quote` across the blocks, scored by context. */
+function findExactCandidates(anchor: Anchor, blocks: Block[], quote: string): Candidate[] {
   const candidates: Candidate[] = [];
   blocks.forEach((block, blockIndex) => {
     const text = block.text;
@@ -52,42 +87,88 @@ function findCandidates(anchor: Anchor, blocks: Block[], quote: string): Candida
       matchStart !== -1;
       matchStart = text.indexOf(quote, matchStart + 1)
     ) {
-      const prefix = text.slice(Math.max(0, matchStart - ANCHOR_CONTEXT_CHARS), matchStart);
-      const suffix = text.slice(
-        matchStart + quote.length,
-        matchStart + quote.length + ANCHOR_CONTEXT_CHARS,
-      );
-      let score = 0;
-      if (prefix === anchor.prefix) score += 2;
-      if (suffix === anchor.suffix) score += 2;
-      if (anchor.blockIndex === blockIndex) score += 1;
-      if (anchor.blockIndex === blockIndex && anchor.start === matchStart) score += 1;
+      const matchEnd = matchStart + quote.length;
       candidates.push({
         blockIndex,
         start: matchStart,
-        end: matchStart + quote.length,
-        approximate: false,
-        score,
+        end: matchEnd,
+        score: contextScore(anchor, blockIndex, block, matchStart, matchEnd),
       });
     }
   });
   return candidates;
 }
 
+/**
+ * How much a full context agreement (0-6) may nudge the fuzzy ranking. Kept far
+ * below any real similarity gap so text quality always leads and context only
+ * breaks near-ties - a stale position hint must never pull the anchor onto a
+ * meaningfully less-similar passage.
+ */
+const FUZZY_CONTEXT_TIE_BREAK = 0.01;
+
+/** The best fuzzy window per block, ranked by similarity with context as a tiebreak. */
+function findFuzzyCandidates(
+  anchor: Anchor,
+  blocks: Block[],
+  needle: string,
+  minimumSimilarity: number,
+): Candidate[] {
+  const candidates: Candidate[] = [];
+  blocks.forEach((block, blockIndex) => {
+    const match = fuzzyFindBestMatch(needle, block.text, minimumSimilarity);
+    if (match === null) return;
+    const score =
+      match.similarity +
+      contextScore(anchor, blockIndex, block, match.start, match.end) * FUZZY_CONTEXT_TIE_BREAK;
+    candidates.push({ blockIndex, start: match.start, end: match.end, score });
+  });
+  return candidates;
+}
+
+/** Highest-scoring candidate, or null when there are none. */
+function pickBest(candidates: Candidate[]): Candidate | null {
+  if (candidates.length === 0) return null;
+  candidates.sort((left, right) => right.score - left.score);
+  return candidates[0]!;
+}
+
+function resolved(candidate: Candidate, strategy: AnchorMatchStrategy): ResolvedAnchor {
+  return {
+    blockIndex: candidate.blockIndex,
+    start: candidate.start,
+    end: candidate.end,
+    approximate: strategy !== "exact",
+    strategy,
+  };
+}
+
 /** Resolve an anchor against the current blocks, or null when orphaned. */
 export function resolveAnchor(anchor: Anchor, blocks: Block[]): ResolvedAnchor | null {
   if (anchor.quote === "") return null;
-  let candidates = findCandidates(anchor, blocks, anchor.quote);
-  let approximate = false;
-  if (candidates.length === 0) {
-    const trimmed = anchor.quote.trim();
-    if (trimmed !== "" && trimmed !== anchor.quote) {
-      candidates = findCandidates(anchor, blocks, trimmed);
-      approximate = true;
-    }
+
+  const exact = pickBest(findExactCandidates(anchor, blocks, anchor.quote));
+  if (exact) return resolved(exact, "exact");
+
+  const trimmedQuote = anchor.quote.trim();
+  if (trimmedQuote !== "" && trimmedQuote !== anchor.quote) {
+    const trimmed = pickBest(findExactCandidates(anchor, blocks, trimmedQuote));
+    if (trimmed) return resolved(trimmed, "trimmed");
   }
-  if (candidates.length === 0) return null;
-  candidates.sort((left, right) => right.score - left.score);
-  const best = candidates[0]!;
-  return { ...best, approximate };
+
+  const normalizedQuote = stripLeadingBlockMarker(anchor.quote).trim();
+  if (normalizedQuote !== "" && normalizedQuote !== trimmedQuote) {
+    const normalized = pickBest(findExactCandidates(anchor, blocks, normalizedQuote));
+    if (normalized) return resolved(normalized, "normalized");
+  }
+
+  const fuzzyNeedle = normalizedQuote !== "" ? normalizedQuote : trimmedQuote;
+  if (fuzzyNeedle !== "") {
+    const fuzzy = pickBest(
+      findFuzzyCandidates(anchor, blocks, fuzzyNeedle, FUZZY_MINIMUM_SIMILARITY),
+    );
+    if (fuzzy) return resolved(fuzzy, "fuzzy");
+  }
+
+  return null;
 }

package/src/fuzzy.test.ts

@@ -0,0 +1,92 @@
+import { describe, expect, test } from "bun:test";
+import { fuzzyFindBestMatch, levenshteinDistance, similarityRatio } from "./fuzzy";
+
+describe("levenshteinDistance", () => {
+  test("identical strings cost nothing", () => {
+    // Arrange / Act / Assert
+    expect(levenshteinDistance("anchor", "anchor")).toBe(0);
+  });
+
+  test("counts a single substitution", () => {
+    // Arrange / Act / Assert
+    expect(levenshteinDistance("plan", "plane")).toBe(1);
+  });
+
+  test("empty against non-empty costs the whole length", () => {
+    // Arrange / Act / Assert
+    expect(levenshteinDistance("", "quote")).toBe(5);
+  });
+
+  test("is symmetric", () => {
+    // Arrange / Act / Assert
+    expect(levenshteinDistance("kitten", "sitting")).toBe(levenshteinDistance("sitting", "kitten"));
+  });
+});
+
+describe("similarityRatio", () => {
+  test("two empty strings are identical", () => {
+    // Arrange / Act / Assert
+    expect(similarityRatio("", "")).toBe(1);
+  });
+
+  test("identical strings score one", () => {
+    // Arrange / Act / Assert
+    expect(similarityRatio("review", "review")).toBe(1);
+  });
+
+  test("a one-character typo in a long string stays close to one", () => {
+    // Arrange
+    const original = "demonstration plan";
+
+    // Act
+    const ratio = similarityRatio(original, "demonstration plann");
+
+    // Assert
+    expect(ratio).toBeGreaterThan(0.9);
+  });
+
+  test("unrelated strings score low", () => {
+    // Arrange / Act / Assert
+    expect(similarityRatio("abcdef", "zyxwvu")).toBeLessThan(0.2);
+  });
+});
+
+describe("fuzzyFindBestMatch", () => {
+  test("finds a lightly edited window and reports its offsets", () => {
+    // Arrange
+    const haystack = "the quick brown fox jumps";
+
+    // Act
+    const match = fuzzyFindBestMatch("quikc brown", haystack, 0.75)!;
+
+    // Assert
+    expect(match).not.toBeNull();
+    expect(haystack.slice(match.start, match.end)).toBe("quick brown");
+    expect(match.similarity).toBeGreaterThan(0.75);
+  });
+
+  test("returns null when nothing clears the similarity floor", () => {
+    // Arrange / Act
+    const match = fuzzyFindBestMatch("completely unrelated", "short text", 0.75);
+
+    // Assert
+    expect(match).toBeNull();
+  });
+
+  test("an empty needle or haystack never matches", () => {
+    // Arrange / Act / Assert
+    expect(fuzzyFindBestMatch("", "text", 0.75)).toBeNull();
+    expect(fuzzyFindBestMatch("text", "", 0.75)).toBeNull();
+  });
+
+  test("ties keep the earliest window", () => {
+    // Arrange
+    const haystack = "word word";
+
+    // Act
+    const match = fuzzyFindBestMatch("word", haystack, 0.9)!;
+
+    // Assert
+    expect(match.start).toBe(0);
+  });
+});

package/src/fuzzy.ts

@@ -0,0 +1,94 @@
+/**
+ * Approximate string matching for anchor re-binding. The anchor resolver falls
+ * back to these when exact and marker-normalized quote lookups both miss - the
+ * quoted text is still present but was lightly edited (a word changed, a typo
+ * fixed, punctuation adjusted). Everything here is pure and offset-exact so it
+ * can be tested in isolation from the resolver.
+ */
+
+/** A window of the haystack that best matches the needle. */
+export interface FuzzyMatch {
+  /** Inclusive start offset into the haystack. */
+  start: number;
+  /** Exclusive end offset into the haystack. */
+  end: number;
+  /** Similarity of the matched window to the needle, in [0, 1]. */
+  similarity: number;
+}
+
+/**
+ * Levenshtein edit distance: the fewest single-character insertions, deletions,
+ * or substitutions that turn `left` into `right`. Uses two rolling rows, so it
+ * costs O(left.length * right.length) time and O(right.length) space.
+ */
+export function levenshteinDistance(left: string, right: string): number {
+  if (left === right) return 0;
+  if (left.length === 0) return right.length;
+  if (right.length === 0) return left.length;
+
+  let previousRow: number[] = Array.from({ length: right.length + 1 }, (_unused, column) => column);
+  let currentRow: number[] = Array.from({ length: right.length + 1 }, () => 0);
+
+  for (let leftIndex = 1; leftIndex <= left.length; leftIndex++) {
+    currentRow[0] = leftIndex;
+    for (let rightIndex = 1; rightIndex <= right.length; rightIndex++) {
+      const substitutionCost = left[leftIndex - 1] === right[rightIndex - 1] ? 0 : 1;
+      currentRow[rightIndex] = Math.min(
+        currentRow[rightIndex - 1]! + 1, // insertion
+        previousRow[rightIndex]! + 1, // deletion
+        previousRow[rightIndex - 1]! + substitutionCost, // substitution
+      );
+    }
+    const swap = previousRow;
+    previousRow = currentRow;
+    currentRow = swap;
+  }
+
+  return previousRow[right.length]!;
+}
+
+/**
+ * Similarity of two strings in [0, 1]: 1 is identical, 0 shares nothing. Derived
+ * from the edit distance normalized by the longer string. Two empty strings are
+ * defined as identical (1).
+ */
+export function similarityRatio(left: string, right: string): number {
+  const longestLength = Math.max(left.length, right.length);
+  if (longestLength === 0) return 1;
+  return 1 - levenshteinDistance(left, right) / longestLength;
+}
+
+/**
+ * Find the haystack window most similar to the needle, or null when nothing
+ * clears `minimumSimilarity`. Windows range around the needle length so the
+ * match tolerates a few inserted or deleted characters, not only substitutions.
+ * Ties keep the earliest window. Offsets index the haystack directly.
+ */
+export function fuzzyFindBestMatch(
+  needle: string,
+  haystack: string,
+  minimumSimilarity: number,
+): FuzzyMatch | null {
+  if (needle === "" || haystack === "") return null;
+
+  const lengthTolerance = Math.max(2, Math.round(needle.length * 0.25));
+  const minimumWindowLength = Math.max(1, needle.length - lengthTolerance);
+  const maximumWindowLength = needle.length + lengthTolerance;
+
+  let bestMatch: FuzzyMatch | null = null;
+  for (let start = 0; start < haystack.length; start++) {
+    for (
+      let windowLength = minimumWindowLength;
+      windowLength <= maximumWindowLength && start + windowLength <= haystack.length;
+      windowLength++
+    ) {
+      const window = haystack.slice(start, start + windowLength);
+      const similarity = similarityRatio(needle, window);
+      if (similarity < minimumSimilarity) continue;
+      if (bestMatch === null || similarity > bestMatch.similarity) {
+        bestMatch = { start, end: start + windowLength, similarity };
+      }
+    }
+  }
+  return bestMatch;
+}

package/src/index.ts

@@ -4,5 +4,6 @@ export * from "./markdown";
 export * from "./working-copy";
 export * from "./diff";
 export * from "./anchor";
+export * from "./fuzzy";
 export * from "./feedback";
 export * from "./herdr";

package/src/markdown.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, test } from "bun:test";
-import { parseBlocks, blockToMd, sectionOf } from "./markdown";
+import { parseBlocks, blockToMd, sectionOf, stripLeadingBlockMarker } from "./markdown";
 
 const SAMPLE = `# Title
 
@@ -122,3 +122,30 @@ describe("sectionOf", () => {
     expect(sectionOf(blocks, paragraphIndex)).toBe("Context");
   });
 });
+
+describe("stripLeadingBlockMarker", () => {
+  test("strips the marker the parser strips for every marked block kind", () => {
+    // Assert
+    expect(stripLeadingBlockMarker("# Heading")).toBe("Heading");
+    expect(stripLeadingBlockMarker("### Heading")).toBe("Heading");
+    expect(stripLeadingBlockMarker("- bullet item")).toBe("bullet item");
+    expect(stripLeadingBlockMarker("3. ordered item")).toBe("ordered item");
+    expect(stripLeadingBlockMarker("> quoted line")).toBe("quoted line");
+  });
+
+  test("agrees with the text the parser produces", () => {
+    // Arrange
+    const source = "- This is a dummy plan - confirm the item";
+
+    // Act
+    const [listBlock] = parseBlocks(source);
+
+    // Assert
+    expect(stripLeadingBlockMarker(source)).toBe(listBlock!.text);
+  });
+
+  test("leaves an unmarked line untouched", () => {
+    // Assert
+    expect(stripLeadingBlockMarker("a plain paragraph")).toBe("a plain paragraph");
+  });
+});

package/src/markdown.ts

@@ -19,16 +19,21 @@ export interface Block {
   lineEnd: number;
 }
 
+/**
+ * The leading block markers the parser strips from a line: headings (h1-h3),
+ * bullet list, ordered list, and blockquote. Single source of truth so the
+ * anchor resolver strips exactly the same set and the two never drift.
+ */
+export const LEADING_BLOCK_MARKER = /^(?:#{1,3} |- |\d+\. |> )/;
+
+/** Remove one leading markdown block marker, or return the line unchanged. */
+export function stripLeadingBlockMarker(line: string): string {
+  return line.replace(LEADING_BLOCK_MARKER, "");
+}
+
 function isMarkerLine(line: string): boolean {
   return (
-    line.startsWith("```") ||
-    line.startsWith("# ") ||
-    line.startsWith("## ") ||
-    line.startsWith("### ") ||
-    line.startsWith("> ") ||
-    line.startsWith("- ") ||
-    /^\d+\. /.test(line) ||
-    /^(---|\*\*\*|___)\s*$/.test(line)
+    line.startsWith("```") || LEADING_BLOCK_MARKER.test(line) || /^(---|\*\*\*|___)\s*$/.test(line)
   );
 }
 
@@ -55,13 +60,28 @@ export function parseBlocks(markdown: string): Block[] {
       lineIndex++;
       blocks.push({ kind: "code", text: body.join("\n"), lang, lineStart: start, lineEnd: end });
     } else if (line.startsWith("### ")) {
-      blocks.push({ kind: "h3", text: line.slice(4), lineStart: lineIndex, lineEnd: lineIndex });
+      blocks.push({
+        kind: "h3",
+        text: stripLeadingBlockMarker(line),
+        lineStart: lineIndex,
+        lineEnd: lineIndex,
+      });
       lineIndex++;
     } else if (line.startsWith("## ")) {
-      blocks.push({ kind: "h2", text: line.slice(3), lineStart: lineIndex, lineEnd: lineIndex });
+      blocks.push({
+        kind: "h2",
+        text: stripLeadingBlockMarker(line),
+        lineStart: lineIndex,
+        lineEnd: lineIndex,
+      });
       lineIndex++;
     } else if (line.startsWith("# ")) {
-      blocks.push({ kind: "h1", text: line.slice(2), lineStart: lineIndex, lineEnd: lineIndex });
+      blocks.push({
+        kind: "h1",
+        text: stripLeadingBlockMarker(line),
+        lineStart: lineIndex,
+        lineEnd: lineIndex,
+      });
       lineIndex++;
     } else if (/^(---|\*\*\*|___)\s*$/.test(line)) {
       blocks.push({ kind: "hr", text: "", lineStart: lineIndex, lineEnd: lineIndex });
@@ -70,7 +90,7 @@ export function parseBlocks(markdown: string): Block[] {
       const start = lineIndex;
       const body: string[] = [];
       while (lineIndex < lines.length && lines[lineIndex]!.startsWith("> ")) {
-        body.push(lines[lineIndex]!.slice(2));
+        body.push(stripLeadingBlockMarker(lines[lineIndex]!));
         lineIndex++;
       }
       blocks.push({
@@ -80,12 +100,17 @@ export function parseBlocks(markdown: string): Block[] {
         lineEnd: lineIndex - 1,
       });
     } else if (line.startsWith("- ")) {
-      blocks.push({ kind: "li", text: line.slice(2), lineStart: lineIndex, lineEnd: lineIndex });
+      blocks.push({
+        kind: "li",
+        text: stripLeadingBlockMarker(line),
+        lineStart: lineIndex,
+        lineEnd: lineIndex,
+      });
       lineIndex++;
     } else if (/^\d+\. /.test(line)) {
       blocks.push({
         kind: "oli",
-        text: line.replace(/^\d+\. /, ""),
+        text: stripLeadingBlockMarker(line),
         lineStart: lineIndex,
         lineEnd: lineIndex,
       });