mixdog 0.7.1

package/scripts/edit-normalize-fuzz.mjs

@@ -0,0 +1,130 @@
+// Fuzz test: _findActualString invariants under random drift.
+//
+// Approach: pick a random substring of randomly-generated content, apply
+// one of several deterministic drift transforms (NFC/NFD swap, codepoint
+// fold variants, trailing-whitespace injection, indent shift, line-end
+// drop) to the substring, and feed it through _findActualString.
+//
+// Invariants checked when the matcher returns a non-null slice:
+//   • returned string IS a substring of `content` (content.includes(slice))
+//   • content.indexOf(slice) finds a position (no synthesised bytes)
+//   • slice.normalize('NFD').indexOf(searchAfterDriftNfd) === 0 or close
+//     — the matched slice's NFD form contains the search's NFD form at
+//     the start, confirming the matcher did not pick something unrelated
+//
+// Seed: a deterministic LCG so the same DEV_FUZZ_SEED env reproduces
+// failures bit-for-bit.
+
+import { findActualString as _findActualString } from '../src/agent/orchestrator/tools/edit-normalize.mjs';
+
+const ITERATIONS = Number(process.env.DEV_FUZZ_ITER) || 1000;
+const SEED = Number(process.env.DEV_FUZZ_SEED) || 0xC0DE1234;
+
+// LCG PRNG for reproducible fuzz.
+let _state = SEED >>> 0;
+function rand() {
+    _state = (_state * 1664525 + 1013904223) >>> 0;
+    return _state / 0x100000000;
+}
+function randInt(min, max) {
+    return min + Math.floor(rand() * (max - min + 1));
+}
+function pick(arr) {
+    return arr[Math.floor(rand() * arr.length)];
+}
+
+const ASCII = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_(){}[],.;:!? \t\n';
+const HANGUL = '가나다라마바사아자차카타파하각간갈감갑값나라말사람';
+const FOLD_VARIANTS = '     　‐–—―−－‘’“”＂＇';
+
+function randomChar() {
+    const r = rand();
+    if (r < 0.70) return ASCII[Math.floor(rand() * ASCII.length)];
+    if (r < 0.85) return HANGUL[Math.floor(rand() * HANGUL.length)];
+    return FOLD_VARIANTS[Math.floor(rand() * FOLD_VARIANTS.length)];
+}
+
+function randomContent() {
+    const len = randInt(80, 300);
+    let out = '';
+    for (let i = 0; i < len; i++) out += randomChar();
+    return out;
+}
+
+// Drift transforms — each takes a slice and produces a possibly different
+// representation. The matcher must still locate the original slice.
+const drifts = [
+    (s) => s,                                              // identity
+    (s) => s.normalize('NFC'),                             // NFC
+    (s) => s.normalize('NFD'),                             // NFD
+    (s) => s.replace(/-/g, '—'),                       // hyphen → em dash
+    (s) => s.replace(/'/g, '‘'),                       // straight → curly single
+    (s) => s.replace(/"/g, '“'),                       // straight → curly double
+    (s) => s.replace(/ /g, ' '),                       // space → NBSP
+    (s) => s.replace(/—/g, '-'),                       // em dash → hyphen
+    (s) => s.replace(/‘/g, "'"),                       // curly → straight
+    (s) => s.replace(/“/g, '"'),                       // curly → straight
+    (s) => s.replace(/ /g, ' '),                       // NBSP → space
+    (s) => s.split('\n').map((l) => l + '   ').join('\n'),  // add trailing spaces
+    (s) => s.split('\n').map((l) => l.replace(/ +$/, '')).join('\n'), // strip trailing
+    (s) => s.split('\n').map((l) => '  ' + l).join('\n'),   // add leading indent
+    (s) => s.split('\n').map((l) => l.replace(/^ +/, '')).join('\n'), // strip leading
+];
+
+let matched = 0;
+let unmatched = 0;
+let invariantFails = 0;
+const failSamples = [];
+
+for (let i = 0; i < ITERATIONS; i++) {
+    const content = randomContent();
+    const start = randInt(0, Math.max(0, content.length - 10));
+    const len = randInt(3, Math.min(60, content.length - start));
+    const original = content.substring(start, start + len);
+    if (original.length < 2) continue;
+    const drift = pick(drifts);
+    const search = drift(original);
+    const info = {};
+    let result;
+    try {
+        result = _findActualString(content, search, info);
+    } catch (err) {
+        invariantFails++;
+        failSamples.push({ reason: 'throw', err: String(err), content, search });
+        if (failSamples.length > 5) break;
+        continue;
+    }
+    if (result === null) {
+        unmatched++;
+        continue;
+    }
+    matched++;
+    if (typeof result !== 'string') {
+        invariantFails++;
+        failSamples.push({ reason: 'non-string', result, content, search });
+        if (failSamples.length > 5) break;
+        continue;
+    }
+    if (!content.includes(result)) {
+        invariantFails++;
+        failSamples.push({ reason: 'not-substring', result, content, search });
+        if (failSamples.length > 5) break;
+    }
+}
+
+console.log(`edit-normalize-fuzz: ${ITERATIONS} iterations (seed=0x${SEED.toString(16).toUpperCase()})`);
+console.log(`  matched:         ${matched}`);
+console.log(`  unmatched:       ${unmatched}`);
+console.log(`  invariant fails: ${invariantFails}`);
+
+if (invariantFails > 0) {
+    console.error('\nFAIL samples:');
+    for (const s of failSamples) {
+        console.error(`  reason: ${s.reason}`);
+        console.error(`    content: ${JSON.stringify(s.content).slice(0, 200)}`);
+        console.error(`    search:  ${JSON.stringify(s.search).slice(0, 200)}`);
+        if (s.result !== undefined) console.error(`    result:  ${JSON.stringify(s.result).slice(0, 200)}`);
+        if (s.err) console.error(`    err:     ${s.err}`);
+    }
+    process.exit(1);
+}

package/scripts/edit-normalize-smoke.mjs

@@ -0,0 +1,401 @@
+// Smoke test: edit-normalize.mjs edit-input normalization helpers.
+//
+// Validates the invariant-safe matching helpers extracted live from the
+// implementation so the smoke stays in sync:
+//   • _normalizeForMatch  — curly-quote → straight fold (length-preserving).
+//   • _nfcFoldMatch       — NFC/NFD canonical matcher (byte-exact slice).
+//   • _stripTrailingWhitespacePerLine — insert-side trailing space/tab strip
+//                                       with line-terminator preservation.
+//   • _preserveQuoteTypography — heuristic insert-side typography helper.
+//
+// Last-resort rstrip-fold / indent-fold / eol-fold tiers are enabled with
+// per-tier ambiguity rejection (diagnoseFoldTierAmbiguity). Case, dash,
+// fullwidth and Unicode-space folding remain removed.
+//
+// Invariants:
+//   1. Every folded codepoint is a single UTF-16 unit (BMP only).
+//   2. Fold preserves input.length so indexOf hits re-index original bytes.
+//   3. nfc-fold returns content.substring(start, end) — never synthesised.
+//   4. Trailing-whitespace strip preserves LF / CRLF / lone CR byte-exact.
+//   5. Non-curly codepoints (dash, fullwidth, space, ZWSP, BOM, ASCII)
+//      never match the fold.
+
+import {
+    normalizeForMatch as _normalizeForMatch,
+    nfcFoldMatch as _nfcFoldMatch,
+    findActualString as _findActualString,
+    diagnoseFoldTierAmbiguity as _diagnoseFoldTierAmbiguity,
+    diagnoseEolFoldAmbiguity as _diagnoseEolFoldAmbiguity,
+    stripTrailingWhitespacePerLine as _stripTrailingWhitespacePerLine,
+    preserveQuoteTypography as _preserveQuoteTypography,
+} from '../src/agent/orchestrator/tools/edit-normalize.mjs';
+
+let failures = 0;
+function fail(name, msg) { console.error(`FAIL ${name}: ${msg}`); failures++; }
+
+// ── Section 1: fold codepoint matrix (curly quotes ONLY) ───────────────────
+const matchCases = [
+    ['U+2018 left single quote',      '\u2018', "'"],
+    ['U+2019 right single quote',     '\u2019', "'"],
+    ['U+201A single low-9',           '\u201A', "'"],
+    ['U+201B single high-rev-9',      '\u201B', "'"],
+    ['U+201C left double quote',      '\u201C', '"'],
+    ['U+201D right double quote',     '\u201D', '"'],
+    ['U+201E double low-9',           '\u201E', '"'],
+    ['U+201F double high-rev-9',      '\u201F', '"'],
+];
+
+// Heuristic-risky folds intentionally removed: these must pass through
+// UNCHANGED now (each could map genuinely-different text to the same view).
+const skipCases = [
+    ['U+2010 hyphen',                 '\u2010'],
+    ['U+2013 en dash',                '\u2013'],
+    ['U+2014 em dash',                '\u2014'],
+    ['U+2015 horizontal bar',         '\u2015'],
+    ['U+2212 minus',                  '\u2212'],
+    ['U+FF0D fullwidth hyphen-minus', '\uFF0D'],
+    ['U+FF07 fullwidth apostrophe',   '\uFF07'],
+    ['U+FF02 fullwidth quote',        '\uFF02'],
+    ['U+00A0 NBSP',                   '\u00A0'],
+    ['U+2003 em space',               '\u2003'],
+    ['U+202F narrow NBSP',            '\u202F'],
+    ['U+205F medium math space',      '\u205F'],
+    ['U+3000 ideographic space',      '\u3000'],
+    ['U+200B ZWSP',                   '\u200B'],
+    ['U+FEFF BOM',                    '\uFEFF'],
+    ['U+0020 ASCII space',            ' '],
+    ['U+002D ASCII hyphen',           '-'],
+    ['U+0027 ASCII apos',             "'"],
+    ['U+0022 ASCII quote',            '"'],
+    ['U+0041 ASCII A',                'A'],
+    ['U+AC00 Hangul GA',              '\uAC00'],
+];
+
+for (const [name, input, expected] of matchCases) {
+    if (input.length !== 1) { fail(name, `single UTF-16 unit expected, got length ${input.length}`); continue; }
+    const folded = _normalizeForMatch(input);
+    if (folded !== expected) { fail(name, `folded to ${JSON.stringify(folded)}, expected ${JSON.stringify(expected)}`); continue; }
+    if (folded.length !== input.length) { fail(name, `length drift (${input.length} -> ${folded.length})`); continue; }
+}
+for (const [name, input] of skipCases) {
+    const folded = _normalizeForMatch(input);
+    if (folded !== input) { fail(name, `should not be folded, got ${JSON.stringify(folded)}`); }
+}
+
+// Mixed string: curly quotes fold, dash + ideographic space stay verbatim.
+const mixed = 'foo\u2014bar baz\u2018qux\u2019\u3000end';
+const mixedFolded = _normalizeForMatch(mixed);
+if (mixedFolded !== 'foo\u2014bar baz\'qux\'\u3000end') fail('mixed-fold', `got ${JSON.stringify(mixedFolded)}`);
+if (mixedFolded.length !== mixed.length) fail('mixed-fold-length', `${mixed.length} -> ${mixedFolded.length}`);
+
+// ── Section 2: findActualString pipeline (exact -> fold -> nfc) ─────────────
+// Curly-quote fold tier still recovers a unique straight-vs-curly mismatch.
+{
+    const content = 'the value is \u201Chello\u201D today\n';
+    const search = 'the value is "hello" today';
+    const info = {};
+    const slice = _findActualString(content, search, info);
+    if (slice === null) fail('pipeline:curly-fold', 'expected a fold match, got null');
+    else if (info.stage !== 'fold') fail('pipeline:curly-fold', `expected stage fold, got ${info.stage}`);
+    else if (!content.includes(slice)) fail('pipeline:curly-fold', 'returned slice is NOT a substring of content');
+}
+
+// Fold-tier uniqueness gate: two curly variants of the same straight search
+// must collapse to null rather than silently picking the first.
+{
+    const content = 'say \u2018hi\u2019 / say \u201Bhi\u2019\n';
+    const search = "say 'hi'";
+    const info = {};
+    const slice = _findActualString(content, search, info);
+    if (slice !== null) fail('pipeline:fold-multi-match-rejected', `expected null, got ${JSON.stringify(slice)} (stage=${info.stage})`);
+}
+
+// rstrip-fold: unique trailing-whitespace drift on a line body.
+{
+    const content = 'header\nfoo   \nbar\nfooter\n';
+    const search = 'foo\nbar';
+    const info = {};
+    const slice = _findActualString(content, search, info);
+    if (slice === null) fail('pipeline:rstrip-fold', 'expected unique match');
+    else if (info.stage !== 'rstrip-fold') fail('pipeline:rstrip-fold', `stage ${info.stage}`);
+    else if (slice !== 'foo   \nbar') fail('pipeline:rstrip-fold', `slice ${JSON.stringify(slice)}`);
+}
+
+// indent-fold: leading tabs in file vs spaces in search (CC tab width 2).
+{
+    const content = 'header\n\tif (x) {\n\t\ty;\n\t}\nfooter\n';
+    const search = '  if (x) {\n    y;\n  }';
+    const info = {};
+    const slice = _findActualString(content, search, info);
+    if (slice === null) fail('pipeline:indent-fold', 'expected unique match');
+    else if (info.stage !== 'indent-fold') fail('pipeline:indent-fold', `stage ${info.stage}`);
+    else if (!content.includes(slice)) fail('pipeline:indent-fold', 'slice not substring');
+}
+
+// eol-fold: search has trailing newline; file slice does not.
+{
+    const content = 'alpha';
+    const search = 'alpha\n';
+    const info = {};
+    const slice = _findActualString(content, search, info);
+    if (slice === null) fail('pipeline:eol-fold', 'expected match');
+    else if (info.stage !== 'eol-fold') fail('pipeline:eol-fold', `stage ${info.stage}`);
+    else if (slice !== 'alpha') fail('pipeline:eol-fold', `got ${JSON.stringify(slice)}`);
+}
+
+// eol-fold must not match bare text mid-line.
+{
+    const content = 'prefix alpha suffix\n';
+    const search = 'alpha\n';
+    const slice = _findActualString(content, search, {});
+    if (slice !== null) fail('pipeline:eol-fold-midline-reject', JSON.stringify(slice));
+}
+
+// crlf-fold wins over rstrip-fold when both sites exist.
+{
+    const content = 'foo  \nbar\nzzz\nfoo\r\nbar\n';
+    const search = 'foo\nbar';
+    const info = {};
+    const slice = _findActualString(content, search, info);
+    if (slice === null) fail('pipeline:crlf-beats-rstrip', 'expected crlf site');
+    else if (info.stage !== 'crlf-fold') fail('pipeline:crlf-beats-rstrip', `stage ${info.stage}`);
+    else if (slice !== 'foo\r\nbar') fail('pipeline:crlf-beats-rstrip', `slice ${JSON.stringify(slice)}`);
+}
+
+// Ambiguity: two rstrip-fold sites → diagnose code 9 payload.
+{
+    const content = 'foo   \nbar\nzzz\nfoo   \nbar\n';
+    const search = 'foo\nbar';
+    const amb = _diagnoseFoldTierAmbiguity(content, search);
+    if (!amb || amb.stage !== 'rstrip-fold' || amb.count !== 2) {
+        fail('pipeline:rstrip-ambiguity', JSON.stringify(amb));
+    }
+}
+
+// Ambiguity: two indent-fold sites.
+{
+    const content = 'header\n\tif (a) {}\nmid\n\tif (a) {}\nfooter\n';
+    const search = '  if (a) {}';
+    const slice = _findActualString(content, search, {});
+    const amb = _diagnoseFoldTierAmbiguity(content, search);
+    if (slice !== null) fail('pipeline:indent-ambiguity-find', JSON.stringify(slice));
+    if (!amb || amb.stage !== 'indent-fold' || amb.count !== 2) {
+        fail('pipeline:indent-ambiguity', JSON.stringify(amb));
+    }
+}
+
+// Ambiguity: two eol-fold boundary sites.
+{
+    const content = 'alpha\nbeta\nalpha';
+    const search = 'alpha\n';
+    const amb = _diagnoseEolFoldAmbiguity(content, search);
+    if (!amb || amb.stage !== 'eol-fold' || amb.count !== 2) {
+        fail('pipeline:eol-ambiguity', JSON.stringify(amb));
+    }
+}
+
+// Whole-block re-indent still does not match (indent-shift not enabled).
+{
+    const content = 'header\nif (x) {\n    y;\n}\nfooter\n';
+    const search = '    if (x) {\n        y;\n    }';
+    const slice = _findActualString(content, search, {});
+    if (slice !== null) fail('pipeline:indent-shift-still-removed', JSON.stringify(slice));
+}
+
+// ── Section 3: _stripTrailingWhitespacePerLine (insert-side helper) ─────────
+const stripCases = [
+    { name: 'LF-trailing-spaces',     input: 'foo  \nbar\t\nbaz\n',         output: 'foo\nbar\nbaz\n' },
+    { name: 'CRLF-preserved',         input: 'foo  \r\nbar\t\r\nbaz\r\n',   output: 'foo\r\nbar\r\nbaz\r\n' },
+    { name: 'lone-CR-preserved',      input: 'foo  \rbar\t\rbaz\r',         output: 'foo\rbar\rbaz\r' },
+    { name: 'no-trailing-no-change',  input: 'foo\nbar\nbaz\n',             output: 'foo\nbar\nbaz\n' },
+    { name: 'mixed-LF-CRLF',          input: 'foo  \nbar  \r\nbaz \n',      output: 'foo\nbar\r\nbaz\n' },
+    { name: 'final-line-no-terminator', input: 'foo  \nbar  ',              output: 'foo\nbar' },
+    { name: 'blank-lines-untouched',  input: 'foo\n\n  \nbar\n',            output: 'foo\n\n\nbar\n' },
+];
+
+for (const { name, input, output } of stripCases) {
+    const got = _stripTrailingWhitespacePerLine(input);
+    if (got !== output) { fail(`strip:${name}`, `got ${JSON.stringify(got)}, expected ${JSON.stringify(output)}`); }
+}
+
+// ── Section 4: _nfcFoldMatch ────────────────────────────────────────────────
+// 가 NFC = U+AC00 (1 char); NFD = U+1100 U+1161 (2 chars).
+// 각 = U+AC01 (1 NFC) vs U+1100 U+1161 U+11A8 (3 NFD).
+const NFC_GA = '\uAC00';
+const NFD_GA = '\u1100\u1161';
+const NFC_GAK = '\uAC01';
+const NFD_GAK = '\u1100\u1161\u11A8';
+
+const nfcCases = [
+    {
+        name: 'NFD-content-NFC-search-hangul',
+        content: 'prefix-' + NFD_GAK + '-suffix\n',
+        search: 'prefix-' + NFC_GAK + '-suffix',
+        expected: 'prefix-' + NFD_GAK + '-suffix',
+    },
+    {
+        name: 'NFC-content-NFD-search-hangul',
+        content: 'prefix-' + NFC_GAK + '-suffix\n',
+        search: 'prefix-' + NFD_GAK + '-suffix',
+        expected: 'prefix-' + NFC_GAK + '-suffix',
+    },
+    {
+        name: 'NFC-no-drift-skip',
+        content: 'plain text',
+        search: 'plain text',
+        expected: null,
+    },
+    {
+        name: 'NFC-multi-match-rejected',
+        content: NFD_GAK + ' / ' + NFD_GAK + '\n',
+        search: NFC_GAK,
+        expected: null,
+    },
+    {
+        name: 'NFC-not-found',
+        content: 'completely unrelated\n',
+        search: NFC_GAK,
+        expected: null,
+    },
+    {
+        name: 'NFC-empty-search',
+        content: NFD_GA + '\n',
+        search: '',
+        expected: null,
+    },
+    {
+        name: 'NFC-cafe-latin',
+        content: 'cafe\u0301 menu\n',
+        search: 'caf\u00E9 menu',
+        expected: 'cafe\u0301 menu',
+    },
+    {
+        name: 'NFC-partial-line-byte-slice',
+        content: 'header\nthe ' + NFD_GA + ' is here\nfooter\n',
+        search: NFC_GA,
+        expected: NFD_GA,
+    },
+];
+
+for (const { name, content, search, expected } of nfcCases) {
+    const got = _nfcFoldMatch(content, search);
+    if (got !== expected) { fail(`nfc:${name}`, `got ${JSON.stringify(got)}, expected ${JSON.stringify(expected)}`); continue; }
+    if (got !== null && !content.includes(got)) { fail(`nfc:${name}`, `returned slice is NOT a substring of content`); }
+}
+
+// ── Section 5: edge cases ───────────────────────────────────────────────────
+const edgeCases = [
+    {
+        name: 'fold-empty-string',
+        run: () => _normalizeForMatch(''),
+        expected: '',
+    },
+    {
+        name: 'fold-pure-ascii-no-change',
+        run: () => _normalizeForMatch('plain ASCII line 123'),
+        expected: 'plain ASCII line 123',
+    },
+    {
+        name: 'strip-empty-input',
+        run: () => _stripTrailingWhitespacePerLine(''),
+        expected: '',
+    },
+    {
+        name: 'strip-only-newlines',
+        run: () => _stripTrailingWhitespacePerLine('\n\n\n'),
+        expected: '\n\n\n',
+    },
+    {
+        name: 'strip-tab-only-line',
+        run: () => _stripTrailingWhitespacePerLine('foo\n\t\nbar\n'),
+        expected: 'foo\n\nbar\n',
+    },
+    {
+        name: 'fold-BOM-not-folded',
+        run: () => _normalizeForMatch('\uFEFFfoo'),
+        expected: '\uFEFFfoo',
+    },
+    {
+        name: 'fold-ZWSP-not-folded',
+        run: () => _normalizeForMatch('foo\u200Bbar'),
+        expected: 'foo\u200Bbar',
+    },
+];
+
+for (const { name, run, expected } of edgeCases) {
+    const got = run();
+    if (got !== expected) { fail(`edge:${name}`, `got ${JSON.stringify(got)}, expected ${JSON.stringify(expected)}`); }
+}
+
+// ── Section 6: preserveQuoteTypography (heuristic exception, CC parity) ──
+// L/R curly quote codepoints — explicit escape to avoid encoding ambiguity.
+const LSQ = '\u2018', RSQ = '\u2019', LDQ = '\u201C', RDQ = '\u201D';
+const typoCases = [
+    {
+        name: 'double-quote-preserved',
+        old: 'foo "bar" baz',
+        matched: `foo ${LDQ}bar${RDQ} baz`,
+        new_: 'updated "content" here',
+        expected: `updated ${LDQ}content${RDQ} here`,
+    },
+    {
+        name: 'oldstring-equals-matched-no-change',
+        old: 'plain text',
+        matched: 'plain text',
+        new_: 'still plain "ascii"',
+        expected: 'still plain "ascii"',
+    },
+    {
+        name: 'no-curly-in-matched-no-change',
+        old: 'foo "bar"',
+        matched: 'foo "bar"',
+        new_: 'new "stuff"',
+        expected: 'new "stuff"',
+    },
+    {
+        name: 'contraction-right-single',
+        old: "it's a test",
+        matched: `it${RSQ}s a test`,
+        new_: "they're here",
+        expected: `they${RSQ}re here`,
+    },
+    {
+        name: 'opening-single-quote',
+        old: "say 'hi'",
+        matched: `say ${LSQ}hi${RSQ}`,
+        new_: "say 'bye'",
+        expected: `say ${LSQ}bye${RSQ}`,
+    },
+    {
+        name: 'opening-at-string-start',
+        old: '"hello"',
+        matched: `${LDQ}hello${RDQ}`,
+        new_: '"world"',
+        expected: `${LDQ}world${RDQ}`,
+    },
+    {
+        name: 'opening-after-em-dash',
+        old: 'said \u2014 "yes"',
+        matched: `said \u2014 ${LDQ}yes${RDQ}`,
+        new_: 'said \u2014 "no"',
+        expected: `said \u2014 ${LDQ}no${RDQ}`,
+    },
+];
+
+for (const { name, old, matched, new_, expected } of typoCases) {
+    const got = _preserveQuoteTypography(old, matched, new_);
+    if (got !== expected) {
+        fail(`typo:${name}`, `got ${JSON.stringify(got)}, expected ${JSON.stringify(expected)}`);
+    }
+}
+
+const totalChecks =
+    matchCases.length + skipCases.length + 2
+    + 7 + stripCases.length + nfcCases.length + edgeCases.length
+    + typoCases.length;
+
+if (failures > 0) {
+    console.error(`\n${failures} smoke check(s) failed.`);
+    process.exit(1);
+}
+console.log(`edit-normalize-smoke: ${totalChecks} checks passed.`);