mixdog 0.9.147 → 0.9.149

package/src/runtime/agent/orchestrator/tools/patch/matcher.mjs

@@ -3,8 +3,10 @@
 // patch.mjs; matching/fuzz semantics mirror the native engine and are
 // unchanged.
 
+import { closeSync, openSync, readFileSync, readSync, statSync } from 'node:fs';
 import { classifyEntry, stripDiffPrefix } from './paths.mjs';
 import { normalizeOutputPath } from '../builtin.mjs';
+import { detectReadEncodingFromBuffer } from '../builtin/snapshot-helpers.mjs';
 
 function collectUnifiedOldLines(hunk) {
   const oldLines = [];
@@ -39,6 +41,19 @@ export function collectUnifiedOps(hunk) {
   return ops;
 }
 
+/** collectUnifiedOps plus each op's line text (terminator inheritance needs it). */
+export function collectUnifiedOpEntries(hunk) {
+  const ops = [];
+  for (const raw of hunk?.lines || []) {
+    if (typeof raw !== 'string' || raw.length === 0) continue;
+    const tag = raw[0];
+    if (tag === ' ') ops.push({ op: 'context', line: raw.slice(1) });
+    else if (tag === '-') ops.push({ op: 'delete', line: raw.slice(1) });
+    else if (tag === '+') ops.push({ op: 'add', line: raw.slice(1) });
+  }
+  return ops;
+}
+
 // New-side lines of a unified hunk (context + additions), plus whether the
 // hunk marks the NEW content as lacking a terminating newline
 // (`` directly after a ' ' / '+' line).
@@ -186,6 +201,145 @@ export function decodeValidUtf8OrNull(buf) {
   }
 }
 
+// --- Patch-target codec ------------------------------------------------------
+// Every route that rewrites a WHOLE file from decoded text (V4A rename, the JS
+// unified route) must round-trip the file's encoding. `buf.toString('utf8')`
+// replaced each invalid byte with U+FFFD and read UTF-16 as NUL-laden garbage,
+// so the rewrite transcoded the file irreversibly. The BOM decides the codec;
+// anything that is neither valid UTF-8 nor BOM-marked UTF-16 is refused.
+// Codec of a patch target. A BOM is authoritative; a BOM-LESS file is
+// classified by NUL-byte parity, which is what separates UTF-16 text from
+// UTF-8 text. `certain:false` means the bytes cannot be attributed to one
+// codec — the caller must refuse rather than rewrite them. Routing by BOM
+// alone sent BOM-less UTF-16 to the UTF-8 engine, which produced mixed
+// UTF-16/UTF-8 bytes.
+// Minimum evidence for a BOM-less UTF-16 verdict: 16 code units. Below that a
+// NUL is just as likely to be a stray byte inside UTF-8 text (`61 00`), and a
+// wrong verdict DESTROYS the file, so short input is undecidable by
+// definition. `buf` must be the WHOLE file: a bounded prefix cannot rule out a
+// parity flip further in, so a prefix-only caller passes `partial:true` and
+// gets "undecidable" instead of a guess.
+const UTF16_MIN_EVIDENCE_BYTES = 32;
+const UTF16_MIN_NUL_RATIO = 0.5;
+
+export function detectPatchTargetCodec(buf, { partial = false } = {}) {
+  const bytes = Buffer.isBuffer(buf) ? buf : Buffer.from(buf || []);
+  const undecidable = { encoding: null, bomLen: 0, certain: false };
+  if (bytes.length >= 2 && bytes[0] === 0xFF && bytes[1] === 0xFE) {
+    return { encoding: 'utf16le', bomLen: 2, certain: true };
+  }
+  if (bytes.length >= 2 && bytes[0] === 0xFE && bytes[1] === 0xFF) {
+    return { encoding: 'utf16be', bomLen: 2, certain: true };
+  }
+  if (bytes.length >= 3 && bytes[0] === 0xEF && bytes[1] === 0xBB && bytes[2] === 0xBF) {
+    return { encoding: 'utf8', bomLen: 3, certain: true };
+  }
+  let nulEven = 0;
+  let nulOdd = 0;
+  for (let i = 0; i < bytes.length; i++) {
+    if (bytes[i] !== 0) continue;
+    if (i % 2 === 0) nulEven += 1;
+    else nulOdd += 1;
+  }
+  if (partial) return undecidable;
+  if (nulEven === 0 && nulOdd === 0) return { encoding: 'utf8', bomLen: 0, certain: true };
+  if (bytes.length < UTF16_MIN_EVIDENCE_BYTES || bytes.length % 2 !== 0) return undecidable;
+  const evenSlots = Math.ceil(bytes.length / 2);
+  const oddSlots = Math.floor(bytes.length / 2);
+  if (nulEven === 0 && nulOdd / oddSlots >= UTF16_MIN_NUL_RATIO) {
+    return { encoding: 'utf16le', bomLen: 0, certain: true };
+  }
+  if (nulOdd === 0 && nulEven / evenSlots >= UTF16_MIN_NUL_RATIO) {
+    return { encoding: 'utf16be', bomLen: 0, certain: true };
+  }
+  return undecidable;
+}
+
+export function decodePatchTargetBuffer(buf, displayPath = '') {
+  const enc = detectPatchTargetCodec(buf);
+  if (!enc.certain) {
+    throw new Error(
+      `apply_patch: ${normalizeOutputPath(displayPath)} has NUL bytes without a decidable text encoding — `
+      + 'refusing to rewrite it, because any codec choice would corrupt the file.',
+    );
+  }
+  let text;
+  if (enc.encoding === 'utf16le' || enc.encoding === 'utf16be') {
+    const body = buf.subarray(enc.bomLen);
+    // An odd trailing byte cannot be decoded and would be DROPPED on re-encode:
+    // silent data loss. Refuse the file exactly like invalid UTF-8.
+    if (body.length % 2 !== 0) {
+      throw new Error(
+        `apply_patch: ${normalizeOutputPath(displayPath)} is malformed UTF-16 (odd payload length) — `
+        + 'refusing to rewrite it, because the dangling byte cannot be preserved.',
+      );
+    }
+    text = enc.encoding === 'utf16le'
+      ? body.toString('utf16le')
+      // Node has no 'utf16be': swap pairs into LE and decode.
+      : Buffer.from(body).swap16().toString('utf16le');
+  } else {
+    text = decodeValidUtf8OrNull(buf.subarray(enc.bomLen));
+  }
+  if (typeof text !== 'string') {
+    throw new Error(
+      `apply_patch: ${normalizeOutputPath(displayPath)} is neither valid UTF-8 nor BOM-marked UTF-16 — `
+      + 'refusing to rewrite it, because the edit would transcode every byte.',
+    );
+  }
+  return { text, enc };
+}
+
+// Encoding gate for a target that is about to be rewritten. Returns the
+// refusal message, or null when the bytes round-trip. Both engines use it: the
+// native one speaks UTF-8 only, so invalid UTF-8 must be refused there exactly
+// as the JS writer refuses it, instead of being rewritten byte-lossily.
+export function patchTargetEncodingError(fullPath, displayPath = fullPath) {
+  let buf;
+  try {
+    buf = readFileSync(fullPath);
+  } catch {
+    return null; // absent/unreadable — the engine reports it in its own words
+  }
+  try {
+    decodePatchTargetBuffer(buf, displayPath);
+    return null;
+  } catch (err) {
+    const message = err?.message || String(err);
+    return /^apply_patch:/.test(message) ? message : null;
+  }
+}
+
+// Routing sniff: ANY UTF-16 target (BOM or not) must take the codec-preserving
+// JS writer instead of the UTF-8 native engine. Undecidable files stay on the
+// native route, where the encoding gate refuses them without writing.
+export function patchTargetUsesUtf16(fullPath) {
+  try {
+    // WHOLE file: a prefix cannot rule out a parity flip further in, and this
+    // decides whether the byte-preserving writer is used.
+    const codec = detectPatchTargetCodec(readFileSync(fullPath));
+    return codec.certain && codec.encoding !== 'utf8';
+  } catch {
+    return false;
+  }
+}
+
+// Re-encode a rewritten file into the codec it was read with (BOM included).
+// Plain UTF-8 stays a string so atomicWrite keeps its existing fast path.
+export function encodePatchTargetContent(text, enc) {
+  if (!enc || (enc.encoding === 'utf8' && !enc.bomLen)) return text;
+  const body = enc.encoding === 'utf16le'
+    ? Buffer.from(text, 'utf16le')
+    : (enc.encoding === 'utf16be'
+      ? Buffer.from(text, 'utf16le').swap16()
+      : Buffer.from(text, 'utf8'));
+  if (!enc.bomLen) return body;
+  const bom = enc.encoding === 'utf16le'
+    ? Buffer.from([0xFF, 0xFE])
+    : (enc.encoding === 'utf16be' ? Buffer.from([0xFE, 0xFF]) : Buffer.from([0xEF, 0xBB, 0xBF]));
+  return Buffer.concat([bom, body]);
+}
+
 export function unifiedOldLinesMatchAt(sourceLines, oldLines, startIdx, fuzz, band) {
   if (startIdx < 0 || startIdx + oldLines.length > sourceLines.length) return null;
   let fuzzUsed = 0;
@@ -344,25 +498,212 @@ export function normalizeTypographic(s) {
     .replace(/[\u00A0\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200A\u202F\u205F\u3000]/g, ' ');
 }
 
+// Splits into line CONTENT plus the exact terminator that followed each line
+// (`lines.terminators`). Line content is identical to the historical
+// CRLF-folding split — a lone CR is only a terminator in a CR-dominant file —
+// but recording the real bytes is what lets a rewrite put every untouched
+// line back with its ORIGINAL terminator instead of the file's dominant one.
 export function splitTextLinesForPatch(text) {
   const raw = String(text ?? '');
   const eol = detectDominantEol(raw);
-  const body = eol === '\r' ? raw.replace(/\r/g, '\n') : raw.replace(/\r\n/g, '\n');
-  if (body.length === 0) {
-    const empty = [];
-    empty.hasFinalNewline = true;
-    empty.eol = eol;
-    return empty;
+  const splitLoneCr = eol === '\r';
+  const lines = [];
+  const terminators = [];
+  let start = 0;
+  let i = 0;
+  while (i < raw.length) {
+    const ch = raw[i];
+    if (ch === '\n') {
+      lines.push(raw.slice(start, i));
+      terminators.push('\n');
+      i += 1;
+      start = i;
+      continue;
+    }
+    if (ch === '\r') {
+      if (raw[i + 1] === '\n') {
+        lines.push(raw.slice(start, i));
+        terminators.push('\r\n');
+        i += 2;
+        start = i;
+        continue;
+      }
+      if (splitLoneCr) {
+        lines.push(raw.slice(start, i));
+        terminators.push('\r');
+        i += 1;
+        start = i;
+        continue;
+      }
+    }
+    i += 1;
   }
-  const lines = body.split('\n');
   let hasFinalNewline = true;
-  if (lines[lines.length - 1] === '') lines.pop();
-  else hasFinalNewline = false;
+  if (start < raw.length) {
+    lines.push(raw.slice(start));
+    terminators.push('');
+    hasFinalNewline = false;
+  }
   lines.hasFinalNewline = hasFinalNewline;
   lines.eol = eol;
+  lines.terminators = terminators;
   return lines;
 }
 
+export function cloneTextLinesForPatch(sourceLines, eol) {
+  const lines = [...(sourceLines || [])];
+  lines.hasFinalNewline = sourceLines?.hasFinalNewline !== false;
+  lines.eol = eol || sourceLines?.eol || '\n';
+  lines.terminators = Array.isArray(sourceLines?.terminators)
+    ? [...sourceLines.terminators]
+    : null;
+  return lines;
+}
+
+// The ONLY mutation entry point for a patch line array. Replaced lines hand
+// their own terminator to the new line at the same offset (a 1:1 replacement
+// is byte-identical outside the changed text), extra inserted lines adopt the
+// local convention, and every line the patch did not touch keeps its bytes.
+// Op-wise terminators for one hunk's output lines. Derived from the hunk's OWN
+// ops — never from content similarity, which cannot describe interior context,
+// several change runs, or a moved line:
+//   context → keeps its source line's terminator verbatim
+//   delete  → contributes its terminator to the current run
+//   add     → takes the terminator of the delete it replaces inside that run,
+//             else the local convention.
+export function terminatorsForUnifiedOps(ops, oldTerminators, fallbackEol = '\n') {
+  const olds = Array.isArray(oldTerminators) ? oldTerminators : [];
+  const entries = (ops || []).map((entry) => (typeof entry === 'string'
+    ? { op: entry, line: undefined }
+    : { op: entry?.op, line: entry?.line }));
+  // Pre-pass: EVERY delete of the hunk with the terminator of the source line
+  // it consumes, tagged with its run. Building the pool as we walk only saw
+  // deletes already passed, so a BACKWARD move (the add precedes its delete)
+  // could not claim its own line and lost its terminator.
+  const pool = [];
+  {
+    let cursor = 0;
+    let run = 0;
+    for (const entry of entries) {
+      if (entry.op === 'context') { cursor += 1; run += 1; }
+      else if (entry.op === 'delete') {
+        pool.push({ line: entry.line, terminator: olds[cursor], used: false, run });
+        cursor += 1;
+      }
+    }
+  }
+  const out = [];
+  let oldCursor = 0;
+  let currentRun = 0;
+  let lastClaimed = -1; // pool index claimed by the previous add
+  for (const entry of entries) {
+    const { op, line } = entry;
+    if (op === 'context') {
+      const terminator = olds[oldCursor];
+      oldCursor += 1;
+      currentRun += 1;
+      out.push(terminator !== undefined ? terminator : fallbackEol);
+    } else if (op === 'delete') {
+      oldCursor += 1;
+    } else if (op === 'add') {
+      // 1) identity AFTER the previously claimed delete (order-preserving, so
+      //    duplicates are consumed in sequence instead of first-fit),
+      // 2) identity ANYWHERE in the hunk (a line moved forwards or backwards
+      //    across context keeps its own terminator),
+      // 3) the next unclaimed delete OF THIS RUN (the line it replaces).
+      let index = -1;
+      if (line !== undefined) {
+        index = pool.findIndex((slot, i) => !slot.used && slot.line === line && i > lastClaimed);
+        if (index < 0) index = pool.findIndex((slot) => !slot.used && slot.line === line);
+      }
+      if (index < 0) {
+        index = pool.findIndex((slot) => !slot.used && slot.run === currentRun);
+      }
+      let inherited;
+      if (index >= 0) {
+        pool[index].used = true;
+        lastClaimed = index;
+        inherited = pool[index].terminator;
+      }
+      out.push(inherited !== undefined ? inherited : fallbackEol);
+    }
+  }
+  return out;
+}
+
+/** Local convention for a replaced window: its own last terminator, else its neighbours'. */
+export function localTerminatorForWindow(lines, start, oldLen) {
+  const terms = lines?.terminators;
+  if (!Array.isArray(terms)) return lines?.eol || '\n';
+  const replaced = terms.slice(start, start + oldLen);
+  return (replaced.length ? replaced[replaced.length - 1] : '')
+    || terms[start - 1]
+    || terms[start]
+    || lines.eol
+    || '\n';
+}
+
+export function spliceTextLinesForPatch(lines, start, oldLen, newLines, newTerminators = null) {
+  const terms = lines?.terminators;
+  if (Array.isArray(terms)) {
+    const replacedTerms = terms.slice(start, start + oldLen);
+    const replacedLines = lines.slice(start, start + oldLen);
+    const local = localTerminatorForWindow(lines, start, oldLen);
+    const explicit = Array.isArray(newTerminators) && newTerminators.length === newLines.length
+      ? newTerminators
+      : null;
+    const nextTerms = newLines.map((line, k) => {
+      if (explicit) return explicit[k] !== undefined ? explicit[k] : local;
+      // No op information (a caller that cannot describe its edit): keep a
+      // terminator ONLY for an output line that is byte-identical to the
+      // source line at the same offset; everything else takes the local
+      // convention. Never guess across offsets.
+      return (replacedLines[k] === line && replacedTerms[k] !== undefined)
+        ? replacedTerms[k]
+        : local;
+    });
+    terms.splice(start, oldLen, ...nextTerms);
+  }
+  lines.splice(start, oldLen, ...newLines);
+}
+
+// Explicit end-of-file intent (`` in either
+// direction). The terminator array is the state of record, so the intent is
+// written INTO it; `hasFinalNewline` is kept in sync for matching code.
+export function setFinalNewlineForPatch(lines, hasNewline) {
+  if (!lines) return;
+  lines.hasFinalNewline = hasNewline;
+  const terms = lines.terminators;
+  if (!Array.isArray(terms) || terms.length === 0) return;
+  const last = terms.length - 1;
+  if (!hasNewline) terms[last] = '';
+  else if (!terms[last]) terms[last] = terms[last - 1] || lines.eol || '\n';
+}
+
+export function joinTextLinesForPatch(lines) {
+  const arr = lines || [];
+  const eol = arr.eol || '\n';
+  const terms = Array.isArray(arr.terminators) && arr.terminators.length === arr.length
+    ? arr.terminators
+    : null;
+  if (!terms) {
+    const body = arr.join(eol);
+    return arr.hasFinalNewline !== false ? `${body}${eol}` : body;
+  }
+  // Terminators are authoritative. The file-level `hasFinalNewline` flag must
+  // NOT be re-applied here: after a tail deletion the surviving last line is a
+  // line the patch never touched, and forcing the old "no final newline" state
+  // onto it removed a terminator that belonged to untouched bytes.
+  let out = '';
+  for (let i = 0; i < arr.length; i++) {
+    const isLast = i === arr.length - 1;
+    let terminator = terms[i];
+    if (!terminator && !isLast) terminator = terms[i - 1] || eol;
+    out += `${arr[i]}${terminator}`;
+  }
+  return out;
+}
+
 export function detectDominantEol(text) {
   const raw = String(text ?? '');
   if (raw.includes('\r') && !raw.includes('\n')) return '\r';