@adeu/core 1.22.0 → 1.25.0

package/src/ingest.ts

@@ -9,19 +9,46 @@ import {
   get_run_text,
   apply_formatting_to_segments,
   iter_block_items,
-  iter_document_parts,
+  iter_document_parts_with_kind,
   iter_paragraph_content,
 } from "./utils/docx.js";
 import { findChild } from "./docx/dom.js";
 import { build_structural_appendix } from "./domain.js";
 import { extract_comments_data } from "./comments.js";
 
+/** Text-space extent of one projected table row plus its cell texts. */
+export interface RowGeometry {
+  start: number;
+  end: number;
+  cells: string[];
+}
+
+/** Text-space extent of one projected top-level table. */
+export interface TableGeometry {
+  start: number;
+  end: number;
+  rows: RowGeometry[];
+}
+
+/**
+ * Structural map of a projection: which offset ranges belong to which OPC
+ * part, and where top-level table rows live. Produced in the SAME pass as
+ * the text, so offsets always agree with it. Consumed by the diff pipeline
+ * to keep generated edits from crossing part boundaries (QA 2026-07-18 C1)
+ * and to emit structured row operations for table changes (QA C2).
+ */
+export interface ExtractStructure {
+  part_ranges: [number, number, string][]; // [start, end, kind]
+  tables: TableGeometry[];
+}
+
 export async function extractTextFromBuffer(
   buffer: Buffer,
   cleanView = false,
+  includeAppendix = true,
 ): Promise<string> {
   const doc = await DocumentObject.load(buffer);
-  return _extractTextFromDoc(doc, cleanView) as string;
+  return _extractTextFromDoc(doc, cleanView, includeAppendix) as string;
 }
 
 export function _extractTextFromDoc(
@@ -29,14 +56,21 @@ export function _extractTextFromDoc(
   cleanView = false,
   includeAppendix = true,
   return_paragraph_offsets = false,
-): string | { text: string; paragraph_offsets: Map<any, [number, number]> } {
+  return_structure = false,
+):
+  | string
+  | { text: string; paragraph_offsets: Map<any, [number, number]> }
+  | { text: string; structure: ExtractStructure } {
   const comments_map = extract_comments_data(doc.pkg);
 
   const full_text: string[] = [];
   const paragraph_offsets = new Map<any, [number, number]>();
+  const structure: ExtractStructure | null = return_structure
+    ? { part_ranges: [], tables: [] }
+    : null;
   let cursor = 0;
 
-  for (const part of iter_document_parts(doc)) {
+  for (const [part, part_kind] of iter_document_parts_with_kind(doc)) {
     const part_cursor = full_text.length > 0 ? cursor + 2 : cursor;
     const part_text = _extract_blocks(
       part,
@@ -44,10 +78,14 @@ export function _extractTextFromDoc(
       cleanView,
       part_cursor,
       return_paragraph_offsets ? paragraph_offsets : undefined,
+      structure ? structure.tables : undefined,
     );
     if (part_text) {
       if (full_text.length > 0) cursor += 2;
       full_text.push(part_text);
+      if (structure) {
+        structure.part_ranges.push([cursor, cursor + part_text.length, part_kind]);
+      }
       cursor += part_text.length;
     }
   }
@@ -62,15 +100,24 @@ export function _extractTextFromDoc(
   if (return_paragraph_offsets) {
     return { text: base_text, paragraph_offsets };
   }
+  if (structure) {
+    return { text: base_text, structure };
+  }
   return base_text;
 }
 
+/**
+ * table_acc: optional list collecting TableGeometry for TOP-LEVEL tables.
+ * Deliberately not forwarded into cells — nested tables stay invisible to
+ * the structured row-op diff, whose row pairing assumes one flat grid.
+ */
 function _extract_blocks(
   container: any,
   comments_map: any,
   cleanView: boolean,
   cursor: number,
   paragraph_offsets?: Map<any, [number, number]>,
+  table_acc?: TableGeometry[],
 ): string {
   const part = container.part || container;
   const [style_cache, default_pstyle] = _get_style_cache(part);
@@ -129,17 +176,25 @@ function _extract_blocks(
       is_first_para = false;
       is_first_block = false;
     } else if (item instanceof Table) {
+      const geometry: TableGeometry | null = table_acc
+        ? { start: block_start, end: block_start, rows: [] }
+        : null;
       const table_text = extract_table(
         item,
         comments_map,
         cleanView,
         block_start,
         paragraph_offsets,
+        geometry,
       );
       if (table_text) {
         blocks.push(table_text);
         local_cursor = block_start + table_text.length;
         is_first_block = false;
+        if (geometry && table_acc) {
+          geometry.end = block_start + table_text.length;
+          table_acc.push(geometry);
+        }
       } else if (!is_first_block) {
         local_cursor -= 2;
       }
@@ -156,6 +211,7 @@ export function extract_table(
   cleanView: boolean,
   cursor: number,
   paragraph_offsets?: Map<any, [number, number]>,
+  geometry?: TableGeometry | null,
 ): string {
   const rows_text: string[] = [];
   let rows_processed = 0;
@@ -221,6 +277,13 @@ export function extract_table(
 
     rows_text.push(row_str);
     local_cursor = row_start + row_str.length;
+    if (geometry) {
+      geometry.rows.push({
+        start: row_start,
+        end: local_cursor,
+        cells: [...cell_texts],
+      });
+    }
     rows_processed++;
   }
 
@@ -424,7 +487,16 @@ export function build_paragraph_text(
       else if (ev.type === "del_end") delete active_del[ev.id];
       else if (ev.type === "fmt_start") active_fmt[ev.id] = ev;
       else if (ev.type === "fmt_end") delete active_fmt[ev.id];
-      else if (ev.type === "footnote" || ev.type === "endnote") {
+      else if (ev.type === "image") {
+        // Read-only image marker (QA 2026-07-18 M5); hidden with its run in
+        // clean view when it sits inside an active tracked deletion.
+        if (!(cleanView && Object.keys(active_del).length > 0)) {
+          const alt = (ev.date || "image")
+            .replace(/\]/g, ")")
+            .replace(/\n/g, " ");
+          parts.push(`![${alt}](docx-image:${ev.id})`);
+        }
+      } else if (ev.type === "footnote" || ev.type === "endnote") {
         if (pending_text) {
           parts.push(
             `${current_wrappers[0]}${pending_text}${current_wrappers[1]}`,

package/src/mapper.ts

@@ -11,7 +11,7 @@ import {
   is_heading_paragraph,
   is_native_heading,
   iter_block_items,
-  iter_document_parts,
+  iter_document_parts_with_kind,
   iter_paragraph_content,
 } from "./utils/docx.js";
 
@@ -25,6 +25,12 @@ export interface TextSpan {
   del_id?: string | null;
   hyperlink_id?: string | null;
   comment_ids?: string[];
+  // Which OPC part (index into DocumentMapper.part_ranges) this span was
+  // projected from. Text edits may never resolve across two different
+  // parts — the QA 2026-07-18 C1 corruption wrote body text into a footer.
+  part_index: number;
+  // True for the read-only image marker projection ![alt](docx-image:N).
+  is_image_marker?: boolean;
 }
 
 export function renumber_snapshot_ids(
@@ -120,6 +126,11 @@ export class DocumentMapper {
   public full_text: string = "";
   public spans: TextSpan[] = [];
   public appendix_start_index: number = -1;
+  // [start, end, kind] per projected part, in projection order. Spans carry
+  // the matching index in .part_index. Together these let the engine refuse
+  // or re-anchor edits at OPC part boundaries (QA 2026-07-18 C1).
+  public part_ranges: [number, number, string][] = [];
+  private _current_part_index = 0;
   private _text_chunks: string[] = [];
   private _plain_projection: [string, number[]] | null = null;
 
@@ -141,9 +152,15 @@ export class DocumentMapper {
     this._text_chunks = [];
     this.full_text = "";
     this._plain_projection = null;
+    this.part_ranges = [];
+    this._current_part_index = 0;
 
-    for (const part of iter_document_parts(this.doc)) {
+    let part_idx = 0;
+    for (const [part, part_kind] of iter_document_parts_with_kind(this.doc)) {
+      this._current_part_index = part_idx;
+      const part_start = current_offset;
       current_offset = this._map_blocks(part, current_offset);
+      this.part_ranges.push([part_start, current_offset, part_kind]);
 
       if (
         this.spans.length > 0 &&
@@ -152,6 +169,7 @@ export class DocumentMapper {
         this._add_virtual_text("\n\n", current_offset, null);
         current_offset += 2;
       }
+      part_idx++;
     }
 
     while (
@@ -166,6 +184,51 @@ export class DocumentMapper {
     this.appendix_start_index = -1;
   }
 
+  /** [part_index, start, end, kind] for parts that projected any text. */
+  private _nonempty_part_ranges(): [number, number, number, string][] {
+    const out: [number, number, number, string][] = [];
+    for (let i = 0; i < this.part_ranges.length; i++) {
+      const [s, e, k] = this.part_ranges[i];
+      if (e > s) out.push([i, s, e, k]);
+    }
+    return out;
+  }
+
+  public part_kind_of(part_index: number): string | null {
+    if (part_index >= 0 && part_index < this.part_ranges.length) {
+      return this.part_ranges[part_index][2];
+    }
+    return null;
+  }
+
+  /** Kind of the part whose projected range contains `index`, or null. */
+  public part_kind_at(index: number): string | null {
+    for (const [, start, end, kind] of this._nonempty_part_ranges()) {
+      if (start <= index && index <= end) return kind;
+    }
+    return null;
+  }
+
+  /**
+   * When `index` falls strictly AFTER one part's text and at-or-before the
+   * start of the next part's text (i.e. inside the "\n\n" separator or
+   * exactly at the next part's first character), returns
+   * [previous_part_index, next_part_index]. Returns null everywhere else —
+   * including index == previous part's end, which is an ordinary
+   * end-of-part text position, not a boundary gap.
+   */
+  public part_boundary_at(index: number): [number, number] | null {
+    const ranges = this._nonempty_part_ranges();
+    for (let j = 1; j < ranges.length; j++) {
+      const [prev_i, , prev_end] = ranges[j - 1];
+      const [next_i, next_start] = ranges[j];
+      if (prev_end < index && index <= next_start) {
+        return [prev_i, next_i];
+      }
+    }
+    return null;
+  }
+
   private _map_blocks(container: any, offset: number): number {
     let current = offset;
     const c_type = container.constructor.name;
@@ -335,6 +398,7 @@ export class DocumentMapper {
       text: "",
       run: null,
       paragraph,
+      part_index: this._current_part_index,
     };
     this.spans.push(span);
 
@@ -377,6 +441,7 @@ export class DocumentMapper {
             del_id: d_id || undefined,
             hyperlink_id: active_hyperlink_id || undefined,
             comment_ids: c_ids.length > 0 ? c_ids : undefined,
+            part_index: this._current_part_index,
           };
           this.spans.push(s);
           this._text_chunks.push(txt);
@@ -609,7 +674,21 @@ export class DocumentMapper {
         else if (ev.type === "del_end") delete active_del[ev.id];
         else if (ev.type === "fmt_start") active_fmt[ev.id] = ev;
         else if (ev.type === "fmt_end") delete active_fmt[ev.id];
-        else if (ev.type === "footnote" || ev.type === "endnote") {
+        else if (ev.type === "image") {
+          // Read-only image marker (QA 2026-07-18 M5). Hidden when the run is
+          // filtered by the active view, exactly like its neighbouring text.
+          const hidden =
+            (this.clean_view && Object.keys(active_del).length > 0) ||
+            (this.original_view && Object.keys(active_ins).length > 0);
+          if (!hidden) {
+            const alt = (ev.date || "image")
+              .replace(/\]/g, ")")
+              .replace(/\n/g, " ");
+            const txt = `![${alt}](docx-image:${ev.id})`;
+            this._add_virtual_text(txt, current, paragraph, null, true);
+            current += txt.length;
+          }
+        } else if (ev.type === "footnote" || ev.type === "endnote") {
           flush_pending_runs();
           current_wrappers = ["", ""];
           current_style = ["", ""];
@@ -743,6 +822,7 @@ export class DocumentMapper {
     offset: number,
     context_paragraph: Paragraph | null,
     hyperlink_id: string | null = null,
+    is_image_marker = false,
   ) {
     const span: TextSpan = {
       start: offset,
@@ -751,6 +831,8 @@ export class DocumentMapper {
       run: null,
       paragraph: context_paragraph,
       hyperlink_id: hyperlink_id || undefined,
+      part_index: this._current_part_index,
+      is_image_marker: is_image_marker || undefined,
     };
     this.spans.push(span);
     this._text_chunks.push(text);

package/src/markup.ts

@@ -1,5 +1,14 @@
 import { trim_common_context } from "./diff.js";
 import { ModifyText } from "./models.js";
+import { RegexTimeoutError, userFindAllMatches } from "./utils/safe-regex.js";
+
+/** One entry per input edit in apply_edits_to_markdown's edit_reports. */
+export interface MarkupEditReport {
+  index: number;
+  status: "applied" | "failed";
+  error: string | null;
+  occurrences: number;
+}
 export const AMBIGUITY_EXAMPLES_CAP = 5;
 export const AMBIGUITY_CONTEXT_CHARS = 50;
 function _should_strip_markers(text: string, marker: string): boolean {
@@ -51,6 +60,45 @@ export function _replace_smart_quotes(text: string): string {
     .replace(/’/g, "'");
 }
 
+/**
+ * Strips markdown formatting markers and builds a position map.
+ * Returns [stripped_text, position_map] where position_map[i] = original
+ * index. Mirrors Python markup._strip_markdown_for_matching.
+ */
+function _strip_markdown_for_matching(text: string): [string, number[]] {
+  const result: string[] = [];
+  const position_map: number[] = [];
+  let i = 0;
+
+  while (i < text.length) {
+    // Skip ** or __
+    const pair = text.substring(i, i + 2);
+    if (i < text.length - 1 && (pair === "**" || pair === "__")) {
+      i += 2;
+      continue;
+    }
+    // Skip single * or _ that look like markdown (at word boundaries)
+    if (text[i] === "*" || text[i] === "_") {
+      const prev_char = i > 0 ? text[i - 1] : " ";
+      const next_char = i < text.length - 1 ? text[i + 1] : " ";
+      // If at boundary (space or start/end), likely markdown
+      if (
+        [" ", "\n", "\t"].includes(prev_char) ||
+        [" ", "\n", "\t"].includes(next_char)
+      ) {
+        i += 1;
+        continue;
+      }
+    }
+
+    position_map.push(i);
+    result.push(text[i]);
+    i += 1;
+  }
+
+  return [result.join(""), position_map];
+}
+
 function _find_safe_boundaries(
   text: string,
   start: number,
@@ -198,35 +246,107 @@ export function _find_match_in_text(
   text: string,
   target: string,
 ): [number, number] {
-  if (!target) return [-1, -1];
+  const matches = _find_all_matches_in_text(text, target);
+  if (matches.length > 0) return matches[0];
+  return [-1, -1];
+}
+
+/**
+ * Every non-overlapping match of `target` in `text` as [start, end] pairs,
+ * using the SAME strategy ladder as the apply engine's
+ * DocumentMapper.find_all_match_indices: regex (when requested) or
+ * exact → smart-quote-normalized → fuzzy. Markup previews must resolve
+ * matching identically to apply, or the preview lies (QA 2026-07-18 M1).
+ */
+export function _find_all_matches_in_text(
+  text: string,
+  target: string,
+  is_regex = false,
+): [number, number][] {
+  if (!target) return [];
+
+  if (is_regex) {
+    // Same semantics as the mapper: budgeted user regex; an invalid pattern
+    // simply produces no matches (surfaced as "not found").
+    // RegexTimeoutError propagates for a clean per-edit error.
+    try {
+      return userFindAllMatches(target, text).map((m) => [m.start, m.end]);
+    } catch (e) {
+      if (e instanceof RegexTimeoutError) throw e;
+      return [];
+    }
+  }
+
+  // Literal indexOf scans (no RegExp over an arbitrarily long escaped
+  // target, mirroring the mapper's exact tiers).
+  const findAllLiteral = (
+    haystack: string,
+    needle: string,
+  ): [number, number][] => {
+    const out: [number, number][] = [];
+    let from = 0;
+    while (true) {
+      const idx = haystack.indexOf(needle, from);
+      if (idx === -1) break;
+      out.push([idx, idx + needle.length]);
+      from = idx + needle.length;
+    }
+    return out;
+  };
 
-  let idx = text.indexOf(target);
-  if (idx !== -1) return _find_safe_boundaries(text, idx, idx + target.length);
+  // 1. Exact matches
+  let spans = findAllLiteral(text, target);
+  if (spans.length > 0) {
+    return spans.map(([s, e]) => _find_safe_boundaries(text, s, e));
+  }
 
+  // 2. Smart quote normalization
   const norm_text = _replace_smart_quotes(text);
   const norm_target = _replace_smart_quotes(target);
-  idx = norm_text.indexOf(norm_target);
-  if (idx !== -1)
-    return _find_safe_boundaries(text, idx, idx + norm_target.length);
+  spans = findAllLiteral(norm_text, norm_target);
+  if (spans.length > 0) {
+    return spans.map(([s, e]) => _find_safe_boundaries(text, s, e));
+  }
+
+  // 3. Markdown-stripped match, mirroring the mapper's strip-markdown and
+  // plain-projection rungs: a plain target must find text whose projection
+  // carries **bold**/_italic_ markers (even mid-word), and a marked target
+  // must find plain text.
+  const [stripped_text, pos_map] = _strip_markdown_for_matching(norm_text);
+  const [stripped_target] = _strip_markdown_for_matching(norm_target);
+  if (
+    stripped_target &&
+    (stripped_text !== norm_text || stripped_target !== norm_target)
+  ) {
+    const results: [number, number][] = [];
+    for (const [p_start, p_end] of findAllLiteral(stripped_text, stripped_target)) {
+      const raw_start = pos_map[p_start];
+      const raw_end = pos_map[p_end - 1] + 1;
+      results.push(_find_safe_boundaries(text, raw_start, raw_end));
+    }
+    if (results.length > 0) return results;
+  }
 
+  // 4. Fuzzy regex matches (handles markdown noise, list markers, etc.).
+  // The [*_]* character classes in _make_fuzzy_regex prevent catastrophic
+  // backtracking.
   try {
-    const pattern = new RegExp(_make_fuzzy_regex(target));
-    const match = pattern.exec(text);
-    if (match) {
-      const raw_start = match.index;
-      const raw_end = match.index + match[0].length;
+    const pattern = new RegExp(_make_fuzzy_regex(target), "g");
+    const results: [number, number][] = [];
+    for (const match of text.matchAll(pattern)) {
       const [refined_start, refined_end] = _refine_match_boundaries(
         text,
-        raw_start,
-        raw_end,
+        match.index!,
+        match.index! + match[0].length,
       );
-      return _find_safe_boundaries(text, refined_start, refined_end);
+      results.push(_find_safe_boundaries(text, refined_start, refined_end));
     }
+    if (results.length > 0) return results;
   } catch (e) {
     // Ignore regex compilation errors from edge cases
   }
 
-  return [-1, -1];
+  return [];
 }
 
 export function _build_critic_markup(
@@ -283,31 +403,93 @@ export function _build_critic_markup(
   return parts.join("");
 }
 
+/**
+ * Applies edits to Markdown text and returns CriticMarkup-annotated output.
+ *
+ * Edit resolution follows the SAME semantics as `apply` (QA 2026-07-18 M1):
+ *   - `regex: true` targets match as regular expressions
+ *   - `match_mode` is honored: "strict" (default) refuses ambiguous targets,
+ *     "first" marks the first occurrence, "all" marks every occurrence
+ *   - a missing target is a per-edit failure, never a silent skip
+ *
+ * When `edit_reports` is provided, one entry per input edit is appended:
+ * { index: 0-based input position, status: "applied"|"failed",
+ *   error: string|null, occurrences: number }.
+ */
 export function apply_edits_to_markdown(
   markdown_text: string,
   edits: ModifyText[],
   include_index = false,
   highlight_only = false,
+  edit_reports?: MarkupEditReport[],
 ): string {
   if (!edits || edits.length === 0) return markdown_text;
 
+  const _report = (
+    idx: number,
+    status: "applied" | "failed",
+    error: string | null = null,
+    occurrences = 0,
+  ) => {
+    if (edit_reports) edit_reports.push({ index: idx, status, error, occurrences });
+  };
+
+  // Step 1: Find match positions for each edit
   const matched_edits: [number, number, string, ModifyText, number][] = [];
 
   for (let idx = 0; idx < edits.length; idx++) {
     const edit = edits[idx];
     const target = edit.target_text || "";
+    const match_mode = (edit as any).match_mode || "strict";
+    const is_regex = Boolean((edit as any).regex);
 
     if (!target) {
+      _report(
+        idx,
+        "failed",
+        `- Edit ${idx + 1} Failed: target_text is empty. Pure insertions are expressed as a ` +
+          "replacement: put the text immediately around the insertion point in target_text and " +
+          "repeat it (plus the new text) in new_text.",
+      );
+      continue;
+    }
+
+    let spans: [number, number][];
+    try {
+      spans = _find_all_matches_in_text(markdown_text, target, is_regex);
+    } catch (e) {
+      if (!(e instanceof RegexTimeoutError)) throw e;
+      _report(idx, "failed", `- Edit ${idx + 1} Failed: ${e.message}`);
+      continue;
+    }
+
+    if (spans.length === 0) {
+      _report(
+        idx,
+        "failed",
+        `- Edit ${idx + 1} Failed: Target text not found in document:\n  "${target.substring(0, 80)}"`,
+      );
       continue;
     }
 
-    const [start, end] = _find_match_in_text(markdown_text, target);
-    if (start === -1) continue;
+    if (spans.length > 1 && match_mode === "strict") {
+      _report(
+        idx,
+        "failed",
+        format_ambiguity_error(idx + 1, target, markdown_text, spans),
+      );
+      continue;
+    }
 
-    const actual_matched_text = markdown_text.substring(start, end);
-    matched_edits.push([start, end, actual_matched_text, edit, idx]);
+    const selected =
+      match_mode === "strict" || match_mode === "first" ? spans.slice(0, 1) : spans;
+    for (const [start, end] of selected) {
+      matched_edits.push([start, end, markdown_text.substring(start, end), edit, idx]);
+    }
+    _report(idx, "applied", null, selected.length);
   }
 
+  // Step 2: Check for overlapping edits
   const matched_edits_filtered: [number, number, string, ModifyText, number][] =
     [];
   const occupied_ranges: [number, number][] = [];
@@ -319,6 +501,16 @@ export function apply_edits_to_markdown(
     for (const [occ_start, occ_end] of occupied_ranges) {
       if (start < occ_end && end > occ_start) {
         overlaps = true;
+        if (edit_reports) {
+          const msg = `- Edit ${orig_idx + 1} Failed: overlaps with a previously matched edit.`;
+          for (const r of edit_reports) {
+            if (r.index === orig_idx) {
+              r.status = "failed";
+              r.error = msg;
+              r.occurrences = 0;
+            }
+          }
+        }
         break;
       }
     }