uniword 1.0.10 → 1.2.4

data/lib/uniword/docx/reconciler/body.rb

@@ -0,0 +1,197 @@
+# frozen_string_literal: true
+
+module Uniword
+  module Docx
+    class Reconciler
+      # Document body reconciliation.
+      #
+      # Handles section properties, headers/footers, and their
+      # mc:Ignorable and paragraph backfill.
+      module Body
+        def reconcile_section_properties
+          return unless package.document&.body
+
+          body = package.document.body
+
+          unless body.section_properties
+            body.section_properties = Wordprocessingml::SectionProperties.new(
+              page_size: Wordprocessingml::PageDefaults.default_page_size,
+              page_margins: Wordprocessingml::PageDefaults.default_page_margins,
+              columns: Wordprocessingml::PageDefaults.default_columns,
+              doc_grid: Wordprocessingml::PageDefaults.default_doc_grid,
+            )
+            record_fix(FixCodes::SECTION_PROPERTIES_DEFAULTED,
+                       "Added default section properties with US Letter page size")
+            return
+          end
+
+          sect_pr = body.section_properties
+          fixed = false
+          unless sect_pr.page_size
+            sect_pr.page_size = Wordprocessingml::PageDefaults.default_page_size
+            fixed = true
+          end
+          unless sect_pr.page_margins
+            sect_pr.page_margins = Wordprocessingml::PageDefaults.default_page_margins
+            fixed = true
+          end
+          unless sect_pr.columns
+            sect_pr.columns = Wordprocessingml::PageDefaults.default_columns
+            fixed = true
+          end
+          if fixed
+            record_fix(FixCodes::SECTION_PROPERTIES_DEFAULTED,
+                       "Filled missing pgSz/pgMar/cols in existing section properties")
+          end
+        end
+
+        def reconcile_headers_footers
+          ignorable = Ooxml::Types::McIgnorable.new(FULL_IGNORABLE)
+          set_header_footer_ignorable(package.document&.headers, ignorable)
+          set_header_footer_ignorable(package.document&.footers, ignorable)
+
+          unless allocator
+            rsid = generate_rsid
+            backfill_header_footer_parts(package.document&.headers, rsid, "hdr")
+            backfill_header_footer_parts(package.document&.footers, rsid, "ftr")
+
+            parts = package.document&.header_footer_parts
+            parts&.each_with_index do |part, pidx|
+              backfill_paragraphs(part[:content].paragraphs, rsid, "hfp:#{pidx}")
+            end
+          end
+
+          wire_builder_headers_footers
+        end
+
+        private
+
+        def set_header_footer_ignorable(parts, ignorable)
+          return unless parts
+
+          parts.each_value { |part| part.mc_ignorable = ignorable }
+        end
+
+        def backfill_header_footer_parts(parts, rsid, prefix)
+          return unless parts
+
+          parts.each_with_index do |(_, part), pidx|
+            backfill_paragraphs(part.paragraphs, rsid, "#{prefix}:#{pidx}")
+          end
+        end
+
+        HEADER_REL_TYPE = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/header"
+        FOOTER_REL_TYPE = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/footer"
+
+        # Wire builder-path headers/footers into document_rels and sectPr
+        # during reconciliation so referential integrity checks see valid rIds.
+        def wire_builder_headers_footers
+          doc = package.document
+          return unless doc&.body
+          return unless doc.headers || doc.footers
+
+          rels = package.document_rels
+          return unless rels
+
+          counter = 0
+          wire_parts_to_rels(doc.headers, rels, HEADER_REL_TYPE,
+                             "header", counter) do |type, r_id|
+            wire_sect_pr_reference(doc, :header, type, r_id)
+          end
+
+          counter = 0
+          wire_parts_to_rels(doc.footers, rels, FOOTER_REL_TYPE,
+                             "footer", counter) do |type, r_id|
+            wire_sect_pr_reference(doc, :footer, type, r_id)
+          end
+
+          # Clear element_order so header/footer references serialize correctly.
+          # Ordered mode only outputs elements in element_order; the template's
+          # order may not include newly-added references.
+          sect_pr = doc.body.section_properties
+          sect_pr.element_order = nil if sect_pr&.element_order
+        end
+
+        def wire_parts_to_rels(parts, rels, rel_type, file_prefix, counter)
+          return unless parts && !parts.empty?
+
+          parts.each_key do |type|
+            counter += 1
+            target = "#{file_prefix}#{counter}.xml"
+            r_id = if allocator
+                     allocator.alloc_rid(target: target, type: rel_type)
+                   else
+                     find_or_create_rel(rels, rel_type, target)
+                   end
+            yield type, r_id
+          end
+        end
+
+        def find_or_create_rel(rels, rel_type, target)
+          existing = rels.relationships.find { |r| r.target == target }
+          return existing.id if existing
+
+          r_id = Ooxml::Relationships::PackageRelationships
+            .next_available_rid(rels)
+          rels.relationships << Ooxml::Relationships::Relationship.new(
+            id: r_id, type: rel_type, target: target,
+          )
+          r_id
+        end
+
+        def wire_sect_pr_reference(doc, kind, type, r_id)
+          sect_pr = doc.body.section_properties
+          return unless sect_pr
+
+          refs = case kind
+                 when :header then sect_pr.header_references
+                 when :footer then sect_pr.footer_references
+                 end
+          return unless refs
+
+          existing = refs.find { |r| r.type == type }
+          if existing
+            existing.r_id = r_id
+          else
+            ref_class = if kind == :header
+                          Wordprocessingml::HeaderReference
+                        else
+                          Wordprocessingml::FooterReference
+                        end
+            refs << ref_class.new(type: type, r_id: r_id)
+          end
+        end
+
+        def reconcile_document_body
+          return unless profile
+          return unless package.document&.body
+
+          doc = package.document
+          set_mc_ignorable(doc, prefixes: FULL_IGNORABLE)
+
+          record_fix(FixCodes::MC_IGNORABLE, "Added mc:Ignorable to document body")
+
+          body = doc.body
+          rsid = generate_rsid
+
+          body.paragraphs.each_with_index do |para, idx|
+            strip_empty_runs(para)
+            next if allocator
+
+            para.rsid_r ||= rsid
+            para.rsid_r_default ||= "00000000"
+            para.para_id ||= generate_hex_id(idx)
+            para.text_id ||= "77777777"
+          end
+
+          record_fix(FixCodes::PARAGRAPH_BACKFILL, "Assigned rsid and paraId to paragraphs") unless allocator
+
+          sect_pr = body.section_properties
+          return unless sect_pr
+
+          sect_pr.rsid_r ||= rsid
+        end
+      end
+    end
+  end
+end

data/lib/uniword/docx/reconciler/fix_codes.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Uniword
+  module Docx
+    class Reconciler
+      # Single source of truth for record_fix rule codes.
+      #
+      # Every `record_fix` call site references a constant from this module
+      # instead of a bare string literal. Adding a new fix category means
+      # adding one constant here, not hunting for the next free letter.
+      #
+      # Wire format is preserved — external validation rules and audit-log
+      # consumers that pattern-match on "R1".."R16" continue to work.
+      #
+      # Known debt: R10 is overloaded for several distinct concerns
+      # (note defs, table structure, style defaults, dangling refs).
+      # Splitting requires consumer coordination.
+      module FixCodes
+        # Settings / package structure
+        MC_IGNORABLE = "R1"
+        DOC_ID_GENERATED = "R2"
+
+        # Support parts
+        THEME_CREATED = "R3"
+        NUMBERING_REFERENCED = "R4"
+
+        # Package assembly
+        RELATIONSHIPS_ASSEMBLED = "R6"
+        CONTENT_TYPES_ASSEMBLED = "R7"
+        APP_PROPERTIES_ENSURED = "R8"
+
+        # Notes — pair creation
+        NOTE_PAIR_CREATED = "R9"
+        NOTE_DEFINITION_CREATED = "R10"
+        NOTE_INVALID_TYPE_STRIPPED = "R15"
+        NOTE_DUPLICATE_ID_REMOVED = "R16"
+
+        # Notes — referential integrity warnings and removals
+        DANGLING_NOTE_REFERENCE_WARNING = "R9"
+        DANGLING_NOTE_REFERENCE_REMOVED = "R10"
+        DANGLING_HYPERLINK_WARNING = "R9"
+
+        # Body
+        SECTION_PROPERTIES_DEFAULTED = "R11"
+        PARAGRAPH_BACKFILL = "R12"
+
+        # Parts
+        FONT_TABLE_CREATED = "R13"
+        CORE_PROPERTIES_REBUILT = "R14"
+
+        # Styles (currently shares R10 with other concerns — see debt note above)
+        STYLE_DEFAULTS_ADDED = "R10"
+        SEMI_HIDDEN_ADDED = "R10"
+
+        # Tables (currently shares R10)
+        TABLE_STRUCTURE_RECONCILED = "R10"
+        TABLE_CELL_PR_REORDERED = "R10"
+        TABLE_CELL_DEFAULTS = "R12"
+        TABLE_ROW_GRID_AFTER = "R13"
+
+        # Referential integrity (currently shares R10)
+        DANGLING_STYLE_REFERENCE_REMOVED = "R10"
+        DANGLING_BASED_ON_REMOVED = "R10"
+        DANGLING_NUMBERING_REMOVED = "R4"
+        DANGLING_HYPERLINK_REMOVED = "R10"
+        DANGLING_HEADER_FOOTER_REMOVED = "R11"
+        DANGLING_DRAWING_REMOVED = "R12"
+
+        # Run cleanup (currently shares R10)
+        EMPTY_RUNS_STRIPPED = "R10"
+      end
+    end
+  end
+end

data/lib/uniword/docx/reconciler/helpers.rb

@@ -0,0 +1,337 @@
+# frozen_string_literal: true
+
+require "digest"
+require "set"
+require "yaml"
+
+module Uniword
+  module Docx
+    class Reconciler
+      # Shared helpers used across all reconciliation modules.
+      #
+      # Provides ID generation, document traversal, element_order
+      # manipulation, run utilities, and YAML config loading.
+      module Helpers
+        # -- ID generation --
+
+        def generate_rsid
+          "00#{hex_derive("rsid", 3)}"
+        end
+
+        def generate_hex_id(seed = 0)
+          hex_derive("paraId:#{seed}", 4)
+        end
+
+        def hex_derive(seed, byte_count)
+          Digest::SHA256.hexdigest(
+            "#{document_fingerprint}:#{seed}"
+          )[0...(byte_count * 2)].upcase
+        end
+
+        def document_fingerprint
+          @document_fingerprint ||= begin
+            body = package.document&.body
+            return "empty" unless body
+
+            texts = []
+            walk_body_paragraphs(body) do |para|
+              texts << (para.runs || []).map(&:text_string).join
+            end
+            Digest::SHA256.hexdigest(texts.join("|"))
+          end
+        end
+
+        # -- Traversal --
+
+        def walk_body_paragraphs(body)
+          if body.element_order && !body.element_order.empty?
+            p_idx = 0
+            tbl_idx = 0
+            body.element_order.each do |entry|
+              case entry.name
+              when "p"
+                yield body.paragraphs[p_idx] if body.paragraphs[p_idx]
+                p_idx += 1
+              when "tbl"
+                if body.tables[tbl_idx]
+                  walk_table_paragraphs(body.tables[tbl_idx]) { |p| yield p }
+                end
+                tbl_idx += 1
+              end
+            end
+          else
+            body.paragraphs.each { |p| yield p }
+            body.tables&.each { |tbl| walk_table_paragraphs(tbl) { |p| yield p } }
+          end
+        end
+
+        def walk_body_tables(body)
+          return unless body
+
+          if body.element_order && !body.element_order.empty?
+            tbl_idx = 0
+            body.element_order.each do |entry|
+              next unless entry.name == "tbl"
+
+              yield body.tables[tbl_idx] if body.tables[tbl_idx]
+              tbl_idx += 1
+            end
+          else
+            body.tables&.each { |tbl| yield tbl }
+          end
+        end
+
+        def walk_all_paragraphs(&block)
+          walk_body_paragraphs(package.document.body, &block)
+
+          (package.document&.header_footer_parts || []).each do |part|
+            (part[:content]&.paragraphs || []).each(&block)
+          end
+        end
+
+        def walk_table_paragraphs(table)
+          return unless table
+          table.rows&.each do |row|
+            row.cells&.each do |cell|
+              cell.paragraphs.each { |p| yield p }
+            end
+          end
+        end
+
+        # -- Element order --
+
+        def ensure_element_in_order(model, tag_name, after: nil, before: nil)
+          order = model.element_order
+          return unless order
+
+          return if order.any? { |e| e.name == tag_name }
+
+          entry = Lutaml::Xml::Element.new("Element", tag_name)
+
+          if after
+            idx = order.index { |e| e.name == after }
+            thaw_and_insert(model, order, idx ? idx + 1 : order.size, entry)
+          elsif before
+            idx = order.index { |e| e.name == before }
+            thaw_and_insert(model, order, idx || 0, entry)
+          else
+            thaw_and_append(model, order, entry)
+          end
+        end
+
+        def insert_element_order(obj, name, position)
+          order = obj.element_order
+          return unless order
+
+          return if order.any? { |e| e.name == name }
+
+          entry = Lutaml::Xml::Element.new("Element", name, node_type: :element)
+          thaw_and_insert(obj, order, position, entry)
+        end
+
+        # lutaml-model freezes element_order after XML parsing.
+        # Replace the frozen array with a mutable copy when modification is needed.
+        def thaw_and_insert(model, order, position, entry)
+          model.element_order = order.dup.insert(position, entry)
+        end
+
+        def thaw_and_append(model, order, entry)
+          model.element_order = order.dup << entry
+        end
+
+        # -- Run utilities (delegate to RunUtils) --
+
+        def empty_run?(run)
+          Builder::RunUtils.empty_run?(run)
+        end
+
+        def strip_empty_runs(paragraph)
+          before = paragraph.runs.size
+          paragraph.runs.reject! { |r| empty_run?(r) }
+          removed = before - paragraph.runs.size
+          return unless removed.positive?
+
+          record_fix(FixCodes::EMPTY_RUNS_STRIPPED,
+                     "Stripped #{removed} empty run(s) from " \
+                     "#{paragraph.class.name.split('::').last}")
+        end
+
+        def strip_empty_runs_from_notes(entries)
+          entries.each do |entry|
+            entry.paragraphs.each { |p| strip_empty_runs(p) }
+          end
+        end
+
+        # Shared paragraph backfill: assign rsid, paraId, textId defaults.
+        def backfill_paragraphs(paragraphs, rsid, id_seed)
+          paragraphs.each_with_index do |para, idx|
+            strip_empty_runs(para)
+            para.rsid_r ||= rsid
+            para.rsid_r_default ||= "00000000"
+            para.para_id ||= generate_hex_id("#{id_seed}:#{idx}")
+            para.text_id ||= "77777777"
+          end
+        end
+
+        # Merge adjacent runs with identical formatting and consolidate
+        # standalone tab/br runs into following text runs (F4 + F5).
+        # Respects element_order: only merges runs that are truly adjacent
+        # in document order (not separated by hyperlinks, bookmarks, etc.).
+        def consolidate_runs(paragraph)
+          runs = paragraph.runs
+          return if runs.nil? || runs.size < 2
+
+          run_indices = adjacent_run_indices(paragraph)
+          merged = [runs.first]
+          runs[1..].each_with_index do |run, idx|
+            prev = merged.last
+            if run_indices.include?(idx) &&
+               run_properties_match?(prev, run) && can_merge?(prev, run)
+              merge_run_into(prev, run)
+            else
+              merged << run
+            end
+          end
+
+          paragraph.runs = merged
+        end
+
+        # Returns the set of run indices (0-based into runs[1..]) where
+        # the run at that index is immediately preceded by another run
+        # in element_order (no non-run elements between them).
+        def adjacent_run_indices(paragraph)
+          eo = paragraph.element_order
+          return (0...paragraph.runs.size - 1).to_a unless eo && !eo.empty?
+
+          indices = Set.new
+          run_count = 0
+          prev_was_run = false
+          eo.each do |entry|
+            if entry.name == "r"
+              if prev_was_run
+                indices << (run_count - 1)
+              end
+              run_count += 1
+              prev_was_run = true
+            else
+              prev_was_run = false
+            end
+          end
+          indices
+        end
+
+        def consolidate_runs_in_body(body)
+          walk_body_paragraphs(body) { |p| consolidate_runs(p) }
+        end
+
+        # -- YAML config --
+
+        CONFIG_DIR = File.join(__dir__, "../../../../config")
+
+        def load_font_metadata
+          path = File.join(CONFIG_DIR, "font_metadata.yml")
+          YAML.load_file(path)["fonts"]
+        rescue StandardError => e
+          Uniword.logger&.warn { "Font metadata load failed: #{e.message}" }
+          nil
+        end
+
+        def load_latent_styles_config
+          path = File.join(CONFIG_DIR, "latent_styles.yml")
+          YAML.load_file(path)
+        rescue StandardError => e
+          Uniword.logger&.warn do
+            "Latent styles config load failed: #{e.message}"
+          end
+          nil
+        end
+
+        # -- mc:Ignorable helpers --
+
+        def set_mc_ignorable(model, prefixes: EXTENSION_PREFIXES)
+          model.mc_ignorable = Ooxml::Types::McIgnorable.new(prefixes)
+        end
+
+        # -- Relationship builders --
+
+        def build_rel(id, type, target, target_mode: nil)
+          attrs = { id: id, type: type, target: target }
+          attrs[:target_mode] = target_mode if target_mode
+          Ooxml::Relationships::Relationship.new(**attrs)
+        end
+
+        def run_properties_match?(a, b)
+          Builder::RunUtils.properties_match?(a, b)
+        end
+
+        def can_merge?(prev, current)
+          Builder::RunUtils.text_only_run?(prev) &&
+            Builder::RunUtils.text_only_run?(current)
+        end
+
+        def text_only_run?(run)
+          Builder::RunUtils.text_only_run?(run)
+        end
+
+        def merge_run_into(target, source)
+          Builder::RunUtils.merge_text(target, source)
+
+          target.tab ||= source.tab
+          target.break ||= source.break
+          target.position_tab ||= source.position_tab
+          target.no_break_hyphen ||= source.no_break_hyphen
+          target.del_text ||= source.del_text
+        end
+
+        # -- Note-type dispatch --
+        #
+        # Single source of truth for :footnote/:endnote discrimination.
+        # Adding a third note type means editing only these helpers.
+
+        def notes_collection_for(type)
+          case type
+          when :footnote then package.footnotes
+          when :endnote  then package.endnotes
+          end
+        end
+
+        def note_entries_for(notes, type)
+          return [] unless notes
+
+          case type
+          when :footnote then notes.footnote_entries
+          when :endnote  then notes.endnote_entries
+          end
+        end
+
+        def note_reference_from_run(run, type)
+          case type
+          when :footnote then run.footnote_reference
+          when :endnote  then run.endnote_reference
+          end
+        end
+
+        def build_notes_collection(type, entries:)
+          case type
+          when :footnote then Wordprocessingml::Footnotes.new(footnote_entries: entries)
+          when :endnote  then Wordprocessingml::Endnotes.new(endnote_entries: entries)
+          end
+        end
+
+        def assign_note_pr(settings, type)
+          case type
+          when :footnote then settings.footnote_pr = Wordprocessingml::FootnotePr.new
+          when :endnote  then settings.endnote_pr = Wordprocessingml::EndnotePr.new
+          end
+        end
+
+        def set_notes_collection(notes, type)
+          case type
+          when :footnote then package.footnotes = notes
+          when :endnote  then package.endnotes = notes
+          end
+        end
+      end
+    end
+  end
+end