canon 0.1.16 → 0.1.18

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0eb3c717365f052953d3deaf83a897112709c1a6084b472b99ddfdc2c9e43b67
-  data.tar.gz: fe4b2b513193b87692cd1fcb11569898e69c6818bea08ae3dccc753ad935f6e0
+  metadata.gz: 34ab9a64b52d8598690536908941e38950bb4071fb6222b50d1cb584236e5286
+  data.tar.gz: 2dffcc8e29fcd1f75d78595ef73350208e7b369f2183c2df93672b94df7a6376
 SHA512:
-  metadata.gz: 2c6d351b873ebb745c5abcdb2ff6cdbcf4ce53da1ad7f070c0b1eefeeeb776e315fa62c8d82c24b216e6e93cf5ac1790ebe3c6a171a142036ef0abc356d5a9e6
-  data.tar.gz: 6c0228d16e387e2a7919786cb57636e5c3183f0a1a1e119684fb0e01122a5ac23ffc08849f9df55bf413495024de30c4bf2e420172e13a9197d48b30636f845a
+  metadata.gz: 41d784c820a7bbafd9874bf369ef303376578e74f7171e8f00a7ed1ed0b8800576ca67f51272f1464f6f36cbaec35b6f1f1e9806916e7a8c59bc6c04e827ee79
+  data.tar.gz: d3e354615ed4b40447ae0be7de64bf6250c40650e25f897a5155f52a74f57a8601003efd37d539abf9f514d5cbe614298a2185db7d72b68b4ff95db6da1b5b99

data/.rubocop_todo.yml

@@ -1,6 +1,6 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2026-03-21 03:07:35 UTC using RuboCop version 1.85.1.
+# on 2026-03-24 08:58:24 UTC using RuboCop version 1.85.1.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
@@ -11,13 +11,63 @@ Gemspec/RequiredRubyVersion:
   Exclude:
     - 'canon.gemspec'
 
-# Offense count: 773
+# Offense count: 10
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: EnforcedStyle, IndentationWidth.
+# SupportedStyles: with_first_argument, with_fixed_indentation
+Layout/ArgumentAlignment:
+  Exclude:
+    - 'lib/canon/xml/data_model.rb'
+    - 'spec/canon/comparison/encoding_normalization_spec.rb'
+    - 'spec/canon/comparison/xml_whitespace_spec.rb'
+
+# Offense count: 1
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: EnforcedStyle, IndentationWidth.
+# SupportedStyles: with_first_element, with_fixed_indentation
+Layout/ArrayAlignment:
+  Exclude:
+    - 'lib/canon/comparison/format_detector.rb'
+
+# Offense count: 1
+# This cop supports safe autocorrection (--autocorrect).
+Layout/EmptyLineAfterGuardClause:
+  Exclude:
+    - 'lib/canon/xml/data_model.rb'
+
+# Offense count: 1
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: AllowMultipleStyles, EnforcedHashRocketStyle, EnforcedColonStyle, EnforcedLastArgumentHashStyle.
+# SupportedHashRocketStyles: key, separator, table
+# SupportedColonStyles: key, separator, table
+# SupportedLastArgumentHashStyles: always_inspect, always_ignore, ignore_implicit, ignore_explicit
+Layout/HashAlignment:
+  Exclude:
+    - 'lib/canon/comparison/format_detector.rb'
+
+# Offense count: 831
 # This cop supports safe autocorrection (--autocorrect).
 # Configuration parameters: Max, AllowHeredoc, AllowURI, AllowQualifiedName, URISchemes, AllowRBSInlineAnnotation, AllowCopDirectives, AllowedPatterns, SplitStrings.
 # URISchemes: http, https
 Layout/LineLength:
   Enabled: false
 
+# Offense count: 9
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: EnforcedStyle.
+# SupportedStyles: symmetrical, new_line, same_line
+Layout/MultilineMethodCallBraceLayout:
+  Exclude:
+    - 'spec/canon/comparison/encoding_normalization_spec.rb'
+    - 'spec/canon/comparison/xml_whitespace_spec.rb'
+
+# Offense count: 2
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: AllowInHeredoc.
+Layout/TrailingWhitespace:
+  Exclude:
+    - 'lib/canon/comparison/format_detector.rb'
+
 # Offense count: 49
 # Configuration parameters: IgnoreLiteralBranches, IgnoreConstantBranches, IgnoreDuplicateElseBranch.
 Lint/DuplicateBranch:
@@ -47,6 +97,13 @@ Lint/UnreachableCode:
   Exclude:
     - 'lib/canon/diff_formatter/debug_output.rb'
 
+# Offense count: 1
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: IgnoreEmptyBlocks, AllowUnusedKeywordArguments.
+Lint/UnusedBlockArgument:
+  Exclude:
+    - 'lib/canon/xml/data_model.rb'
+
 # Offense count: 6
 # This cop supports safe autocorrection (--autocorrect).
 # Configuration parameters: AllowUnusedKeywordArguments, IgnoreEmptyMethods, IgnoreNotImplementedMethods, NotImplementedExceptions.
@@ -58,7 +115,7 @@ Lint/UnusedMethodArgument:
     - 'lib/canon/diff_formatter/by_line/xml_formatter.rb'
     - 'lib/canon/diff_formatter/by_object/base_formatter.rb'
 
-# Offense count: 229
+# Offense count: 236
 # Configuration parameters: AllowedMethods, AllowedPatterns, CountRepeatedAttributes, Max.
 Metrics/AbcSize:
   Enabled: false
@@ -69,12 +126,12 @@ Metrics/AbcSize:
 Metrics/BlockLength:
   Max: 84
 
-# Offense count: 187
+# Offense count: 193
 # Configuration parameters: AllowedMethods, AllowedPatterns, Max.
 Metrics/CyclomaticComplexity:
   Enabled: false
 
-# Offense count: 394
+# Offense count: 403
 # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns.
 Metrics/MethodLength:
   Max: 95
@@ -84,7 +141,7 @@ Metrics/MethodLength:
 Metrics/ParameterLists:
   Max: 9
 
-# Offense count: 154
+# Offense count: 160
 # Configuration parameters: AllowedMethods, AllowedPatterns, Max.
 Metrics/PerceivedComplexity:
   Enabled: false
@@ -115,13 +172,19 @@ Performance/CollectionLiteralInLoop:
     - 'lib/canon/comparison/html_comparator.rb'
     - 'lib/canon/xml/xml_base_handler.rb'
 
+# Offense count: 1
+# This cop supports unsafe autocorrection (--autocorrect-all).
+Performance/UnfreezeString:
+  Exclude:
+    - 'spec/canon/comparison/encoding_normalization_spec.rb'
+
 # Offense count: 68
 # Configuration parameters: Prefixes, AllowedPatterns.
 # Prefixes: when, with, without
 RSpec/ContextWording:
   Enabled: false
 
-# Offense count: 27
+# Offense count: 29
 # Configuration parameters: IgnoredMetadata.
 RSpec/DescribeClass:
   Enabled: false
@@ -217,6 +280,11 @@ RSpec/NoExpectationExample:
     - 'spec/canon/isodoc_blockquotes_spec.rb'
     - 'spec/canon/match_scenarios_spec.rb'
 
+# Offense count: 2
+RSpec/RepeatedExample:
+  Exclude:
+    - 'spec/canon/comparison/encoding_normalization_spec.rb'
+
 # Offense count: 7
 # Configuration parameters: CustomTransform, IgnoreMethods, IgnoreMetadata, InflectorPath, EnforcedInflector.
 # SupportedInflectors: default, active_support
@@ -241,6 +309,17 @@ RSpec/VerifiedDoubles:
     - 'spec/canon/diff/xml_serialization_formatter_spec.rb'
     - 'spec/canon/tree_diff/operation_converter_spec.rb'
 
+# Offense count: 1
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: EnforcedStyle, ProceduralMethods, FunctionalMethods, AllowedMethods, AllowedPatterns, AllowBracesOnProceduralOneLiners, BracesRequiredMethods.
+# SupportedStyles: line_count_based, semantic, braces_for_chaining, always_braces
+# ProceduralMethods: benchmark, bm, bmbm, create, each_with_object, measure, new, realtime, tap, with_object
+# FunctionalMethods: let, let!, subject, watch
+# AllowedMethods: lambda, proc, it
+Style/BlockDelimiters:
+  Exclude:
+    - 'spec/canon/comparison/encoding_normalization_spec.rb'
+
 # Offense count: 1
 # This cop supports safe autocorrection (--autocorrect).
 # Configuration parameters: EnforcedStyle, AllowComments.
@@ -263,6 +342,13 @@ Style/IdenticalConditionalBranches:
     - 'lib/canon/diff_formatter/by_object/base_formatter.rb'
     - 'lib/canon/diff_formatter/legend.rb'
 
+# Offense count: 1
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: AllowMethodComparison, ComparisonsThreshold.
+Style/MultipleComparison:
+  Exclude:
+    - 'lib/canon/comparison/format_detector.rb'
+
 # Offense count: 1
 # Configuration parameters: AllowedMethods.
 # AllowedMethods: respond_to_missing?

data/README.adoc

@@ -214,6 +214,7 @@ Compare documents based on meaning, not formatting:
 * Comment handling with display control
 * Multiple match dimensions with behaviors
 * Predefined match profiles (strict, rendered, spec_friendly, content_only)
+* **Cross-encoding comparison**: Compare XML documents with different character encodings (UTF-8, Shift_JIS, ISO-8859-1, UTF-16) — Canon automatically normalizes to UTF-8 before comparison
 
 See link:docs/MATCH_OPTIONS[Match options] for details.
 

data/docs/features/match-options/index.adoc

@@ -151,22 +151,72 @@ sensitivity in XML instance documents:
 </text>
 ----
 
+The `xml:space` attribute affects both structural whitespace and text content:
+
+* **Structural whitespace** (whitespace-only text nodes between child elements)
+* **Text content whitespace** (whitespace within text nodes)
+
+.xml:space with structural_whitespace
+[example]
+====
+[source,ruby]
+----
+# With xml:space="preserve", structural whitespace is preserved
+xml1 = "<root xml:space='preserve'>\n  <text>Hello</text>\n</root>"
+xml2 = "<root xml:space='preserve'><text>Hello</text></root>"
+
+# These are NOT equivalent (structural whitespace differs)
+Canon::Comparison.equivalent?(xml1, xml2)
+# => false
+----
+====
+
+.xml:space with text_content
+[example]
+====
+[source,ruby]
+----
+# With xml:space="preserve", text content whitespace is preserved
+xml1 = '<root xml:space="preserve"><code>  indented  </code></root>'
+xml2 = '<root xml:space="preserve"><code>indented</code></root>'
+
+# These are NOT equivalent (text whitespace differs)
+Canon::Comparison.equivalent?(xml1, xml2,
+  match: { text_content: :strict }
+)
+# => false
+----
+====
+
 ==== Whitelist and blacklist options
 
-You can explicitly specify which elements are whitespace-sensitive:
+You can explicitly specify which elements are whitespace-sensitive using either short or long option names:
 
 [source,ruby]
 ----
-# Specify elements that preserve whitespace
+# Short names (preferred)
+Canon::Comparison.equivalent?(xml1, xml2,
+  match: {
+    structural_whitespace: :strict,
+    sensitive_elements: ["pre", "code", "sample"],
+    insensitive_elements: ["div", "span"]
+  }
+)
+
+# Long names (backward-compatible)
 Canon::Comparison.equivalent?(xml1, xml2,
   match: {
     structural_whitespace: :strict,
-    whitespace_sensitive_elements: [:pre, :code, :sample],
-    whitespace_insensitive_elements: [:p, :div]  # Override defaults/whitelist
+    whitespace_sensitive_elements: ["pre", "code", "sample"],
+    whitespace_insensitive_elements: ["div", "span"]
   }
 )
 ----
 
+**Element names are strings** (not symbols) for consistency with XML/HTML conventions.
+
+**Blacklist takes precedence over whitelist** — if an element appears in both lists, whitespace is stripped.
+
 ==== respect_xml_space option
 
 Control whether xml:space attributes in the document are honored:
@@ -211,34 +261,80 @@ When determining if an element is whitespace-sensitive, Canon uses this priority
 
 ==== Format-specific defaults
 
-**HTML**:: `[:pre, :textarea, :script, :style]` - These elements preserve whitespace by HTML specification
+**HTML**:: `["pre", "textarea", "script", "style"]` - These elements preserve whitespace by HTML specification
 **XML**:: `[]` - No default whitespace-sensitive elements, purely user-controlled
 
+==== Two types of whitespace sensitivity
+
+Canon handles two distinct whitespace concerns:
+
+**1. Structural whitespace stripping** — whitespace-only text nodes between sibling elements (indentation, newlines). These are never semantically meaningful and are stripped by default for XML to enable ElementMatcher to work correctly.
+
+**2. Text content comparison** — how non-whitespace text content is compared. Controlled by `structural_whitespace` and `text_content` dimension behaviors (`:strict`, `:normalize`, `:ignore`).
+
+The `sensitive_elements` / `insensitive_elements` options control both concerns:
+
+[source,ruby]
+----
+# For XML: structural whitespace is stripped by default
+# Use sensitive_elements to preserve whitespace in specific elements
+xml1 = "<root><item>Test</item></root>"
+xml2 = "<root>\n  <item>Test</item>\n</root>"
+
+# With sensitive_elements, whitespace inside <item> is preserved
+Canon::Comparison.equivalent?(xml1, xml2,
+  match: {
+    structural_whitespace: :strict,
+    sensitive_elements: ["item"]
+  }
+)
+# => true
+----
+
+**Precedence**: blacklist (`insensitive_elements`) > whitelist (`sensitive_elements`) > format defaults
+
+**No inheritance**: Only the immediate parent element's name is checked — not ancestor elements.
+
 ==== Examples
 
-.Using xml:space attribute
+.Using xml:space="preserve" for structural whitespace
+[source,ruby]
+----
+xml1 = "<root xml:space='preserve'>\n  <text>Hello</text>\n</root>"
+xml2 = "<root xml:space='preserve'><text>Hello</text></root>"
+
+# Structural whitespace differs - NOT equivalent
+Canon::Comparison.equivalent?(xml1, xml2)
+# => false
+----
+
+.Using xml:space="preserve" for text content
 [source,ruby]
 ----
-xml1 = '<root><code xml:space="preserve">  indented  </code></root>'
-xml2 = '<root><code xml:space="preserve">indented</code></root>'
+xml1 = '<root><code xml:space="preserve">  multiple   spaces  </code></root>'
+xml2 = '<root><code xml:space="preserve">multiple spaces</code></root>'
 
-# These are NOT equivalent (whitespace matters in xml:space="preserve")
+# Text content whitespace differs - NOT equivalent with text_content: :strict
 Canon::Comparison.equivalent?(xml1, xml2,
-  match: { structural_whitespace: :strict }
+  match: { text_content: :strict }
 )
 # => false
 ----
 
-.Using whitelist
+.Using sensitive_elements whitelist
 [source,ruby]
 ----
-# Make <p> elements whitespace-sensitive
+# Make <sample> elements whitespace-sensitive (strings, not symbols)
+xml1 = "<sample>\n  content\n</sample>"
+xml2 = "<sample>content</sample>"
+
 Canon::Comparison.equivalent?(xml1, xml2,
   match: {
     structural_whitespace: :strict,
-    whitespace_sensitive_elements: [:p, :pre]
+    sensitive_elements: ["sample"]
   }
 )
+# => false (structural whitespace differs in <sample>)
 ----
 
 .Overriding HTML defaults
@@ -249,7 +345,7 @@ Canon::Comparison.equivalent?(html1, html2,
   format: :html,
   match: {
     structural_whitespace: :strict,
-    whitespace_insensitive_elements: [:script]
+    insensitive_elements: ["script"]
   }
 )
 ----
@@ -636,12 +732,12 @@ expect(actual).to be_xml_equivalent_to(expected,
     element_hierarchy: :ignore
   )
 
-# Element-level whitespace sensitivity
+# Element-level whitespace sensitivity (strings, not symbols)
 expect(actual).to be_xml_equivalent_to(expected,
   match: { structural_whitespace: :strict }
 )
   .with_options(
-    whitespace_sensitive_elements: [:pre, :code, :sample],
+    sensitive_elements: ["pre", "code", "sample"],
     respect_xml_space: true
   )
 
@@ -650,7 +746,7 @@ expect(html).to be_html_equivalent_to(expected,
   match: { structural_whitespace: :strict }
 )
   .with_options(
-    whitespace_insensitive_elements: [:script, :style]
+    insensitive_elements: ["script", "style"]
   )
 ====
 

data/docs/lychee.toml

@@ -9,9 +9,6 @@ max_cache_age = "1d"
 # Check both source files and built site
 include_verbatim = true
 
-# Recursively check all files
-recursive = true
-
 # File types to check (regex patterns)
 include = [
   "_site/**/*.html",
@@ -50,9 +47,6 @@ user_agent = "lychee/canon-docs-link-checker"
 # Check HTTP, HTTPS, and file:// schemes
 scheme = ["https", "http", "file"]
 
-# Include file:// URLs for local link checking
-include_file = true
-
 # Handle different link types
 include_mail = false  # Don't check mailto: links
 
@@ -66,7 +60,4 @@ verbose = "warn"
 require_https = false  # Don't enforce
 
 # Index files for directory URLs
-index_files = ["index.html"]
-
-# Ignore patterns file
-ignore_file = ".lycheeignore"
+index_files = ["index.html"]

data/docs/understanding/formats/xml.adoc

@@ -340,6 +340,44 @@ Special attributes like `xml:lang`, `xml:space`, `xml:id`, and `xml:base` are pr
 When `xml:space="preserve"` is set, whitespace is preserved in descendants.
 ----
 
+=== Cross-encoding comparison
+
+Canon automatically normalizes XML character encodings before comparison, enabling
+cross-encoding comparisons to work correctly.
+
+**Supported encodings**: UTF-8, UTF-16 (all variants), Shift_JIS, EUC-JP, ISO-8859-1, and more.
+
+**How it works**:
+
+1. Extract the declared encoding from the XML declaration (e.g., `encoding="Shift_JIS"`)
+2. If declared encoding differs from UTF-8, transcode to UTF-8
+3. Handle cases where the declared encoding doesn't match actual bytes
+4. Use safe transcoding with replacement characters for invalid sequences
+
+.Cross-encoding comparison example
+[example]
+====
+[source,ruby]
+----
+# UTF-8 vs Shift_JIS - automatically normalized
+xml1 = "<root>日本語</root>"  # UTF-8
+xml2 = "<root>日本語</root>".encode("Shift_JIS")  # Shift_JIS
+
+Canon::Comparison.equivalent?(xml1, xml2)
+# => true (automatically transcoded to UTF-8 before comparison)
+
+# ASCII content works across all encodings
+xml3 = "<root>hello</root>"
+xml4 = "<root>hello</root>".encode("ISO-8859-1")
+
+Canon::Comparison.equivalent?(xml3, xml4)
+# => true
+----
+====
+
+This means you can compare XML files from different sources or systems without
+worrying about their native encoding.
+
 == Usage examples
 
 === Basic XML comparison

data/lib/canon/cache.rb

@@ -89,7 +89,8 @@ module Canon
       # @return [String] Cache key
       def key_for_format_detection(content)
         # Use first 100 chars for quick key, plus length
-        preview = content[0..100]
+        # Force to binary to avoid encoding compatibility issues
+        preview = content[0..100].b
         digest = Digest::SHA256.hexdigest(preview + content.length.to_s)
         "fmt:#{digest[0..16]}"
       end

data/lib/canon/comparison/format_detector.rb

@@ -62,7 +62,21 @@ module Canon
         # @param str [String] String to detect format of
         # @return [Symbol] Format type
         def detect_string_uncached(str)
-          trimmed = str.strip
+          # Convert to UTF-8 for consistent handling if possible
+          # This handles cases like UTF-16 encoded XML that would otherwise fail string operations
+          str_utf8 = if ["UTF-16", "UTF-16BE",
+                         "UTF-16LE"].include?(str.encoding.name)
+                       begin
+                         str.encode("UTF-8", str.encoding, invalid: :replace,
+                                                           undef: :replace, replace: "?")
+                       rescue EncodingError
+                         str.dup.force_encoding("BINARY").encode("UTF-8")
+                       end
+                     else
+                       str
+                     end
+
+          trimmed = str_utf8.strip
 
           # YAML indicators
           return :yaml if trimmed.start_with?("---")

data/lib/canon/comparison/markup_comparator.rb

@@ -174,12 +174,17 @@ module Canon
             end
           end
 
-          # Filter out whitespace-only text nodes based on structural_whitespace setting
-          if match_opts && %i[ignore
-                              normalize].include?(match_opts[:structural_whitespace]) && text_node?(node)
-            text = node_text(node)
-            return true if MatchOptions.normalize_text(text).empty?
-          end
+          # Strip whitespace-only text nodes based on parent element configuration.
+          # Use sensitive_elements / insensitive_elements to control.
+          # Blacklist (insensitive) > whitelist (sensitive) > format defaults.
+          return false unless text_node?(node) && node.parent
+          return false unless MatchOptions.normalize_text(node_text(node)).empty?
+
+          return true unless WhitespaceSensitivity.whitespace_preserved?(
+            node.parent, match_opts
+          )
+
+          false
 
           false
         end

data/lib/canon/comparison/match_options/base_resolver.rb

@@ -121,6 +121,8 @@ module Canon
               hash_matching
               similarity_matching
               propagation
+              sensitive_elements
+              insensitive_elements
               whitespace_sensitive_elements
               whitespace_insensitive_elements
               respect_xml_space

data/lib/canon/comparison/match_options/xml_resolver.rb

@@ -8,6 +8,12 @@ module Canon
       # XML/HTML-specific match options resolver
       class XmlResolver < BaseResolver
         # Format-specific defaults for XML/HTML
+        #
+        # Sensitive elements (preserve structural whitespace):
+        # - XML: none by default — all structural whitespace stripped
+        # - HTML: pre, code, textarea, script, style by default
+        # Use sensitive_elements option to add elements that preserve whitespace.
+        #
         FORMAT_DEFAULTS = {
           html: {
             preprocessing: :rendered,
@@ -33,7 +39,9 @@ module Canon
 
         # Predefined match profiles for XML/HTML
         MATCH_PROFILES = {
-          # Strict: Match exactly as written in source (XML default)
+          # Strict: Match exactly as written in source (XML default).
+          # Structural whitespace is stripped by default for XML.
+          # Use sensitive_elements to preserve structural whitespace in specific elements.
           strict: {
             preprocessing: :none,
             text_content: :strict,

data/lib/canon/comparison/whitespace_sensitivity.rb

@@ -66,6 +66,88 @@ module Canon
           element_sensitive?(node, opts)
         end
 
+        # Check if structural whitespace is preserved (not stripped) for an element.
+        #
+        # Uses sensitive_elements (whitelist) and insensitive_elements (blacklist)
+        # from match_opts. Blacklist takes precedence over whitelist.
+        # Format defaults apply when neither is configured.
+        #
+        # No inheritance from ancestors — checks only the immediate parent element name.
+        #
+        # @param element [Object] Element node to check
+        # @param match_opts [Hash] Resolved match options
+        # @return [Boolean] true if whitespace is preserved (not stripped)
+        def whitespace_preserved?(element, match_opts)
+          return false unless element
+          return false unless element.respond_to?(:name)
+
+          elem_name = element.name.to_s
+
+          # Blacklist: always strip (highest priority)
+          insensitive_raw = match_opts[:insensitive_elements]
+          insensitive_raw ||= match_opts[:whitespace_insensitive_elements]
+          insensitive = (insensitive_raw || []).map(&:to_s)
+          return false if insensitive.include?(elem_name)
+
+          # Check if we should ignore xml:space (user override)
+          if respect_xml_space?(match_opts)
+            # Check xml:space="preserve" (document declaration)
+            return true if xml_space_preserve?(element)
+
+            # Check xml:space="default" (use configured behavior)
+            return false if xml_space_default?(element)
+          end
+
+          # Whitelist: preserve whitespace
+          sensitive = resolved_sensitive_elements(match_opts)
+          return true if sensitive.include?(elem_name)
+
+          # Default: preserve for HTML, strip for XML
+          format = match_opts[:format] || :xml
+          case format
+          when :html, :html4, :html5
+            true
+          else
+            false
+          end
+        end
+
+        # Get resolved list of whitespace-sensitive element names (strings).
+        #
+        # Combines format defaults + user whitelist, minus user blacklist.
+        # Supports both short names (sensitive_elements) and long names
+        # (whitespace_sensitive_elements) for backward compatibility.
+        #
+        # @param match_opts [Hash] Resolved match options
+        # @return [Array<String>] Sensitive element names
+        def resolved_sensitive_elements(match_opts)
+          sensitive = []
+
+          # 1. Format defaults
+          format = match_opts[:format] || :xml
+          case format
+          when :html, :html4, :html5
+            sensitive += %w[pre code textarea script style]
+          end
+
+          # 2. User whitelist (additive to format defaults)
+          whitelist = match_opts[:sensitive_elements]
+          whitelist ||= match_opts[:whitespace_sensitive_elements]
+          if whitelist
+            sensitive += whitelist.map(&:to_s)
+          end
+
+          # 3. User blacklist removes from combined set
+          blacklist_raw = match_opts[:insensitive_elements]
+          blacklist_raw ||= match_opts[:whitespace_insensitive_elements]
+          if blacklist_raw
+            blacklist = blacklist_raw.to_set(&:to_s)
+            sensitive.reject! { |e| blacklist.include?(e) }
+          end
+
+          sensitive.uniq
+        end
+
         # Get format-specific default sensitive elements
         #
         # This is the SINGLE SOURCE OF TRUTH for default whitespace-sensitive

data/lib/canon/comparison/xml_comparator/node_parser.rb

@@ -25,6 +25,9 @@ module Canon
                                      preserve_whitespace: preserve_whitespace)
           end
 
+          # Normalize encoding before preprocessing (UTF-16 strings can't use strip, etc.)
+          node = Canon::Xml::DataModel.normalize_encoding(node)
+
           # Apply preprocessing to XML string before parsing
           xml_string = apply_preprocessing(node, preprocessing).strip
 

data/lib/canon/comparison/xml_node_comparison.rb

@@ -190,14 +190,17 @@ diff_children, differences)
           end
         end
 
-        # Filter out whitespace-only text nodes based on structural_whitespace setting
-        # - :ignore or :normalize: Filter all whitespace-only text nodes
-        # - :strict: Preserve all whitespace-only text nodes (don't filter any)
-        if text_node?(node) && %i[ignore
-                                  normalize].include?(match_opts[:structural_whitespace])
-          text = node_text(node)
-          return true if MatchOptions.normalize_text(text).empty?
-        end
+        # Strip whitespace-only text nodes based on parent element configuration.
+        # Use sensitive_elements / insensitive_elements to control.
+        # Blacklist (insensitive) > whitelist (sensitive) > format defaults.
+        return false unless text_node?(node) && node.parent
+        return false unless MatchOptions.normalize_text(node_text(node)).empty?
+
+        return true unless WhitespaceSensitivity.whitespace_preserved?(
+          node.parent, match_opts
+        )
+
+        false
 
         false
       end

data/lib/canon/diff_formatter/diff_detail_formatter/dimension_formatter.rb

@@ -326,9 +326,9 @@ module Canon
           # Handle cases where one node is missing (e.g. text added or removed)
           if node1.nil? || node2.nil?
             if node1.nil?
-              text2 = node2.to_s
+              text2 = NodeUtils.get_node_text(node2)
             else
-              text1 = node1.to_s
+              text1 = NodeUtils.get_node_text(node1)
             end
           end
 

data/lib/canon/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Canon
-  VERSION = "0.1.16"
+  VERSION = "0.1.18"
 end

data/lib/canon/xml/data_model.rb

@@ -21,8 +21,11 @@ module Canon
       # @param preserve_whitespace [Boolean] Whether to preserve whitespace-only text nodes
       # @return [Nodes::RootNode] Root of the data model tree
       def self.from_xml(xml_string, preserve_whitespace: false)
+        # Normalize encoding before parsing
+        normalized_xml = normalize_encoding(xml_string)
+
         # Parse with Nokogiri
-        doc = Nokogiri::XML(xml_string, &:nonet)
+        doc = Nokogiri::XML(normalized_xml, &:nonet)
 
         # Check for relative namespace URIs (prohibited by C14N 1.1)
         check_for_relative_namespace_uris(doc)
@@ -31,6 +34,132 @@ module Canon
         build_from_nokogiri(doc, preserve_whitespace: preserve_whitespace)
       end
 
+      # Normalize XML string encoding to UTF-8
+      #
+      # Handles cases where:
+      # 1. The XML declaration specifies an encoding that doesn't match the actual encoding
+      # 2. The string's internal encoding is non-UTF-8 (without a declaration)
+      #
+      # For case 1, we check if the declared encoding matches the actual bytes.
+      # If bytes are valid UTF-8 despite the declaration, we update the declaration to UTF-8.
+      #
+      # @param xml_string [String] XML string to normalize
+      # @return [String] Normalized XML string with UTF-8 encoding
+      def self.normalize_encoding(xml_string)
+        return xml_string unless xml_string.is_a?(String)
+
+        # Extract declared encoding from XML declaration
+        declared_encoding = extract_xml_encoding(xml_string)
+
+        if declared_encoding
+          # Case 1: XML has a declaration
+          if declared_encoding.upcase != "UTF-8"
+            # Check if bytes are actually valid UTF-8 despite the declaration
+            utf8_reinterpreted = try_utf8_reinterpretation(xml_string)
+            if utf8_reinterpreted
+              # Bytes are valid UTF-8 - update declaration to UTF-8
+              return update_xml_declaration(xml_string, "UTF-8")
+            end
+
+            # Bytes aren't valid UTF-8 - must really be in declared encoding
+            return transcode_to_utf8(xml_string, declared_encoding)
+          end
+        elsif xml_string.encoding.name != "UTF-8"
+          # Case 2: No declaration but string encoding is non-UTF-8
+          # First, try to re-interpret bytes as UTF-8 (handles mislabeled strings)
+          reinterpreted = try_utf8_reinterpretation(xml_string)
+          return reinterpreted if reinterpreted
+
+          # If re-interpretation fails, try transcoding with the labeled encoding
+          return transcode_to_utf8(xml_string, xml_string.encoding.name)
+        end
+
+        xml_string
+      end
+
+      # Update the encoding declaration in an XML string
+      #
+      # @param xml_string [String] XML string
+      # @param new_encoding [String] New encoding to declare
+      # @return [String] XML string with updated declaration
+      def self.update_xml_declaration(xml_string, new_encoding)
+        xml_string.sub(/\bencoding\s*=\s*["'][^"']+["']/i) do |_match|
+          %(encoding="#{new_encoding}")
+        end
+      end
+
+      # Transcode string to UTF-8
+      #
+      # @param xml_string [String] String to transcode
+      # @param source_encoding [String] Source encoding to interpret bytes as
+      # @return [String] UTF-8 transcoded string
+      def self.transcode_to_utf8(xml_string, source_encoding)
+        # First, check if the bytes are actually valid UTF-8 despite the declared encoding
+        # If so, just re-interpret as UTF-8 (common case: declaration is wrong)
+        if source_encoding != "UTF-8"
+          # Force the bytes to be interpreted as the declared encoding, then check validity
+          forced = xml_string.dup.force_encoding(source_encoding)
+          if forced.valid_encoding?
+            # Now check if the same bytes are valid UTF-8
+            utf8_check = xml_string.dup.force_encoding("UTF-8")
+            if utf8_check.valid_encoding?
+              # Bytes are valid UTF-8 - the declaration is likely wrong
+              # Return the string as UTF-8 (already is)
+              return xml_string.dup.force_encoding("UTF-8")
+            end
+
+            # Bytes aren't valid UTF-8, so they must really be in source_encoding
+            # Proceed with transcoding
+            return forced.encode("UTF-8", source_encoding,
+                                 invalid: :replace,
+                                 undef: :replace,
+                                 replace: "?")
+          end
+        end
+
+        # Already UTF-8 or transcoding failed, return as-is
+        xml_string.dup.force_encoding("UTF-8")
+      rescue EncodingError
+        xml_string
+      end
+
+      # Attempt to re-interpret string as UTF-8 if bytes are valid UTF-8
+      #
+      # This handles the case where a string was incorrectly labeled with a different
+      # encoding (e.g., `.encode("Shift_JIS")` on a UTF-8 string) but the actual
+      # bytes are valid UTF-8.
+      #
+      # @param xml_string [String] XML string to check
+      # @return [String, nil] UTF-8 re-interpreted string, or nil if not possible
+      def self.try_utf8_reinterpretation(xml_string)
+        return xml_string if xml_string.encoding.name == "UTF-8"
+
+        # Try forcing to UTF-8 and see if it's valid
+        forced = xml_string.dup.force_encoding("UTF-8")
+        return forced if forced.valid_encoding?
+
+        nil
+      end
+
+      # Extract encoding from XML declaration
+      #
+      # @param xml_string [String] XML string
+      # @return [String, nil] Declared encoding or nil if not found
+      def self.extract_xml_encoding(xml_string)
+        # Match XML declaration with encoding attribute
+        # Handles: <?xml version="1.0" encoding="UTF-8"?>
+        # and: <?xml version='1.0' encoding='UTF-8'?>
+        #
+        # Use binary encoding to avoid encoding compatibility issues
+        # when the string has non-ASCII compatible encoding (e.g., UTF-16)
+        binary_string = xml_string.dup.force_encoding("BINARY")
+        if binary_string =~ /\A\s*<\?xml[^>]*\bencoding\s*=\s*["']([^"']+)["'][^>]*\?>/i
+          return Regexp.last_match(1)
+        end
+
+        nil
+      end
+
       # Alias for compatibility with base class interface
       def self.parse(xml_string)
         from_xml(xml_string)

data/lib/tasks/benchmark_runner.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "benchmark/ips"
+require "table_tennis"
 
 # Ensure lib/ is on the load path regardless of tmp location
 lib_path = File.expand_path(File.join(__dir__, "..", "..", "lib"))
@@ -101,7 +102,8 @@ class BenchmarkRunner
     end
 
     # Category section with description
-    def self.category(title, icon:, description:, failure_means:, compare_against: nil)
+    def self.category(title, icon:, description:, failure_means:,
+compare_against: nil)
       puts
       puts "#{CYAN}#{VL}#{CLEAR}  #{BOLD}#{MAGENTA}#{icon} #{title}#{CLEAR}"
       puts
@@ -124,26 +126,21 @@ class BenchmarkRunner
       puts
     end
 
-    # Results table for a category
-    def self.table_header
-      puts "  #{BOLD}#{'%-35s'} #{'%10s'} #{'%8s'} #{'%s'}#{CLEAR}"
-      puts "  Test                                       IPS       ±% Speedup"
-      sep(char: "─", width: 76)
-    end
+    # Results table for a category using TableTennis
+    def self.table(results)
+      rows = results.map do |r|
+        {
+          test: r[:name],
+          ips: r[:ips],
+          deviation: "#{r[:deviation].round(1)}%",
+          status: r[:is_best] ? "BEST" : "",
+        }
+      end
 
-    def self.table_row(label, ips, deviation, speedup: nil, is_best: false)
-      speedup_str = speedup ? "  ⚡#{speedup.round(2)}x" : ""
-      label_str = is_best ? "#{GREEN}#{label}#{CLEAR}" : label
-      bar = render_bar(ips)
+      return if rows.empty?
 
-      puts "  #{label_str}"
-      puts "  #{DIM}#{bar}#{CLEAR}  #{format('%10.1f', ips)}  #{format('%6.1f%%', deviation)}#{speedup_str}"
-      puts
-    end
-
-    def self.table_footer
-      sep(char: "─", width: 76)
-      puts
+      table = TableTennis.new(rows, theme: :dark)
+      table.render
     end
 
     def self.speedup_badge(factor, label)
@@ -151,24 +148,7 @@ class BenchmarkRunner
       puts "  #{GREEN}   #{factor.round(2)}x faster#{CLEAR}"
     end
 
-    def self.reset_max_ips
-      @max_ips = nil
-    end
-
-    def self.set_max_ips(val)
-      @max_ips = val
-    end
-
-    def self.render_bar(ips, max_width: 20)
-      @max_ips ||= ips
-      ratio = ips / @max_ips.to_f
-      width = [(ratio * max_width).round, 1].max
-      filled = [width, max_width].min
-      empty = max_width - filled
-      ("█" * filled) + ("░" * empty)
-    end
-
-    # Summary card
+    # Summary card using TableTennis
     def self.summary_card(results)
       puts
       sep(width: 78)
@@ -176,16 +156,23 @@ class BenchmarkRunner
       puts "  #{BOLD}#{MAGENTA}SUMMARY#{CLEAR}"
       puts
 
-      total = results.length
-
-      results.each do |r|
-        # For standalone runs, all results are shown as "current" without comparison
-        ips_str = r[:ips] ? format("%10.1f IPS", r[:ips]) : ""
-        puts "  #{DIM}◆#{CLEAR} #{format('%-35s', r[:label])} #{ips_str}"
+      rows = results.map do |r|
+        {
+          benchmark: r[:label],
+          ips: r[:ips]&.round(1),
+        }
       end
 
+      return if rows.empty?
+
+      table = TableTennis.new(rows,
+                              title: "Performance Results",
+                              theme: :dark,
+                              headers: { benchmark: "Benchmark", ips: "IPS" })
+      table.render
+
       puts
-      puts "  #{DIM}#{total} benchmarks completed#{CLEAR}"
+      puts "  #{DIM}#{results.length} benchmarks completed#{CLEAR}"
       puts
     end
   end
@@ -239,24 +226,35 @@ class BenchmarkRunner
   # Test definitions
   BENCHMARKS = {
     xml_parsing: [
-      { name: "DOM (simple)", method: :xml_parse_dom_simple, desc: "Standard DOM parsing" },
-      { name: "SAX (simple)", method: :xml_parse_sax_simple, desc: "Streaming SAX parsing" },
-      { name: "DOM (large)", method: :xml_parse_dom_large, desc: "Large document DOM" },
-      { name: "SAX (large)", method: :xml_parse_sax_large, desc: "Large document SAX" },
+      { name: "DOM (simple)", method: :xml_parse_dom_simple,
+        desc: "Standard DOM parsing" },
+      { name: "SAX (simple)", method: :xml_parse_sax_simple,
+        desc: "Streaming SAX parsing" },
+      { name: "DOM (large)", method: :xml_parse_dom_large,
+        desc: "Large document DOM" },
+      { name: "SAX (large)", method: :xml_parse_sax_large,
+        desc: "Large document SAX" },
     ],
     html_parsing: [
       { name: "Simple HTML", method: :html_parse_simple, desc: "Basic HTML" },
-      { name: "Complex HTML", method: :html_parse_complex, desc: "HTML with scripts/tables" },
+      { name: "Complex HTML", method: :html_parse_complex,
+        desc: "HTML with scripts/tables" },
     ],
     xml_comparison: [
-      { name: "Identical XML", method: :xml_compare_identical, desc: "Same documents" },
-      { name: "Similar XML", method: :xml_compare_similar, desc: "Slightly different" },
-      { name: "Different XML", method: :xml_compare_different, desc: "Different namespaces" },
+      { name: "Identical XML", method: :xml_compare_identical,
+        desc: "Same documents" },
+      { name: "Similar XML", method: :xml_compare_similar,
+        desc: "Slightly different" },
+      { name: "Different XML", method: :xml_compare_different,
+        desc: "Different namespaces" },
     ],
     html_comparison: [
-      { name: "Identical HTML", method: :html_compare_identical, desc: "Same HTML" },
-      { name: "Similar HTML", method: :html_compare_similar, desc: "Slightly different" },
-      { name: "Different HTML", method: :html_compare_different, desc: "Different structure" },
+      { name: "Identical HTML", method: :html_compare_identical,
+        desc: "Same HTML" },
+      { name: "Similar HTML", method: :html_compare_similar,
+        desc: "Slightly different" },
+      { name: "Different HTML", method: :html_compare_different,
+        desc: "Different structure" },
     ],
     formatting: [
       { name: "XML C14N", method: :xml_c14n_format, desc: "Canonical XML" },
@@ -287,7 +285,8 @@ class BenchmarkRunner
           end.join
           "<#{prefix}root#{ns_attr}#{attrs}>#{children}</#{prefix}root>"
         else
-          child = build_xml_element(items / 2, depth - 1, prefix, with_attrs, "")
+          child = build_xml_element(items / 2, depth - 1, prefix, with_attrs,
+                                    "")
           "<#{prefix}root#{ns_attr}#{attrs}>#{child}</#{prefix}root>"
         end
       end
@@ -401,8 +400,6 @@ class BenchmarkRunner
   end
 
   def run_benchmarks
-    Term.reset_max_ips
-
     # Header
     Term.header("Canon Performance Benchmarks", color: Term::CYAN)
 
@@ -434,8 +431,6 @@ class BenchmarkRunner
       compare_against: config[:compare_against],
     )
 
-    Term.table_header
-
     # Run each test in category
     category_results = []
     max_ips = 0
@@ -457,23 +452,32 @@ class BenchmarkRunner
       $stdout = original_stdout
     end
 
-    # Reset for relative bars within category
-    Term.set_max_ips(max_ips)
-
-    # Print results with relative bars
-    category_results.each do |r|
+    # Build results for TableTennis table
+    table_rows = category_results.map do |r|
       is_best = r[:result][:upper] >= max_ips
-      Term.table_row(r[:name], (r[:result][:lower] + r[:result][:upper]) / 2.0,
-                     calculate_deviation(r[:result]), is_best: is_best)
-      @all_results << { label: "#{config[:name]}: #{r[:name]}", ips: (r[:result][:lower] + r[:result][:upper]) / 2.0 }
+      label = "#{config[:name]}: #{r[:name]}"
+      @all_results << { label: label,
+                        ips: (r[:result][:lower] + r[:result][:upper]) / 2.0 }
+      @results[label] = r[:result] # Populate @results for comparison
+      {
+        name: r[:name],
+        ips: (r[:result][:lower] + r[:result][:upper]) / 2.0,
+        deviation: calculate_deviation(r[:result]),
+        is_best: is_best,
+      }
     end
 
-    Term.table_footer
+    # Render TableTennis table
+    Term.table(table_rows)
 
     # SAX vs DOM comparison for XML parsing
     if category == :xml_parsing && SAX_AVAILABLE
-      sax = category_results.find { |r| r[:name].include?("SAX") && r[:name].include?("large") }
-      dom = category_results.find { |r| r[:name].include?("DOM") && r[:name].include?("large") }
+      sax = category_results.find do |r|
+        r[:name].include?("SAX") && r[:name].include?("large")
+      end
+      dom = category_results.find do |r|
+        r[:name].include?("DOM") && r[:name].include?("large")
+      end
 
       if sax && dom
         sax_ips = (sax[:result][:lower] + sax[:result][:upper]) / 2.0
@@ -481,9 +485,11 @@ class BenchmarkRunner
         speedup = sax_ips / dom_ips
 
         if speedup > 1.0
-          Term.speedup_badge(speedup, "SAX is faster than DOM for large documents")
+          Term.speedup_badge(speedup,
+                             "SAX is faster than DOM for large documents")
         else
-          Term.hint("DOM is #{format('%.2f', 1 / speedup)}x faster than SAX for large documents")
+          Term.hint("DOM is #{format('%.2f',
+                                     1 / speedup)}x faster than SAX for large documents")
         end
       end
     end
@@ -509,7 +515,8 @@ class BenchmarkRunner
       html = DataGenerator.generate_html(items: @items)
       measure { Canon.parse_html(html) }
     when :html_parse_complex
-      html = DataGenerator.generate_html(items: @items, with_scripts: true, with_tables: true)
+      html = DataGenerator.generate_html(items: @items, with_scripts: true,
+                                         with_tables: true)
       measure { Canon.parse_html(html) }
     when :xml_compare_identical
       xml = DataGenerator.generate_xml(items: @items)
@@ -566,7 +573,8 @@ class BenchmarkRunner
     error_margin = std_dev / mean
     error_pct = error_margin.round(4)
 
-    { lower: mean.round(4) * (1 - error_pct), upper: mean.round(4) * (1 + error_pct) }
+    { lower: mean.round(4) * (1 - error_pct),
+      upper: mean.round(4) * (1 + error_pct) }
   end
 
   def measure_time

data/lib/tasks/performance_helpers.rb

@@ -4,6 +4,7 @@ require "json"
 require "open3"
 require "tmpdir"
 require "fileutils"
+require "table_tennis"
 
 module PerformanceHelpers
   # ANSI color codes for terminal output
@@ -97,9 +98,49 @@ module PerformanceHelpers
       all_base.merge!(base_results)
       all_current.merge!(curr_results)
 
+      # Collect comparison results for TableTennis table
+      comparison_rows = []
+
       curr_results.each do |label, result|
-        print_realtime_comparison(label, result, base_results[label], threshold)
+        base_result = base_results[label]
+        cmp = compare_metrics(label, result, base_result, threshold)
+        comparison_rows << cmp
       end
+
+      print_comparison_table(comparison_rows, threshold)
+    end
+
+    def print_comparison_table(comparison_rows, threshold)
+      rows = comparison_rows.map do |cmp|
+        {
+          benchmark: cmp[:label],
+          base_ips: cmp[:base_ips]&.round(1),
+          curr_ips: cmp[:curr_ips]&.round(1),
+          change: cmp[:change] ? "#{(cmp[:change] * 100).round(1)}%" : "N/A",
+          status: if cmp[:base_ips].nil?
+                    "NEW"
+                  elsif cmp[:change] < -threshold
+                    "REGRESSED"
+                  else
+                    "OK"
+                  end,
+        }
+      end
+
+      return if rows.empty?
+
+      table = TableTennis.new(rows,
+                              title: "Performance Comparison",
+                              theme: :dark,
+                              headers: {
+                                benchmark: "Benchmark",
+                                base_ips: "Base IPS",
+                                curr_ips: "Curr IPS",
+                                change: "Change",
+                                status: "Status",
+                              })
+      table.render
+      puts
     end
 
     def compare_metrics(label, curr, base, threshold)
@@ -197,7 +238,9 @@ module PerformanceHelpers
       # Handle new benchmarks that don't exist in base
       if base_metrics.nil?
         curr_ips = (curr_metrics[:lower] + curr_metrics[:upper]) / 2.0
-        puts "#{format('%-30s', label)}: #{GREEN}NEW#{CLEAR} (current: #{format('%.2f', curr_ips)} IPS) [N/A]\n\n"
+        puts "#{format('%-30s',
+                       label)}: #{GREEN}NEW#{CLEAR} (current: #{format('%.2f',
+                                                                       curr_ips)} IPS) [N/A]\n\n"
         return
       end
 
@@ -212,7 +255,8 @@ module PerformanceHelpers
       base_str = format("%.2f", base_ips)
       curr_str = format("%.2f", curr_ips)
 
-      puts "#{format('%-30s', label)}: #{GRAY}#{base_str}#{CLEAR} → #{color}#{curr_str}#{CLEAR} IPS " \
+      puts "#{format('%-30s',
+                     label)}: #{GRAY}#{base_str}#{CLEAR} → #{color}#{curr_str}#{CLEAR} IPS " \
            "(change: #{color}#{delta_str}#{CLEAR}) [#{color}#{status}#{CLEAR}]\n\n"
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: canon
 version: !ruby/object:Gem::Version
-  version: 0.1.16
+  version: 0.1.18
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-03-21 00:00:00.000000000 Z
+date: 2026-03-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: diff-lcs