canon 0.1.17 → 0.1.18

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4bf32847db2d5c418daebb9ad5221646edecaf4b67b4f25c4e2a9e8a68167a8e
-  data.tar.gz: 6e595f08701e61f73ad62dc5aec3ec3b95da8f41fc75d579e70721f2d9af42e5
+  metadata.gz: 34ab9a64b52d8598690536908941e38950bb4071fb6222b50d1cb584236e5286
+  data.tar.gz: 2dffcc8e29fcd1f75d78595ef73350208e7b369f2183c2df93672b94df7a6376
 SHA512:
-  metadata.gz: 42a21e5e1badd2c1b96b1b86dce89551ee5b0794150fd2844b345fcabeb3d9bb484ca3beb423209e0bd455887d3597aa7d5973aaa0985ee77c450f20ff755866
-  data.tar.gz: 8799d74f6a3738317387336308a3f95ffabaa5779d96dbde0ee9bccc424d360131230752031b0a0ee907af5907134b1ca8dec75e8cd0024fb600e090d3b681b7
+  metadata.gz: 41d784c820a7bbafd9874bf369ef303376578e74f7171e8f00a7ed1ed0b8800576ca67f51272f1464f6f36cbaec35b6f1f1e9806916e7a8c59bc6c04e827ee79
+  data.tar.gz: d3e354615ed4b40447ae0be7de64bf6250c40650e25f897a5155f52a74f57a8601003efd37d539abf9f514d5cbe614298a2185db7d72b68b4ff95db6da1b5b99

data/.rubocop_todo.yml

@@ -1,6 +1,6 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2026-03-24 03:04:40 UTC using RuboCop version 1.85.1.
+# on 2026-03-24 08:58:24 UTC using RuboCop version 1.85.1.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
@@ -11,13 +11,63 @@ Gemspec/RequiredRubyVersion:
   Exclude:
     - 'canon.gemspec'
 
-# Offense count: 802
+# Offense count: 10
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: EnforcedStyle, IndentationWidth.
+# SupportedStyles: with_first_argument, with_fixed_indentation
+Layout/ArgumentAlignment:
+  Exclude:
+    - 'lib/canon/xml/data_model.rb'
+    - 'spec/canon/comparison/encoding_normalization_spec.rb'
+    - 'spec/canon/comparison/xml_whitespace_spec.rb'
+
+# Offense count: 1
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: EnforcedStyle, IndentationWidth.
+# SupportedStyles: with_first_element, with_fixed_indentation
+Layout/ArrayAlignment:
+  Exclude:
+    - 'lib/canon/comparison/format_detector.rb'
+
+# Offense count: 1
+# This cop supports safe autocorrection (--autocorrect).
+Layout/EmptyLineAfterGuardClause:
+  Exclude:
+    - 'lib/canon/xml/data_model.rb'
+
+# Offense count: 1
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: AllowMultipleStyles, EnforcedHashRocketStyle, EnforcedColonStyle, EnforcedLastArgumentHashStyle.
+# SupportedHashRocketStyles: key, separator, table
+# SupportedColonStyles: key, separator, table
+# SupportedLastArgumentHashStyles: always_inspect, always_ignore, ignore_implicit, ignore_explicit
+Layout/HashAlignment:
+  Exclude:
+    - 'lib/canon/comparison/format_detector.rb'
+
+# Offense count: 831
 # This cop supports safe autocorrection (--autocorrect).
 # Configuration parameters: Max, AllowHeredoc, AllowURI, AllowQualifiedName, URISchemes, AllowRBSInlineAnnotation, AllowCopDirectives, AllowedPatterns, SplitStrings.
 # URISchemes: http, https
 Layout/LineLength:
   Enabled: false
 
+# Offense count: 9
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: EnforcedStyle.
+# SupportedStyles: symmetrical, new_line, same_line
+Layout/MultilineMethodCallBraceLayout:
+  Exclude:
+    - 'spec/canon/comparison/encoding_normalization_spec.rb'
+    - 'spec/canon/comparison/xml_whitespace_spec.rb'
+
+# Offense count: 2
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: AllowInHeredoc.
+Layout/TrailingWhitespace:
+  Exclude:
+    - 'lib/canon/comparison/format_detector.rb'
+
 # Offense count: 49
 # Configuration parameters: IgnoreLiteralBranches, IgnoreConstantBranches, IgnoreDuplicateElseBranch.
 Lint/DuplicateBranch:
@@ -47,6 +97,13 @@ Lint/UnreachableCode:
   Exclude:
     - 'lib/canon/diff_formatter/debug_output.rb'
 
+# Offense count: 1
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: IgnoreEmptyBlocks, AllowUnusedKeywordArguments.
+Lint/UnusedBlockArgument:
+  Exclude:
+    - 'lib/canon/xml/data_model.rb'
+
 # Offense count: 6
 # This cop supports safe autocorrection (--autocorrect).
 # Configuration parameters: AllowUnusedKeywordArguments, IgnoreEmptyMethods, IgnoreNotImplementedMethods, NotImplementedExceptions.
@@ -58,7 +115,7 @@ Lint/UnusedMethodArgument:
     - 'lib/canon/diff_formatter/by_line/xml_formatter.rb'
     - 'lib/canon/diff_formatter/by_object/base_formatter.rb'
 
-# Offense count: 235
+# Offense count: 236
 # Configuration parameters: AllowedMethods, AllowedPatterns, CountRepeatedAttributes, Max.
 Metrics/AbcSize:
   Enabled: false
@@ -69,12 +126,12 @@ Metrics/AbcSize:
 Metrics/BlockLength:
   Max: 84
 
-# Offense count: 192
+# Offense count: 193
 # Configuration parameters: AllowedMethods, AllowedPatterns, Max.
 Metrics/CyclomaticComplexity:
   Enabled: false
 
-# Offense count: 401
+# Offense count: 403
 # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns.
 Metrics/MethodLength:
   Max: 95
@@ -84,7 +141,7 @@ Metrics/MethodLength:
 Metrics/ParameterLists:
   Max: 9
 
-# Offense count: 158
+# Offense count: 160
 # Configuration parameters: AllowedMethods, AllowedPatterns, Max.
 Metrics/PerceivedComplexity:
   Enabled: false
@@ -115,13 +172,19 @@ Performance/CollectionLiteralInLoop:
     - 'lib/canon/comparison/html_comparator.rb'
     - 'lib/canon/xml/xml_base_handler.rb'
 
+# Offense count: 1
+# This cop supports unsafe autocorrection (--autocorrect-all).
+Performance/UnfreezeString:
+  Exclude:
+    - 'spec/canon/comparison/encoding_normalization_spec.rb'
+
 # Offense count: 68
 # Configuration parameters: Prefixes, AllowedPatterns.
 # Prefixes: when, with, without
 RSpec/ContextWording:
   Enabled: false
 
-# Offense count: 27
+# Offense count: 29
 # Configuration parameters: IgnoredMetadata.
 RSpec/DescribeClass:
   Enabled: false
@@ -217,6 +280,11 @@ RSpec/NoExpectationExample:
     - 'spec/canon/isodoc_blockquotes_spec.rb'
     - 'spec/canon/match_scenarios_spec.rb'
 
+# Offense count: 2
+RSpec/RepeatedExample:
+  Exclude:
+    - 'spec/canon/comparison/encoding_normalization_spec.rb'
+
 # Offense count: 7
 # Configuration parameters: CustomTransform, IgnoreMethods, IgnoreMetadata, InflectorPath, EnforcedInflector.
 # SupportedInflectors: default, active_support
@@ -241,6 +309,17 @@ RSpec/VerifiedDoubles:
     - 'spec/canon/diff/xml_serialization_formatter_spec.rb'
     - 'spec/canon/tree_diff/operation_converter_spec.rb'
 
+# Offense count: 1
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: EnforcedStyle, ProceduralMethods, FunctionalMethods, AllowedMethods, AllowedPatterns, AllowBracesOnProceduralOneLiners, BracesRequiredMethods.
+# SupportedStyles: line_count_based, semantic, braces_for_chaining, always_braces
+# ProceduralMethods: benchmark, bm, bmbm, create, each_with_object, measure, new, realtime, tap, with_object
+# FunctionalMethods: let, let!, subject, watch
+# AllowedMethods: lambda, proc, it
+Style/BlockDelimiters:
+  Exclude:
+    - 'spec/canon/comparison/encoding_normalization_spec.rb'
+
 # Offense count: 1
 # This cop supports safe autocorrection (--autocorrect).
 # Configuration parameters: EnforcedStyle, AllowComments.
@@ -263,6 +342,13 @@ Style/IdenticalConditionalBranches:
     - 'lib/canon/diff_formatter/by_object/base_formatter.rb'
     - 'lib/canon/diff_formatter/legend.rb'
 
+# Offense count: 1
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: AllowMethodComparison, ComparisonsThreshold.
+Style/MultipleComparison:
+  Exclude:
+    - 'lib/canon/comparison/format_detector.rb'
+
 # Offense count: 1
 # Configuration parameters: AllowedMethods.
 # AllowedMethods: respond_to_missing?

data/README.adoc

@@ -214,6 +214,7 @@ Compare documents based on meaning, not formatting:
 * Comment handling with display control
 * Multiple match dimensions with behaviors
 * Predefined match profiles (strict, rendered, spec_friendly, content_only)
+* **Cross-encoding comparison**: Compare XML documents with different character encodings (UTF-8, Shift_JIS, ISO-8859-1, UTF-16) — Canon automatically normalizes to UTF-8 before comparison
 
 See link:docs/MATCH_OPTIONS[Match options] for details.
 

data/docs/features/match-options/index.adoc

@@ -151,6 +151,43 @@ sensitivity in XML instance documents:
 </text>
 ----
 
+The `xml:space` attribute affects both structural whitespace and text content:
+
+* **Structural whitespace** (whitespace-only text nodes between child elements)
+* **Text content whitespace** (whitespace within text nodes)
+
+.xml:space with structural_whitespace
+[example]
+====
+[source,ruby]
+----
+# With xml:space="preserve", structural whitespace is preserved
+xml1 = "<root xml:space='preserve'>\n  <text>Hello</text>\n</root>"
+xml2 = "<root xml:space='preserve'><text>Hello</text></root>"
+
+# These are NOT equivalent (structural whitespace differs)
+Canon::Comparison.equivalent?(xml1, xml2)
+# => false
+----
+====
+
+.xml:space with text_content
+[example]
+====
+[source,ruby]
+----
+# With xml:space="preserve", text content whitespace is preserved
+xml1 = '<root xml:space="preserve"><code>  indented  </code></root>'
+xml2 = '<root xml:space="preserve"><code>indented</code></root>'
+
+# These are NOT equivalent (text whitespace differs)
+Canon::Comparison.equivalent?(xml1, xml2,
+  match: { text_content: :strict }
+)
+# => false
+----
+====
+
 ==== Whitelist and blacklist options
 
 You can explicitly specify which elements are whitespace-sensitive using either short or long option names:
@@ -260,29 +297,44 @@ Canon::Comparison.equivalent?(xml1, xml2,
 
 ==== Examples
 
-.Using xml:space attribute
+.Using xml:space="preserve" for structural whitespace
+[source,ruby]
+----
+xml1 = "<root xml:space='preserve'>\n  <text>Hello</text>\n</root>"
+xml2 = "<root xml:space='preserve'><text>Hello</text></root>"
+
+# Structural whitespace differs - NOT equivalent
+Canon::Comparison.equivalent?(xml1, xml2)
+# => false
+----
+
+.Using xml:space="preserve" for text content
 [source,ruby]
 ----
-xml1 = '<root><code xml:space="preserve">  indented  </code></root>'
-xml2 = '<root><code xml:space="preserve">indented</code></root>'
+xml1 = '<root><code xml:space="preserve">  multiple   spaces  </code></root>'
+xml2 = '<root><code xml:space="preserve">multiple spaces</code></root>'
 
-# These are NOT equivalent (whitespace matters in xml:space="preserve")
+# Text content whitespace differs - NOT equivalent with text_content: :strict
 Canon::Comparison.equivalent?(xml1, xml2,
-  match: { structural_whitespace: :strict }
+  match: { text_content: :strict }
 )
 # => false
 ----
 
-.Using whitelist
+.Using sensitive_elements whitelist
 [source,ruby]
 ----
-# Make <p> elements whitespace-sensitive (strings, not symbols)
+# Make <sample> elements whitespace-sensitive (strings, not symbols)
+xml1 = "<sample>\n  content\n</sample>"
+xml2 = "<sample>content</sample>"
+
 Canon::Comparison.equivalent?(xml1, xml2,
   match: {
     structural_whitespace: :strict,
-    sensitive_elements: ["p", "pre"]
+    sensitive_elements: ["sample"]
   }
 )
+# => false (structural whitespace differs in <sample>)
 ----
 
 .Overriding HTML defaults

data/docs/understanding/formats/xml.adoc

@@ -340,6 +340,44 @@ Special attributes like `xml:lang`, `xml:space`, `xml:id`, and `xml:base` are pr
 When `xml:space="preserve"` is set, whitespace is preserved in descendants.
 ----
 
+=== Cross-encoding comparison
+
+Canon automatically normalizes XML character encodings before comparison, enabling
+cross-encoding comparisons to work correctly.
+
+**Supported encodings**: UTF-8, UTF-16 (all variants), Shift_JIS, EUC-JP, ISO-8859-1, and more.
+
+**How it works**:
+
+1. Extract the declared encoding from the XML declaration (e.g., `encoding="Shift_JIS"`)
+2. If declared encoding differs from UTF-8, transcode to UTF-8
+3. Handle cases where the declared encoding doesn't match actual bytes
+4. Use safe transcoding with replacement characters for invalid sequences
+
+.Cross-encoding comparison example
+[example]
+====
+[source,ruby]
+----
+# UTF-8 vs Shift_JIS - automatically normalized
+xml1 = "<root>日本語</root>"  # UTF-8
+xml2 = "<root>日本語</root>".encode("Shift_JIS")  # Shift_JIS
+
+Canon::Comparison.equivalent?(xml1, xml2)
+# => true (automatically transcoded to UTF-8 before comparison)
+
+# ASCII content works across all encodings
+xml3 = "<root>hello</root>"
+xml4 = "<root>hello</root>".encode("ISO-8859-1")
+
+Canon::Comparison.equivalent?(xml3, xml4)
+# => true
+----
+====
+
+This means you can compare XML files from different sources or systems without
+worrying about their native encoding.
+
 == Usage examples
 
 === Basic XML comparison

data/lib/canon/cache.rb

@@ -89,7 +89,8 @@ module Canon
       # @return [String] Cache key
       def key_for_format_detection(content)
         # Use first 100 chars for quick key, plus length
-        preview = content[0..100]
+        # Force to binary to avoid encoding compatibility issues
+        preview = content[0..100].b
         digest = Digest::SHA256.hexdigest(preview + content.length.to_s)
         "fmt:#{digest[0..16]}"
       end

data/lib/canon/comparison/format_detector.rb

@@ -62,7 +62,21 @@ module Canon
         # @param str [String] String to detect format of
         # @return [Symbol] Format type
         def detect_string_uncached(str)
-          trimmed = str.strip
+          # Convert to UTF-8 for consistent handling if possible
+          # This handles cases like UTF-16 encoded XML that would otherwise fail string operations
+          str_utf8 = if ["UTF-16", "UTF-16BE",
+                         "UTF-16LE"].include?(str.encoding.name)
+                       begin
+                         str.encode("UTF-8", str.encoding, invalid: :replace,
+                                                           undef: :replace, replace: "?")
+                       rescue EncodingError
+                         str.dup.force_encoding("BINARY").encode("UTF-8")
+                       end
+                     else
+                       str
+                     end
+
+          trimmed = str_utf8.strip
 
           # YAML indicators
           return :yaml if trimmed.start_with?("---")

data/lib/canon/comparison/whitespace_sensitivity.rb

@@ -89,6 +89,15 @@ module Canon
           insensitive = (insensitive_raw || []).map(&:to_s)
           return false if insensitive.include?(elem_name)
 
+          # Check if we should ignore xml:space (user override)
+          if respect_xml_space?(match_opts)
+            # Check xml:space="preserve" (document declaration)
+            return true if xml_space_preserve?(element)
+
+            # Check xml:space="default" (use configured behavior)
+            return false if xml_space_default?(element)
+          end
+
           # Whitelist: preserve whitespace
           sensitive = resolved_sensitive_elements(match_opts)
           return true if sensitive.include?(elem_name)

data/lib/canon/comparison/xml_comparator/node_parser.rb

@@ -25,6 +25,9 @@ module Canon
                                      preserve_whitespace: preserve_whitespace)
           end
 
+          # Normalize encoding before preprocessing (UTF-16 strings can't use strip, etc.)
+          node = Canon::Xml::DataModel.normalize_encoding(node)
+
           # Apply preprocessing to XML string before parsing
           xml_string = apply_preprocessing(node, preprocessing).strip
 

data/lib/canon/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Canon
-  VERSION = "0.1.17"
+  VERSION = "0.1.18"
 end

data/lib/canon/xml/data_model.rb

@@ -21,8 +21,11 @@ module Canon
       # @param preserve_whitespace [Boolean] Whether to preserve whitespace-only text nodes
       # @return [Nodes::RootNode] Root of the data model tree
       def self.from_xml(xml_string, preserve_whitespace: false)
+        # Normalize encoding before parsing
+        normalized_xml = normalize_encoding(xml_string)
+
         # Parse with Nokogiri
-        doc = Nokogiri::XML(xml_string, &:nonet)
+        doc = Nokogiri::XML(normalized_xml, &:nonet)
 
         # Check for relative namespace URIs (prohibited by C14N 1.1)
         check_for_relative_namespace_uris(doc)
@@ -31,6 +34,132 @@ module Canon
         build_from_nokogiri(doc, preserve_whitespace: preserve_whitespace)
       end
 
+      # Normalize XML string encoding to UTF-8
+      #
+      # Handles cases where:
+      # 1. The XML declaration specifies an encoding that doesn't match the actual encoding
+      # 2. The string's internal encoding is non-UTF-8 (without a declaration)
+      #
+      # For case 1, we check if the declared encoding matches the actual bytes.
+      # If bytes are valid UTF-8 despite the declaration, we update the declaration to UTF-8.
+      #
+      # @param xml_string [String] XML string to normalize
+      # @return [String] Normalized XML string with UTF-8 encoding
+      def self.normalize_encoding(xml_string)
+        return xml_string unless xml_string.is_a?(String)
+
+        # Extract declared encoding from XML declaration
+        declared_encoding = extract_xml_encoding(xml_string)
+
+        if declared_encoding
+          # Case 1: XML has a declaration
+          if declared_encoding.upcase != "UTF-8"
+            # Check if bytes are actually valid UTF-8 despite the declaration
+            utf8_reinterpreted = try_utf8_reinterpretation(xml_string)
+            if utf8_reinterpreted
+              # Bytes are valid UTF-8 - update declaration to UTF-8
+              return update_xml_declaration(xml_string, "UTF-8")
+            end
+
+            # Bytes aren't valid UTF-8 - must really be in declared encoding
+            return transcode_to_utf8(xml_string, declared_encoding)
+          end
+        elsif xml_string.encoding.name != "UTF-8"
+          # Case 2: No declaration but string encoding is non-UTF-8
+          # First, try to re-interpret bytes as UTF-8 (handles mislabeled strings)
+          reinterpreted = try_utf8_reinterpretation(xml_string)
+          return reinterpreted if reinterpreted
+
+          # If re-interpretation fails, try transcoding with the labeled encoding
+          return transcode_to_utf8(xml_string, xml_string.encoding.name)
+        end
+
+        xml_string
+      end
+
+      # Update the encoding declaration in an XML string
+      #
+      # @param xml_string [String] XML string
+      # @param new_encoding [String] New encoding to declare
+      # @return [String] XML string with updated declaration
+      def self.update_xml_declaration(xml_string, new_encoding)
+        xml_string.sub(/\bencoding\s*=\s*["'][^"']+["']/i) do |_match|
+          %(encoding="#{new_encoding}")
+        end
+      end
+
+      # Transcode string to UTF-8
+      #
+      # @param xml_string [String] String to transcode
+      # @param source_encoding [String] Source encoding to interpret bytes as
+      # @return [String] UTF-8 transcoded string
+      def self.transcode_to_utf8(xml_string, source_encoding)
+        # First, check if the bytes are actually valid UTF-8 despite the declared encoding
+        # If so, just re-interpret as UTF-8 (common case: declaration is wrong)
+        if source_encoding != "UTF-8"
+          # Force the bytes to be interpreted as the declared encoding, then check validity
+          forced = xml_string.dup.force_encoding(source_encoding)
+          if forced.valid_encoding?
+            # Now check if the same bytes are valid UTF-8
+            utf8_check = xml_string.dup.force_encoding("UTF-8")
+            if utf8_check.valid_encoding?
+              # Bytes are valid UTF-8 - the declaration is likely wrong
+              # Return the string as UTF-8 (already is)
+              return xml_string.dup.force_encoding("UTF-8")
+            end
+
+            # Bytes aren't valid UTF-8, so they must really be in source_encoding
+            # Proceed with transcoding
+            return forced.encode("UTF-8", source_encoding,
+                                 invalid: :replace,
+                                 undef: :replace,
+                                 replace: "?")
+          end
+        end
+
+        # Already UTF-8 or transcoding failed, return as-is
+        xml_string.dup.force_encoding("UTF-8")
+      rescue EncodingError
+        xml_string
+      end
+
+      # Attempt to re-interpret string as UTF-8 if bytes are valid UTF-8
+      #
+      # This handles the case where a string was incorrectly labeled with a different
+      # encoding (e.g., `.encode("Shift_JIS")` on a UTF-8 string) but the actual
+      # bytes are valid UTF-8.
+      #
+      # @param xml_string [String] XML string to check
+      # @return [String, nil] UTF-8 re-interpreted string, or nil if not possible
+      def self.try_utf8_reinterpretation(xml_string)
+        return xml_string if xml_string.encoding.name == "UTF-8"
+
+        # Try forcing to UTF-8 and see if it's valid
+        forced = xml_string.dup.force_encoding("UTF-8")
+        return forced if forced.valid_encoding?
+
+        nil
+      end
+
+      # Extract encoding from XML declaration
+      #
+      # @param xml_string [String] XML string
+      # @return [String, nil] Declared encoding or nil if not found
+      def self.extract_xml_encoding(xml_string)
+        # Match XML declaration with encoding attribute
+        # Handles: <?xml version="1.0" encoding="UTF-8"?>
+        # and: <?xml version='1.0' encoding='UTF-8'?>
+        #
+        # Use binary encoding to avoid encoding compatibility issues
+        # when the string has non-ASCII compatible encoding (e.g., UTF-16)
+        binary_string = xml_string.dup.force_encoding("BINARY")
+        if binary_string =~ /\A\s*<\?xml[^>]*\bencoding\s*=\s*["']([^"']+)["'][^>]*\?>/i
+          return Regexp.last_match(1)
+        end
+
+        nil
+      end
+
       # Alias for compatibility with base class interface
       def self.parse(xml_string)
         from_xml(xml_string)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: canon
 version: !ruby/object:Gem::Version
-  version: 0.1.17
+  version: 0.1.18
 platform: ruby
 authors:
 - Ribose Inc.