canon 0.1.5 → 0.1.7

data/lib/canon/comparison/html_comparator.rb

@@ -1,11 +1,16 @@
 # frozen_string_literal: true
 
 require "nokogiri"
+require_relative "../comparison" # Load base module with constants first
 require_relative "xml_comparator"
 require_relative "match_options"
 require_relative "comparison_result"
+require_relative "compare_profile"
+require_relative "html_compare_profile"
 require_relative "../diff/diff_node"
 require_relative "../diff/diff_classifier"
+require_relative "strategies/match_strategy_factory"
+require_relative "../html/data_model"
 
 module Canon
   module Comparison
@@ -59,24 +64,41 @@ module Canon
             global_options: opts[:global_options],
           )
 
+          # Parse nodes to detect HTML version before creating profile
+          # We need to parse early to know if we're dealing with HTML4 or HTML5
+          node1 = parse_node(html1, match_opts_hash[:preprocessing],
+                             match_opts_hash)
+          node2 = parse_node(html2, match_opts_hash[:preprocessing],
+                             match_opts_hash)
+
+          # Detect HTML version from parsed nodes
+          html_version = detect_html_version_from_node(node1)
+
+          # Create HTML-specific compare profile
+          compare_profile = HtmlCompareProfile.new(
+            match_opts_hash,
+            html_version: html_version,
+          )
+
           # Wrap in ResolvedMatchOptions for DiffClassifier
           match_opts = Canon::Comparison::ResolvedMatchOptions.new(
             match_opts_hash,
             format: :html,
+            compare_profile: compare_profile,
           )
 
           # Store resolved match options hash for use in comparison logic
           opts[:match_opts] = match_opts_hash
 
+          # Use tree diff if semantic_diff option is enabled
+          if match_opts.semantic_diff?
+            return perform_semantic_tree_diff(html1, html2, opts,
+                                              match_opts_hash)
+          end
+
           # Create child_opts with resolved options
           child_opts = opts.merge(child_opts)
 
-          # Parse nodes if they are strings, applying preprocessing if needed
-          node1 = parse_node(html1, match_opts_hash[:preprocessing],
-                             match_opts_hash)
-          node2 = parse_node(html2, match_opts_hash[:preprocessing],
-                             match_opts_hash)
-
           # Serialize preprocessed nodes for diff display (avoid re-preprocessing)
           preprocessed_str1 = serialize_for_display(node1)
           preprocessed_str2 = serialize_for_display(node2)
@@ -86,11 +108,19 @@ module Canon
 
           # DocumentFragment nodes need special handling - compare their children
           # instead of the fragment nodes themselves
-          if node1.is_a?(Nokogiri::HTML4::DocumentFragment) &&
-              node2.is_a?(Nokogiri::HTML4::DocumentFragment)
-            # Compare children of fragments
-            children1 = node1.children.to_a
-            children2 = node2.children.to_a
+          if (node1.is_a?(Nokogiri::HTML4::DocumentFragment) ||
+              node1.is_a?(Nokogiri::XML::DocumentFragment)) &&
+              (node2.is_a?(Nokogiri::HTML4::DocumentFragment) ||
+              node2.is_a?(Nokogiri::XML::DocumentFragment))
+            # Compare children of fragments - filter them first
+            all_children1 = node1.children.to_a
+            all_children2 = node2.children.to_a
+
+            # Filter children based on match options (e.g., ignore comments)
+            children1 = XmlComparator.send(:filter_children, all_children1,
+                                           opts)
+            children2 = XmlComparator.send(:filter_children, all_children2,
+                                           opts)
 
             if children1.length != children2.length
               result = Comparison::UNEQUAL_ELEMENTS
@@ -129,79 +159,205 @@ module Canon
               format: :html,
               html_version: detect_html_version_from_node(node1),
               match_options: match_opts_hash,
+              algorithm: :dom,
             )
+          elsif result != Comparison::EQUIVALENT && !differences.empty?
+            # Non-verbose mode: check equivalence
+            # If comparison found differences, classify them to determine if normative
+            classifier = Canon::Diff::DiffClassifier.new(match_opts)
+            classifier.classify_all(differences.select do |d|
+              d.is_a?(Canon::Diff::DiffNode)
+            end)
+            # Equivalent if no normative differences (matches semantic algorithm)
+            differences.none?(&:normative?)
           else
+            # Either equivalent or no differences tracked
             result == Comparison::EQUIVALENT
           end
         end
 
         private
 
+        # Perform semantic tree diff using SemanticTreeMatchStrategy
+        #
+        # @param html1 [String, Nokogiri::HTML::Document] First HTML
+        # @param html2 [String, Nokogiri::HTML::Document] Second HTML
+        # @param opts [Hash] Comparison options
+        # @param match_opts_hash [Hash] Resolved match options
+        # @return [Boolean, ComparisonResult] Result of tree diff comparison
+        def perform_semantic_tree_diff(html1, html2, opts, match_opts_hash)
+          # Parse to Canon::Xml::Node (preserves preprocessing)
+          # For HTML, we parse as XML to get Canon::Xml::Node structure
+          node1 = parse_node_for_semantic(html1,
+                                          match_opts_hash[:preprocessing])
+          node2 = parse_node_for_semantic(html2,
+                                          match_opts_hash[:preprocessing])
+
+          # Create strategy using factory
+          strategy = Strategies::MatchStrategyFactory.create(
+            format: :html,
+            match_options: match_opts_hash,
+          )
+
+          # Pass Canon::Xml::Node directly - adapter now handles it
+          differences = strategy.match(node1, node2)
+
+          # Return based on verbose mode
+          if opts[:verbose]
+            # Get preprocessed strings for display
+            preprocessed = strategy.preprocess_for_display(node1, node2)
+
+            # Detect HTML version (default to HTML5 for Canon nodes)
+            html_version = :html5
+
+            # Return ComparisonResult with strategy metadata
+            ComparisonResult.new(
+              differences: differences,
+              preprocessed_strings: preprocessed,
+              format: :html,
+              html_version: html_version,
+              match_options: match_opts_hash.merge(strategy.metadata),
+              algorithm: :semantic,
+            )
+          else
+            # Simple boolean result - equivalent if no normative differences
+            differences.none?(&:normative?)
+          end
+        end
+
+        # Parse node as fragment to preserve actual content
+        # Uses HTML4.fragment or HTML5.fragment based on content detection
+        #
+        # @param node [String, Nokogiri node] Node to parse
+        # @param preprocessing [Symbol] Preprocessing mode
+        # @param match_opts [Hash] Match options
+        # @return [Nokogiri::HTML::DocumentFragment] Parsed fragment
+        def parse_node_as_fragment(node, preprocessing = :none, match_opts = {})
+          # If already an XML fragment (no meta tags), return it
+          if node.is_a?(Nokogiri::XML::DocumentFragment)
+            return node
+          end
+
+          # Convert HTML fragments to string and re-parse as XML to remove phantom tags
+          # This handles cases where pre-parsed HTML4/HTML5 fragments have auto-inserted meta
+          html_string = if node.is_a?(Nokogiri::HTML4::DocumentFragment) ||
+              node.is_a?(Nokogiri::HTML5::DocumentFragment)
+                          node.to_s # Use to_s to avoid re-inserting meta tags
+                        elsif node.is_a?(String)
+                          node
+                        else
+                          node.to_html
+                        end
+
+          # Use XML fragment parser to preserve structure without auto-generated elements
+          # This avoids both HTML4's meta tag insertion and HTML5's tag stripping
+          # See: https://stackoverflow.com/questions/25998824/stop-nokogiri-from-adding-doctype-and-meta-tags
+          frag = Nokogiri::XML.fragment(html_string)
+
+          # Apply preprocessing if needed
+          if preprocessing == :rendered
+            normalize_html_style_script_comments(frag)
+            normalize_rendered_whitespace(frag, match_opts)
+            remove_whitespace_only_text_nodes(frag)
+          end
+
+          frag
+        end
+
+        # Parse HTML for semantic tree diff using Canon::Html::DataModel
+        # Returns Canon::Xml::Node for preprocessing preservation
+        #
+        # @param html [String, Object] HTML to parse
+        # @param preprocessing [Symbol] Preprocessing mode
+        # @return [Canon::Xml::Node] Parsed Canon node
+        def parse_node_for_semantic(html, preprocessing = :none)
+          # If already a Canon::Xml::Node, return as-is
+          return html if html.is_a?(Canon::Xml::Node)
+
+          # Convert to string if needed
+          html_string = if html.is_a?(String)
+                          html
+                        elsif html.respond_to?(:to_html)
+                          html.to_html
+                        elsif html.respond_to?(:to_s)
+                          html.to_s
+                        else
+                          raise Canon::Error,
+                                "Unable to convert HTML to string: #{html.class}"
+                        end
+
+          # Strip DOCTYPE for consistent parsing
+          html_string = html_string.gsub(/<!DOCTYPE[^>]*>/i, "").strip
+
+          # Apply preprocessing to HTML string before parsing
+          processed_html = case preprocessing
+                           when :normalize
+                             # Normalize whitespace
+                             html_string.lines.map(&:strip).reject(&:empty?).join("\n")
+                           when :c14n
+                             # Canonicalize
+                             Canon::Xml::C14n.canonicalize(html_string,
+                                                           with_comments: false)
+                           when :format
+                             # Pretty format
+                             Canon.format(html_string, :html)
+                           else
+                             # :none or unrecognized
+                             html_string
+                           end
+
+          # Parse using Canon::Html::DataModel to get Canon::Xml::Node
+          # HTML parsing with proper HTML-specific handling
+          Canon::Html::DataModel.from_html(processed_html)
+        end
+
         # Parse a node from string or return as-is
         # Applies preprocessing transformation before parsing if specified
+        # For DOM comparison, returns Nokogiri nodes (not Canon::Xml::Node)
         def parse_node(node, preprocessing = :none, match_opts = {})
           # If already a Nokogiri node, check for incompatible XML documents
-          # Only raise error for non-string incompatible formats
           unless node.is_a?(String)
             # Detect if this is an XML document (not HTML)
-            # Strings are allowed since they can be wrapped/parsed as needed
             if is_xml_document?(node)
               raise Canon::CompareFormatMismatchError.new(:xml, :html)
             end
 
-            # For :rendered preprocessing, apply normalization even to pre-parsed nodes
-            if preprocessing == :rendered
-              # If already a DocumentFragment with :rendered, just normalize it
-              if node.is_a?(Nokogiri::HTML4::DocumentFragment) ||
-                  node.is_a?(Nokogiri::HTML5::DocumentFragment) ||
-                  node.is_a?(Nokogiri::XML::DocumentFragment)
-                # Normalize whitespace directly without re-parsing
-                normalize_html_style_script_comments(node)
-                normalize_rendered_whitespace(node, match_opts)
-                return node
+            # Normalize HTML documents to fragments to avoid DTD differences
+            # This ensures comparing string with document works correctly
+            if node.is_a?(Nokogiri::HTML::Document) ||
+                node.is_a?(Nokogiri::HTML4::Document) ||
+                node.is_a?(Nokogiri::HTML5::Document)
+              # Get root element and create fragment from its outer HTML
+              # This avoids DOCTYPE and other document-level nodes
+              root = node.at_css("html") || node.root
+              if root
+                node = Nokogiri::XML.fragment(root.to_html)
               end
+            end
 
-              # Normalize whitespace directly without re-parsing
-              normalize_html_style_script_comments(node)
-              normalize_rendered_whitespace(node, match_opts)
-              return node
+            # For :rendered preprocessing with Nokogiri nodes
+            if preprocessing == :rendered
+              # Normalize and return
+              frag = node.is_a?(Nokogiri::XML::DocumentFragment) ? node : Nokogiri::XML.fragment(node.to_html)
+              normalize_html_style_script_comments(frag)
+              normalize_rendered_whitespace(frag, match_opts)
+              remove_whitespace_only_text_nodes(frag)
+              return frag
             end
 
-            # For other preprocessing, just return the node (including DocumentFragments)
+            # Return Nokogiri node (now normalized if it was a document)
             return node
           end
 
           # Check if string contains XML declaration but is actually HTML
-          # Nokogiri::HTML4.to_s adds <?xml...?> but the content is still HTML
-          # Check if this is actually HTML content after the declaration
-          # Look for <html tag which indicates HTML
           if node.strip.start_with?("<?xml") && !node.match?(/<html[\s>]/i)
             # No <html> tag, this is likely pure XML
             raise Canon::CompareFormatMismatchError.new(:xml, :html)
           end
 
-          # Has <?xml but also <html> tag, so it's HTML with XML declaration
-          # (common output from Nokogiri::HTML4#to_s)
-
-          # For :rendered preprocessing, handle separately to avoid double-parsing
-          if preprocessing == :rendered
-            # Check if this is a full HTML document or a fragment
-            # Use full document parsing if it has <html> tag
-            if node.match?(/<html[\s>]/i)
-              doc = Nokogiri::HTML(node, &:noblanks)
-              normalize_html_style_script_comments(doc)
-              normalize_rendered_whitespace(doc, match_opts)
-              remove_whitespace_only_text_nodes(doc)
-              return doc
-            else
-              # Use fragment for partial HTML
-              frag = Nokogiri::HTML4.fragment(node)
-              normalize_html_style_script_comments(frag)
-              normalize_rendered_whitespace(frag, match_opts)
-              remove_whitespace_only_text_nodes(frag)
-              return frag
-            end
-          end
+          # Strip DOCTYPE declarations from HTML strings
+          # This normalizes parsed HTML (which includes DOCTYPE) with raw HTML strings
+          node = node.gsub(/<!DOCTYPE[^>]*>/i, "").strip
 
           # Apply preprocessing to HTML string before parsing
           html_string = case preprocessing
@@ -216,15 +372,52 @@ module Canon
                           # Pretty format the HTML
                           Canon.format(node, :html)
                         else
-                          # :none or unrecognized - use as-is
+                          # :none, :rendered or unrecognized - use as-is
                           node
                         end
 
-          # Use Nokogiri for HTML and normalize style/script comments
-          # Use noblanks to prevent Nokogiri from adding structural whitespace
-          doc = Nokogiri::HTML(html_string, &:noblanks)
-          normalize_html_style_script_comments(doc)
-          doc
+          # Parse as Nokogiri fragment for DOM comparison
+          # Use XML fragment parser to avoid auto-inserted meta tags
+          frag = Nokogiri::XML.fragment(html_string)
+
+          # Apply :rendered preprocessing if needed
+          if preprocessing == :rendered
+            normalize_html_style_script_comments(frag)
+            normalize_rendered_whitespace(frag, match_opts)
+            remove_whitespace_only_text_nodes(frag)
+          end
+
+          frag
+        end
+
+        # Normalize HTML comments within style and script tags for DataModel nodes
+        def normalize_html_style_script_comments_datamodel(root)
+          # Walk the tree to find style/script elements
+          find_and_normalize_style_script(root)
+        end
+
+        def find_and_normalize_style_script(node)
+          return unless node.respond_to?(:children)
+
+          node.children.each do |child|
+            next unless child.is_a?(Canon::Xml::Nodes::ElementNode)
+
+            # If this is a style or script element, normalize its text content
+            if %w[style script].include?(child.name.downcase)
+              # Get text children and remove HTML comments from them
+              child.children.each do |text_child|
+                next unless text_child.is_a?(Canon::Xml::Nodes::TextNode)
+
+                # Remove HTML comments from text content
+                normalized = text_child.value.gsub(/<!--.*?-->/m, "").strip
+                # Update the text value
+                text_child.instance_variable_set(:@value, normalized)
+              end
+            end
+
+            # Recursively process children
+            find_and_normalize_style_script(child)
+          end
         end
 
         # Detect HTML version from content
@@ -244,12 +437,12 @@ module Canon
           end
         end
 
-        # Detect HTML version from Nokogiri node
+        # Detect HTML version from node
         #
-        # @param node [Nokogiri::XML::Node] Nokogiri HTML node
+        # @param node [Canon::Xml::Node, Nokogiri::XML::Node] HTML node
         # @return [Symbol] :html5 or :html4
         def detect_html_version_from_node(node)
-          # Check node type
+          # Check node type for Nokogiri
           if node.is_a?(Nokogiri::HTML5::Document) ||
               node.is_a?(Nokogiri::HTML5::DocumentFragment)
             :html5
@@ -257,20 +450,27 @@ module Canon
               node.is_a?(Nokogiri::HTML4::DocumentFragment)
             :html4
           else
-            # Default to HTML4 for compatibility
-            :html4
+            # Default to HTML5 for Canon::Xml::Node and unknown types
+            :html5
           end
         end
 
         # Serialize node to string for diff display
         # This ensures the displayed diff matches what was compared
         #
-        # @param node [Nokogiri::HTML::Document] Parsed HTML node
+        # @param node [Canon::Xml::Node, Nokogiri::HTML::Document] Parsed node
         # @return [String] Serialized HTML string
         def serialize_for_display(node)
-          # Get string representation with formatting for line-by-line diffs
-          # Use to_html which preserves line structure for diff display
-          node.to_html
+          # Use XmlComparator's serializer for Canon::Xml::Node
+          if node.is_a?(Canon::Xml::Node)
+            XmlComparator.send(:serialize_node_to_xml, node)
+          elsif node.respond_to?(:to_html)
+            node.to_html
+          elsif node.respond_to?(:to_xml)
+            node.to_xml
+          else
+            node.to_s
+          end
         end
 
         # Normalize HTML comments within style and script tags
@@ -301,14 +501,25 @@ module Canon
         #
         # @param doc [Nokogiri::HTML::Document] Document to normalize
         # @param match_opts [Hash] Match options to respect during normalization
-        def normalize_rendered_whitespace(doc, match_opts = {})
+        # @param compare_profile [HtmlCompareProfile] Optional profile for whitespace rules
+        def normalize_rendered_whitespace(doc, match_opts = {},
+compare_profile = nil)
           # If text_content is :strict, don't normalize ANY text content
           # This allows users to explicitly request strict text matching
           return if match_opts[:text_content] == :strict
 
           # Elements where whitespace is significant - don't normalize
-          # This is an HTML rendering rule, not a match option
-          preserve_whitespace = %w[pre code textarea script style]
+          # Use profile if available, otherwise use default list
+          preserve_whitespace = if compare_profile.is_a?(HtmlCompareProfile)
+                                  # Profile handles HTML-specific whitespace rules
+                                  %w[pre code textarea script
+                                     style].select do |elem|
+                                    compare_profile.preserve_whitespace?(elem)
+                                  end
+                                else
+                                  # Fallback to default list
+                                  %w[pre code textarea script style]
+                                end
 
           # Walk all text nodes
           doc.xpath(".//text()").each do |text_node|
@@ -360,8 +571,18 @@ module Canon
         # Remove whitespace-only text nodes from the document
         # These are typically insignificant in HTML rendering (e.g., between
         # block elements)
+        #
+        # CRITICAL: Do NOT remove whitespace-only text nodes from whitespace-sensitive
+        # elements like <pre>, <code>, <textarea>, <script>, <style>
         def remove_whitespace_only_text_nodes(doc)
+          # Elements where whitespace is significant - don't remove whitespace-only nodes
+          preserve_whitespace = %w[pre code textarea script style]
+
           doc.xpath(".//text()").each do |text_node|
+            # CRITICAL: Skip if this text node is inside a whitespace-preserving element
+            parent = text_node.parent
+            next if ancestor_preserves_whitespace?(parent, preserve_whitespace)
+
             # Remove if the text is only whitespace (after normalization)
             if text_node.content.strip.empty?
               text_node.remove

data/lib/canon/comparison/html_compare_profile.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+require_relative "compare_profile"
+
+module Canon
+  module Comparison
+    # HtmlCompareProfile extends CompareProfile with HTML-specific comparison policies
+    #
+    # HTML has different semantics than XML:
+    # 1. Comments are presentational (default to :ignore unless explicitly :strict)
+    # 2. Whitespace preservation required in specific elements
+    # 3. Case sensitivity differs between HTML4 and HTML5
+    # 4. Self-closing tags handled differently
+    #
+    # This class provides HTML-specific policy decisions while maintaining
+    # the separation of concerns established by CompareProfile.
+    class HtmlCompareProfile < CompareProfile
+      attr_reader :html_version
+
+      # @param match_options [ResolvedMatchOptions, Hash] The match options to use
+      # @param html_version [Symbol] The HTML version (:html4 or :html5)
+      def initialize(match_options, html_version: :html5)
+        super(match_options)
+        @html_version = html_version
+      end
+
+      # Override for HTML-specific comment handling
+      #
+      # In HTML, comments are presentational content (not part of the DOM semantics)
+      # unless explicitly set to :strict. This differs from XML where comments
+      # may carry semantic meaning.
+      #
+      # HTML default for comments is :ignore, so comments don't affect equivalence
+      # unless the user explicitly sets comments: :strict
+      #
+      # @param dimension [Symbol] The match dimension to check
+      # @return [Boolean] true if differences affect equivalence
+      def affects_equivalence?(dimension)
+        # Comments in HTML: default is :ignore (presentational)
+        # Only affect equivalence if explicitly set to :strict
+        if dimension == :comments
+          # Check if comments key exists in options
+          if match_options.is_a?(Hash)
+            # If comments key doesn't exist, default to false (HTML default: ignore)
+            return false unless match_options.key?(:comments)
+
+            # If key exists, check if it's :strict
+            return match_options[:comments] == :strict
+          elsif match_options.respond_to?(:behavior_for)
+            behavior = behavior_for(dimension)
+            # In HTML, only :strict makes comments affect equivalence
+            return behavior == :strict
+          end
+          # Default: comments don't affect equivalence in HTML
+          return false
+        end
+
+        # All other dimensions use base class behavior
+        super
+      end
+
+      # Check if whitespace should be preserved for a given element
+      #
+      # HTML has specific elements where whitespace is significant:
+      # <pre>, <code>, <textarea>, <script>, <style>
+      #
+      # @param element_name [String] The element name to check
+      # @return [Boolean] true if whitespace should be preserved
+      def preserve_whitespace?(element_name)
+        whitespace_sensitive_elements.include?(element_name.to_s.downcase)
+      end
+
+      # Check if element names should be compared case-sensitively
+      #
+      # HTML4 is case-insensitive, HTML5 is case-sensitive
+      #
+      # @return [Boolean] true if case-sensitive comparison
+      def case_sensitive?
+        @html_version == :html5
+      end
+
+      private
+
+      # Elements where whitespace is semantically significant in HTML
+      # @return [Array<String>] List of element names
+      def whitespace_sensitive_elements
+        %w[pre code textarea script style]
+      end
+
+      # Check if a dimension is explicitly set to :strict
+      # @param dimension [Symbol] The match dimension
+      # @return [Boolean] true if explicitly :strict
+      def explicitly_strict?(dimension)
+        behavior_for(dimension) == :strict
+      end
+
+      # Check if an option was explicitly provided in match_options
+      # @param dimension [Symbol] The match dimension
+      # @return [Boolean] true if option was explicitly set
+      def has_explicit_option?(dimension)
+        if match_options.is_a?(Hash)
+          match_options.key?(dimension)
+        elsif match_options.respond_to?(:[])
+          # For ResolvedMatchOptions, check if key exists
+          begin
+            match_options[dimension]
+            true
+          rescue StandardError
+            false
+          end
+        else
+          false
+        end
+      end
+    end
+  end
+end