canon 0.1.8 → 0.1.9

data/scripts/investigate_remaining_16.rb

@@ -1,214 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-# Investigation script for the remaining 16 semantic vs DOM differences
-# Goal: Achieve DOM parity (43 → 39 failures)
-
-require "bundler/setup"
-require "canon"
-
-# The 16 test cases that fail with semantic but pass with DOM
-REMAINING_FAILURES = [
-  { file: "blocks_spec.rb", line: 352, category: "Whitespace in <pre>" },
-  { file: "footnotes_spec.rb", line: 740, category: "Element matching" },
-  { file: "inline_spec.rb", line: 1012, category: "Element matching" },
-  { file: "inline_spec.rb", line: 1251, category: "Element matching" },
-  { file: "postproc_spec.rb", line: 948, category: "HTML escapes" },
-  { file: "postproc_word_spec.rb", line: 372, category: "Word processing" },
-  { file: "postproc_word_spec.rb", line: 576, category: "Word processing" },
-  { file: "presentation_xml_numbers_override_spec.rb", line: 2095,
-    category: "Number formatting" },
-  { file: "presentation_xml_spec.rb", line: 1288, category: "Presentation" },
-  { file: "presentation_xml_spec.rb", line: 1500, category: "Presentation" },
-  { file: "ref_spec.rb", line: 906, category: "References" },
-  { file: "sourcecode_spec.rb", line: 124, category: "Sourcecode" },
-  { file: "sourcecode_spec.rb", line: 610, category: "Sourcecode" },
-  { file: "terms_spec.rb", line: 1445, category: "Terms" },
-  { file: "xref_format_spec.rb", line: 628, category: "Cross-references" },
-  { file: "xref_spec.rb", line: 315, category: "Cross-references" },
-].freeze
-
-class FailureInvestigator
-  def initialize(test_case)
-    @test_case = test_case
-    @spec_file = File.join(ENV["HOME"], "mn/isodoc/spec", test_case[:file])
-  end
-
-  def investigate
-    puts "\n#{'=' * 80}"
-    puts "Investigating: #{@test_case[:file]}:#{@test_case[:line]}"
-    puts "Category: #{@test_case[:category]}"
-    puts "=" * 80
-
-    extract_test_context
-    analyze_failure_mode
-    suggest_fix
-  end
-
-  private
-
-  def extract_test_context
-    puts "\n--- Test Context ---"
-
-    unless File.exist?(@spec_file)
-      puts "ERROR: Spec file not found: #{@spec_file}"
-      return
-    end
-
-    lines = File.readlines(@spec_file)
-    target_line = @test_case[:line] - 1
-
-    # Find the test block
-    start_line = target_line
-    while start_line.positive? && !lines[start_line].strip.start_with?("it ",
-                                                                       "specify ")
-      start_line -= 1
-    end
-
-    # Extract test name
-    test_name = lines[start_line].strip if start_line >= 0
-    puts "Test: #{test_name}"
-
-    # Show context around the assertion
-    context_start = [target_line - 5, 0].max
-    context_end = [target_line + 5, lines.length - 1].min
-
-    puts "\nContext (lines #{context_start + 1}-#{context_end + 1}):"
-    (context_start..context_end).each do |i|
-      marker = i == target_line ? ">>>" : "   "
-      puts "#{marker} #{i + 1}: #{lines[i].rstrip}"
-    end
-  end
-
-  def analyze_failure_mode
-    puts "\n--- Failure Analysis ---"
-
-    case @test_case[:category]
-    when "Whitespace in <pre>"
-      puts "Pattern: Whitespace handling in <pre> elements"
-      puts "Known issue: Semantic correctly detects whitespace differences"
-      puts "DOM behavior: May normalize whitespace incorrectly"
-      puts "Action needed: Verify if DOM is wrong or test expectations need adjustment"
-
-    when "Element matching"
-      puts "Pattern: Element matching/comparison issues"
-      puts "Possible causes:"
-      puts "  - Attribute ordering differences"
-      puts "  - Namespace handling"
-      puts "  - Element signature computation"
-      puts "Action needed: Check signature and matching logic"
-
-    when "HTML escapes"
-      puts "Pattern: HTML entity/escape handling"
-      puts "Possible causes:"
-      puts "  - Entity normalization differences"
-      puts "  - Character reference handling"
-      puts "Action needed: Check entity handling in adapters"
-
-    when "Word processing"
-      puts "Pattern: Word-specific HTML processing"
-      puts "Possible causes:"
-      puts "  - Word-specific markup handling"
-      puts "  - Style attribute differences"
-      puts "Action needed: Check Word HTML adapter behavior"
-
-    when "Number formatting", "Presentation"
-      puts "Pattern: XML presentation/formatting"
-      puts "Possible causes:"
-      puts "  - Number/formatting element handling"
-      puts "  - Presentation attribute differences"
-      puts "Action needed: Check XML adapter presentation logic"
-
-    when "References", "Cross-references"
-      puts "Pattern: Reference/cross-reference handling"
-      puts "Possible causes:"
-      puts "  - Reference ID matching"
-      puts "  - Link element comparison"
-      puts "Action needed: Check reference element signatures"
-
-    when "Sourcecode"
-      puts "Pattern: Source code block handling"
-      puts "Possible causes:"
-      puts "  - Whitespace in code blocks (like <pre>)"
-      puts "  - Code element attribute matching"
-      puts "Action needed: Check sourcecode element handling"
-
-    when "Terms"
-      puts "Pattern: Term definition handling"
-      puts "Possible causes:"
-      puts "  - Term element matching"
-      puts "  - Definition structure differences"
-      puts "Action needed: Check term element signatures"
-    end
-  end
-
-  def suggest_fix
-    puts "\n--- Suggested Investigation Steps ---"
-    puts "1. Run the specific test with both DOM and semantic:"
-    puts "   cd ~/mn/isodoc"
-    puts "   CANON_DIFF_MODE=dom bundle exec rspec #{@test_case[:file]}:#{@test_case[:line]}"
-    puts "   CANON_DIFF_MODE=semantic_tree bundle exec rspec #{@test_case[:file]}:#{@test_case[:line]}"
-
-    puts "\n2. If test fails with semantic, extract actual vs expected:"
-    puts "   - Look for be_equivalent_to matcher"
-    puts "   - Save actual and expected to temp files"
-    puts "   - Compare manually"
-
-    puts "\n3. Determine root cause:"
-    puts "   - Is semantic detecting a real difference? → Test may need adjustment"
-    puts "   - Is semantic wrongly flagging? → Bug in semantic algorithm"
-    puts "   - Is DOM wrongly passing? → DOM has a bug we're fixing"
-
-    puts "\n4. Check ROOT_CAUSE_ANALYSIS.md for similar patterns"
-  end
-end
-
-# Main execution
-puts "Investigating 16 Remaining Semantic vs DOM Differences"
-puts "Goal: Achieve DOM parity (43 → 39 failures)"
-puts "\nTotal cases to investigate: #{REMAINING_FAILURES.length}"
-
-if ARGV.empty?
-  puts "\nUsage:"
-  puts "  #{$0}                    # Show all cases"
-  puts "  #{$0} <number>           # Investigate specific case (1-#{REMAINING_FAILURES.length})"
-  puts "  #{$0} <category>         # Investigate all cases in category"
-  puts "\nAvailable categories:"
-  REMAINING_FAILURES.map do |t|
-    t[:category]
-  end.uniq.sort.each { |c| puts "  - #{c}" }
-
-  puts "\nAll cases:"
-  REMAINING_FAILURES.each_with_index do |test, idx|
-    puts "  #{idx + 1}. #{test[:file]}:#{test[:line]} - #{test[:category]}"
-  end
-else
-  arg = ARGV[0]
-
-  if /^\d+$/.match?(arg)
-    # Specific case number
-    idx = arg.to_i - 1
-    if idx >= 0 && idx < REMAINING_FAILURES.length
-      investigator = FailureInvestigator.new(REMAINING_FAILURES[idx])
-      investigator.investigate
-    else
-      puts "ERROR: Invalid case number. Must be 1-#{REMAINING_FAILURES.length}"
-    end
-  else
-    # Category filter
-    category = arg
-    matching = REMAINING_FAILURES.select do |t|
-      t[:category].downcase.include?(category.downcase)
-    end
-
-    if matching.empty?
-      puts "ERROR: No cases found for category: #{category}"
-    else
-      puts "\nInvestigating #{matching.length} cases matching '#{category}':"
-      matching.each do |test|
-        investigator = FailureInvestigator.new(test)
-        investigator.investigate
-      end
-    end
-  end
-end

data/scripts/run_single_test.rb

@@ -1,29 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-# Run a specific test with both algorithms to see if it's a false positive
-# Usage: ruby scripts/run_single_test.rb FILE:LINE
-
-require "bundler/setup"
-require_relative "../../mn/isodoc/spec/spec_helper"
-
-# Run tests and capture results
-test_file = ARGV[0] || "../../mn/isodoc/spec/isodoc/sourcecode_spec.rb:124"
-
-puts "=" * 80
-puts "Running test: #{test_file}"
-puts "=" * 80
-
-# First with DOM
-puts "\n1. With DOM algorithm:"
-puts "-" * 40
-ENV["CANON_HTML_DIFF_ALGORITHM"] = "dom"
-ENV["CANON_XML_DIFF_ALGORITHM"] = "dom"
-system("cd ../../mn/isodoc && bundle exec rspec #{test_file} --format documentation 2>&1 | tail -20")
-
-# Then with Semantic
-puts "\n2. With Semantic algorithm:"
-puts "-" * 40
-ENV["CANON_HTML_DIFF_ALGORITHM"] = "semantic"
-ENV["CANON_XML_DIFF_ALGORITHM"] = "semantic"
-system("cd ../../mn/isodoc && bundle exec rspec #{test_file} --format documentation 2>&1 | tail -20")

data/scripts/test_all_false_positives.rb

@@ -1,95 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-# Test all 16 false positives with both algorithms
-# Usage: ruby scripts/test_all_false_positives.rb
-
-FALSE_POSITIVES = [
-  "spec/isodoc/blocks_spec.rb:352",
-  "spec/isodoc/footnotes_spec.rb:740",
-  "spec/isodoc/inline_spec.rb:1012",
-  "spec/isodoc/inline_spec.rb:1251",
-  "spec/isodoc/postproc_spec.rb:948",
-  "spec/isodoc/postproc_word_spec.rb:372",
-  "spec/isodoc/postproc_word_spec.rb:576",
-  "spec/isodoc/presentation_xml_numbers_override_spec.rb:2095",
-  "spec/isodoc/presentation_xml_spec.rb:1288",
-  "spec/isodoc/presentation_xml_spec.rb:1500",
-  "spec/isodoc/ref_spec.rb:906",
-  "spec/isodoc/sourcecode_spec.rb:124",
-  "spec/isodoc/sourcecode_spec.rb:610",
-  "spec/isodoc/terms_spec.rb:1445",
-  "spec/isodoc/xref_format_spec.rb:628",
-  "spec/isodoc/xref_spec.rb:315",
-].freeze
-
-ISODOC_DIR = "/Users/mulgogi/src/mn/isodoc"
-
-results = {}
-
-puts "=" * 80
-puts "TESTING ALL 16 FALSE POSITIVES"
-puts "=" * 80
-puts
-
-FALSE_POSITIVES.each_with_index do |test, idx|
-  puts "\n#{idx + 1}. Testing: #{test}"
-  puts "-" * 60
-
-  # Test with DOM
-  ENV["CANON_HTML_DIFF_ALGORITHM"] = "dom"
-  ENV["CANON_XML_DIFF_ALGORITHM"] = "dom"
-  `cd #{ISODOC_DIR} && bundle exec rspec #{test} 2>&1`
-  dom_pass = $?.success?
-
-  # Test with Semantic
-  ENV["CANON_HTML_DIFF_ALGORITHM"] = "semantic"
-  ENV["CANON_XML_DIFF_ALGORITHM"] = "semantic"
-  `cd #{ISODOC_DIR} && bundle exec rspec #{test} 2>&1`
-  semantic_pass = $?.success?
-
-  results[test] = {
-    dom: dom_pass,
-    semantic: semantic_pass,
-    false_positive: dom_pass && !semantic_pass,
-  }
-
-  status = if dom_pass && !semantic_pass
-             "❌ FALSE POSITIVE (DOM pass, Semantic fail)"
-           elsif !dom_pass && semantic_pass
-             "⚠️  FALSE NEGATIVE (DOM fail, Semantic pass)"
-           elsif dom_pass && semantic_pass
-             "✅ BOTH PASS"
-           else
-             "⏺  BOTH FAIL"
-           end
-
-  puts "   #{status}"
-end
-
-# Summary
-puts "\n#{'=' * 80}"
-puts "SUMMARY"
-puts "=" * 80
-
-false_positives = results.select { |_, r| r[:false_positive] }
-false_negatives = results.select { |_, r| !r[:dom] && r[:semantic] }
-both_pass = results.select { |_, r| r[:dom] && r[:semantic] }
-both_fail = results.select { |_, r| !r[:dom] && !r[:semantic] }
-
-puts "\nFalse Positives (DOM pass, Semantic fail): #{false_positives.count}"
-false_positives.each_key { |test| puts "  - #{test}" }
-
-puts "\nFalse Negatives (DOM fail, Semantic pass): #{false_negatives.count}"
-false_negatives.each_key { |test| puts "  - #{test}" }
-
-puts "\nBoth Pass: #{both_pass.count}"
-both_pass.each_key { |test| puts "  - #{test}" }
-
-puts "\nBoth Fail: #{both_fail.count}"
-both_fail.each_key { |test| puts "  - #{test}" }
-
-puts "\n#{'=' * 80}"
-puts "Current state: #{false_positives.count} false positives remaining"
-puts "Target: 0 false positives (achieve DOM parity)"
-puts "=" * 80

data/scripts/test_attribute_details.rb

@@ -1,61 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-require_relative "../lib/canon"
-require_relative "../lib/canon/diff_formatter"
-require_relative "../lib/canon/diff_formatter/diff_detail_formatter"
-
-# Test attribute values formatting - same attributes, different values
-html1 = '<table id="T1" class="MsoNormalTable" border="1"></table>'
-html2 = '<table id="T2" class="MsoNormalTable" border="2"></table>'
-
-puts "=" * 70
-puts "TEST 1: Multiple Attribute Values Differ (id removed, border changed)"
-puts "=" * 70
-
-result = Canon::Comparison.equivalent?(
-  html1,
-  html2,
-  match_algorithm: :semantic_tree,
-  ignore_attr_order: true,
-  verbose: true,
-)
-
-# Print semantic diff report
-if result.differences.any?
-  report = Canon::DiffFormatter::DiffDetailFormatter.format_report(
-    result.differences,
-    use_color: true,
-  )
-  puts report
-else
-  puts "No differences found!"
-end
-
-puts "\n\n"
-puts "=" * 70
-puts "TEST 2: Attribute Order Differs"
-puts "=" * 70
-
-# Test attribute order formatting
-html3 = '<table id="T1" class="MsoNormalTable" border="1"></table>'
-html4 = '<table border="1" class="MsoNormalTable" id="T1"></table>'
-
-result2 = Canon::Comparison.equivalent?(
-  html3,
-  html4,
-  match_algorithm: :semantic_tree,
-  ignore_attr_order: false, # Don't ignore order, so we see the difference
-  verbose: true,
-)
-
-# Print semantic diff report
-if result2.differences.any?
-  report2 = Canon::DiffFormatter::DiffDetailFormatter.format_report(
-    result2.differences,
-    use_color: true,
-  )
-  puts report2
-else
-  puts "No differences found!"
-end

data/scripts/test_both_algorithms.rb

@@ -1,49 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-require_relative "../lib/canon"
-require_relative "../lib/canon/diff_formatter"
-
-# Test XML content
-xml1 = <<~XML
-  <root>
-    <person>
-      <name>Alice</name>
-      <age>30</age>
-    </person>
-  </root>
-XML
-
-xml2 = <<~XML
-  <root>
-    <person>
-      <name>Bob</name>
-      <age>25</age>
-    </person>
-  </root>
-XML
-
-puts "=" * 80
-puts "Testing :both algorithm option"
-puts "=" * 80
-puts
-
-# Test with verbose: true and diff_algorithm: :both
-result = Canon::Comparison.equivalent?(
-  xml1,
-  xml2,
-  verbose: true,
-  diff_algorithm: :both,
-)
-
-puts "Result class: #{result.class}"
-puts "Result equivalent?: #{result.equivalent?}"
-puts
-
-# Create formatter
-formatter = Canon::DiffFormatter.new(use_color: true)
-
-# Format the comparison result
-output = formatter.format_comparison_result(result, xml1, xml2)
-
-puts output

data/scripts/test_both_simple.rb

@@ -1,49 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-require_relative "../lib/canon"
-
-# Test XML content
-xml1 = <<~XML
-  <root>
-    <person>
-      <name>Alice</name>
-      <age>30</age>
-    </person>
-  </root>
-XML
-
-xml2 = <<~XML
-  <root>
-    <person>
-      <name>Bob</name>
-      <age>25</age>
-    </person>
-  </root>
-XML
-
-puts "=" * 80
-puts "Testing :both algorithm option (simple test)"
-puts "=" * 80
-puts
-
-# Test with verbose: true and diff_algorithm: :both
-result = Canon::Comparison.equivalent?(
-  xml1,
-  xml2,
-  verbose: true,
-  diff_algorithm: :both,
-)
-
-puts "Result class: #{result.class}"
-puts "Result equivalent?: #{result.equivalent?}"
-puts "DOM result class: #{result.dom_result.class}"
-puts "DOM result equivalent?: #{result.dom_result.equivalent?}"
-puts "DOM result algorithm: #{result.dom_result.algorithm}"
-puts "DOM differences count: #{result.dom_result.differences.count}"
-puts
-puts "Tree result class: #{result.tree_result.class}"
-puts "Tree result equivalent?: #{result.tree_result.equivalent?}"
-puts "Tree result algorithm: #{result.tree_result.algorithm}"
-puts "Tree differences count: #{result.tree_result.differences.count}"
-puts "Tree operations count: #{result.tree_result.operations.count}"

data/scripts/test_enhanced_semantic_output.rb

@@ -1,125 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-require "bundler/setup"
-require "canon"
-
-# Test XML with differences
-xml1 = <<~XML
-  <root>
-    <section id="1">
-      <title>First Section</title>
-      <paragraph>Original text</paragraph>
-    </section>
-    <section id="2">
-      <title>Second Section</title>
-    </section>
-  </root>
-XML
-
-xml2 = <<~XML
-  <root>
-    <section id="1">
-      <title>First Section Modified</title>
-      <paragraph>Changed text</paragraph>
-    </section>
-    <section id="3">
-      <title>Third Section</title>
-    </section>
-  </root>
-XML
-
-puts "=" * 80
-puts "Testing Enhanced Semantic Tree Diff Output"
-puts "=" * 80
-puts
-
-# Test with semantic diff algorithm
-result = Canon::Comparison.equivalent?(
-  xml1,
-  xml2,
-  diff_algorithm: :semantic,
-  verbose: true,
-)
-
-puts "Algorithm: #{result.algorithm}"
-puts "Equivalent: #{result.equivalent?}"
-puts "Number of differences: #{result.differences.length}"
-puts
-puts "Detailed Operations:"
-puts "-" * 80
-
-result.differences.each_with_index do |diff, i|
-  puts "\n#{i + 1}. #{diff.dimension.to_s.upcase}"
-  puts "   Reason: #{diff.reason}"
-
-  # Access the underlying operation metadata
-  if diff.respond_to?(:node1) && diff.node1
-    puts "   Node1: #{diff.node1.inspect}"
-  end
-
-  if diff.respond_to?(:node2) && diff.node2
-    puts "   Node2: #{diff.node2.inspect}"
-  end
-
-  # Show metadata if available
-  if diff.respond_to?(:metadata)
-    puts "   Metadata: #{diff.metadata.inspect}"
-  end
-end
-
-puts
-puts "=" * 80
-puts "Testing operation metadata directly"
-puts "=" * 80
-
-# Access tree diff operations directly
-if result.respond_to?(:match_options) && result.match_options
-  ops = result.match_options[:tree_diff_operations]
-
-  if ops
-    puts "\nDirect Tree Diff Operations:"
-    puts "-" * 80
-
-    ops.each_with_index do |op, i|
-      puts "\n#{i + 1}. Operation: #{op.type.to_s.upcase}"
-
-      # Show path information
-      if op[:path]
-        puts "   Path: #{op[:path]}"
-      end
-
-      if op[:old_path] && op[:new_path]
-        puts "   Old Path: #{op[:old_path]}"
-        puts "   New Path: #{op[:new_path]}"
-      end
-
-      # Show content information
-      if op[:content]
-        puts "   Content: #{op[:content]}"
-      end
-
-      if op[:old_content] && op[:new_content]
-        puts "   Old Content: #{op[:old_content]}"
-        puts "   New Content: #{op[:new_content]}"
-      end
-
-      # Show changes detail
-      if op[:changes]
-        puts "   Changes:"
-        op[:changes].each do |key, change|
-          puts "     - #{key}: #{change[:old]} => #{change[:new]}"
-        end
-      end
-    end
-  else
-    puts "No tree_diff_operations found in match_options"
-  end
-else
-  puts "No match_options available"
-end
-
-puts
-puts "=" * 80
-puts "Test Complete"
-puts "=" * 80