canon 0.1.8 → 0.1.9

data/scripts/investigate_false_positives.rb

@@ -1,227 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-# Script to systematically investigate false positives
-# where semantic algorithm fails but DOM algorithm passes
-
-require "fileutils"
-require "json"
-
-# False positives to investigate
-FALSE_POSITIVES = [
-  { file: "blocks_spec.rb", line: 352 },
-  { file: "footnotes_spec.rb", line: 740 },
-  { file: "inline_spec.rb", line: 1012 },
-  { file: "inline_spec.rb", line: 1251 },
-  { file: "postproc_spec.rb", line: 948 },
-  { file: "postproc_word_spec.rb", line: 372 },
-  { file: "postproc_word_spec.rb", line: 576 },
-  { file: "presentation_xml_numbers_override_spec.rb", line: 2095 },
-  { file: "presentation_xml_spec.rb", line: 1288 },
-  { file: "presentation_xml_spec.rb", line: 1500 },
-  { file: "ref_spec.rb", line: 906 },
-  { file: "sourcecode_spec.rb", line: 124 },
-  { file: "sourcecode_spec.rb", line: 610 },
-  { file: "terms_spec.rb", line: 1445 },
-  { file: "xref_format_spec.rb", line: 628 },
-  { file: "xref_spec.rb", line: 315 },
-].freeze
-
-ISODOC_SPEC_DIR = File.expand_path("../../../mn/isodoc/spec/isodoc", __dir__)
-
-class FalsePositiveInvestigator
-  attr_reader :results
-
-  def initialize
-    @results = []
-  end
-
-  def investigate_all
-    puts "=" * 80
-    puts "INVESTIGATING 16 FALSE POSITIVES"
-    puts "=" * 80
-    puts
-
-    FALSE_POSITIVES.each_with_index do |test, idx|
-      puts "\n#{idx + 1}/#{FALSE_POSITIVES.size}: #{test[:file]}:#{test[:line]}"
-      puts "-" * 80
-
-      result = investigate_test(test)
-      @results << result
-
-      display_result(result)
-    end
-
-    summarize_results
-  end
-
-  def investigate_test(test)
-    file_path = File.join(ISODOC_SPEC_DIR, test[:file])
-
-    unless File.exist?(file_path)
-      return {
-        test: test,
-        error: "File not found: #{file_path}",
-        dom_passes: nil,
-        semantic_passes: nil,
-      }
-    end
-
-    result = {
-      test: test,
-      file_path: file_path,
-      dom_passes: nil,
-      semantic_passes: nil,
-      semantic_output: nil,
-      error: nil,
-    }
-
-    # Test with DOM algorithm
-    puts "  Testing with DOM algorithm..."
-    dom_output = run_test(file_path, test[:line], "dom")
-    result[:dom_passes] = dom_output[:success]
-    result[:dom_output] = dom_output[:output]
-
-    # Test with semantic algorithm
-    puts "  Testing with semantic algorithm..."
-    semantic_output = run_test(file_path, test[:line], "semantic")
-    result[:semantic_passes] = semantic_output[:success]
-    result[:semantic_output] = semantic_output[:output]
-
-    result
-  rescue StandardError => e
-    {
-      test: test,
-      error: "Exception: #{e.message}",
-      dom_passes: nil,
-      semantic_passes: nil,
-    }
-  end
-
-  def run_test(file_path, line, algorithm)
-    cmd = "cd #{ISODOC_SPEC_DIR}/.. && CANON_ALGORITHM=#{algorithm} bundle exec rspec #{file_path}:#{line} 2>&1"
-    output = `#{cmd}`
-    success = $?.success?
-
-    {
-      success: success,
-      output: output,
-      exit_code: $?.exitstatus,
-    }
-  end
-
-  def display_result(result)
-    if result[:error]
-      puts "  ❌ ERROR: #{result[:error]}"
-      return
-    end
-
-    dom_status = result[:dom_passes] ? "✅ PASS" : "❌ FAIL"
-    sem_status = result[:semantic_passes] ? "✅ PASS" : "❌ FAIL"
-
-    puts "  DOM:      #{dom_status}"
-    puts "  Semantic: #{sem_status}"
-
-    if result[:dom_passes] && !result[:semantic_passes]
-      puts "  ⚠️  CONFIRMED FALSE POSITIVE"
-      analyze_failure(result)
-    elsif !result[:dom_passes] && result[:semantic_passes]
-      puts "  ⚠️  UNEXPECTED: DOM fails but semantic passes!"
-    elsif !result[:dom_passes] && !result[:semantic_passes]
-      puts "  ℹ️  Both algorithms fail (not a false positive)"
-    else
-      puts "  ✅ Both algorithms pass (false positive may be fixed)"
-    end
-  end
-
-  def analyze_failure(result)
-    output = result[:semantic_output]
-
-    # Look for diff patterns
-    if output.include?("Expected XML to be equivalent")
-      puts "  📋 Failure type: XML equivalence check"
-    elsif output.include?("Expected HTML to be equivalent")
-      puts "  📋 Failure type: HTML equivalence check"
-    end
-
-    # Extract key diff lines
-    diff_lines = output.lines.select { |l| l.match?(/^\s*[+-]/) }.take(10)
-    if diff_lines.any?
-      puts "  📊 Sample diff:"
-      diff_lines.each { |l| puts "    #{l.strip}" }
-    end
-
-    # Look for specific patterns
-    if output.include?("whitespace")
-      puts "  🔍 Involves: whitespace differences"
-    end
-    if output.include?("attribute")
-      puts "  🔍 Involves: attribute differences"
-    end
-    if output.include?("text content")
-      puts "  🔍 Involves: text content differences"
-    end
-  end
-
-  def summarize_results
-    puts "\n#{'=' * 80}"
-    puts "SUMMARY"
-    puts "=" * 80
-
-    confirmed_fps = @results.count do |r|
-      r[:dom_passes] && !r[:semantic_passes]
-    end
-    fixed = @results.count { |r| r[:dom_passes] && r[:semantic_passes] }
-    errors = @results.count { |r| r[:error] }
-    both_fail = @results.count { |r| !r[:dom_passes] && !r[:semantic_passes] }
-
-    puts "Confirmed false positives: #{confirmed_fps}/16"
-    puts "Already fixed:             #{fixed}/16"
-    puts "Both fail (not FP):        #{both_fail}/16"
-    puts "Errors:                    #{errors}/16"
-    puts
-
-    if confirmed_fps.positive?
-      puts "FALSE POSITIVES TO FIX:"
-      @results.each do |r|
-        next unless r[:dom_passes] && !r[:semantic_passes]
-
-        puts "  - #{r[:test][:file]}:#{r[:test][:line]}"
-      end
-    end
-
-    puts "\n#{'=' * 80}"
-  end
-
-  def save_detailed_output(output_dir = "tmp/false_positive_investigation")
-    FileUtils.mkdir_p(output_dir)
-
-    @results.each_with_index do |result, idx|
-      next if result[:error]
-
-      test = result[:test]
-      filename = "#{idx + 1}_#{test[:file].gsub('.rb', '')}_#{test[:line]}.txt"
-      filepath = File.join(output_dir, filename)
-
-      File.write(filepath, <<~OUTPUT)
-        Test: #{test[:file]}:#{test[:line]}
-        DOM passes: #{result[:dom_passes]}
-        Semantic passes: #{result[:semantic_passes]}
-
-        ========================================
-        SEMANTIC OUTPUT:
-        ========================================
-        #{result[:semantic_output]}
-      OUTPUT
-    end
-
-    puts "\nDetailed output saved to: #{output_dir}/"
-  end
-end
-
-# Run investigation
-investigator = FalsePositiveInvestigator.new
-investigator.investigate_all
-investigator.save_detailed_output
-
-puts "\nInvestigation complete!"

data/scripts/investigate_false_positives_batch.rb

@@ -1,163 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-# Systematically investigate false positive failures
-# Usage: ruby scripts/investigate_false_positives_batch.rb <spec_file:line> [<spec_file:line> ...]
-
-require "bundler/setup"
-require "fileutils"
-
-# False positive test cases from XMLNS_FIX_VALIDATION.md
-FALSE_POSITIVES = [
-  "blocks_spec.rb:352",
-  "footnotes_spec.rb:740",
-  "inline_spec.rb:1012",
-  "inline_spec.rb:1251",
-  "postproc_spec.rb:948",
-  "postproc_word_spec.rb:372",
-  "postproc_word_spec.rb:576",
-  "presentation_xml_numbers_override_spec.rb:2095",
-  "presentation_xml_spec.rb:1288",
-  "presentation_xml_spec.rb:1500",
-  "ref_spec.rb:906",
-  "sourcecode_spec.rb:124",
-  "sourcecode_spec.rb:610",
-  "terms_spec.rb:1445",
-  "xref_format_spec.rb:628",
-  "xref_spec.rb:315",
-].freeze
-
-def run_test(spec_file, line, algorithm)
-  spec_path = File.expand_path("../../../mn/isodoc/spec/isodoc/#{spec_file}",
-                               __dir__)
-
-  unless File.exist?(spec_path)
-    puts "  ❌ File not found: #{spec_path}"
-    return nil
-  end
-
-  # Run with specific algorithm
-  { "CANON_ALGORITHM" => algorithm }
-  cmd = "cd #{File.dirname(spec_path)} && bundle exec rspec #{spec_path}:#{line} 2>&1"
-
-  output = `#{cmd}`
-  success = $?.success?
-
-  { success: success, output: output }
-end
-
-def analyze_test(test_case)
-  spec_file, line = test_case.split(":")
-
-  puts "\n#{'=' * 80}"
-  puts "ANALYZING: #{test_case}"
-  puts "=" * 80
-
-  # Run with DOM algorithm
-  puts "\n1. Testing with DOM algorithm..."
-  dom_result = run_test(spec_file, line, "dom")
-  return unless dom_result
-
-  dom_pass = dom_result[:success]
-  puts "   Result: #{dom_pass ? '✅ PASS' : '❌ FAIL'}"
-
-  # Run with Semantic algorithm
-  puts "\n2. Testing with Semantic algorithm..."
-  semantic_result = run_test(spec_file, line, "semantic")
-  return unless semantic_result
-
-  semantic_pass = semantic_result[:success]
-  puts "   Result: #{semantic_pass ? '✅ PASS' : '❌ FAIL'}"
-
-  # Analysis
-  puts "\n#{'-' * 80}"
-  puts "ANALYSIS:"
-  puts "-" * 80
-
-  if dom_pass && !semantic_pass
-    puts "✅ CONFIRMED FALSE POSITIVE: DOM passes, Semantic fails"
-    puts "\nThis test should be investigated to understand why semantic is too strict."
-
-    # Extract failure details from semantic output
-    if semantic_result[:output] =~ /Failure\/Error:(.+?)(?=\n\n|\z)/m
-      failure_section = $1
-      puts "\nFailure details:"
-      puts failure_section.lines.take(20).join
-    end
-
-    :false_positive
-  elsif !dom_pass && semantic_pass
-    puts "⚠️  UNEXPECTED: This was listed as false positive but DOM fails, Semantic passes"
-    puts "This is actually a false NEGATIVE, not a false positive!"
-    :false_negative
-  elsif dom_pass && semantic_pass
-    puts "✅ BOTH PASS: This is no longer a false positive!"
-    :fixed
-  else
-    puts "❌ BOTH FAIL: This is a common failure, not a false positive"
-    :common_failure
-  end
-end
-
-def main
-  # Get test cases from arguments or use all false positives
-  test_cases = if ARGV.empty?
-                 FALSE_POSITIVES
-               else
-                 ARGV
-               end
-
-  puts "Investigating #{test_cases.size} false positive test cases..."
-
-  results = {
-    false_positive: [],
-    false_negative: [],
-    fixed: [],
-    common_failure: [],
-    error: [],
-  }
-
-  test_cases.each do |test_case|
-    result = analyze_test(test_case)
-    results[result || :error] << test_case
-  end
-
-  # Summary
-  puts "\n#{'=' * 80}"
-  puts "SUMMARY"
-  puts "=" * 80
-
-  puts "\n✅ Confirmed False Positives (need fixing): #{results[:false_positive].size}"
-  results[:false_positive].each { |tc| puts "   - #{tc}" }
-
-  puts "\n🎉 Already Fixed: #{results[:fixed].size}"
-  results[:fixed].each { |tc| puts "   - #{tc}" }
-
-  puts "\n⚠️  Misclassified (actually false negatives): #{results[:false_negative].size}"
-  results[:false_negative].each { |tc| puts "   - #{tc}" }
-
-  puts "\n❌ Common Failures: #{results[:common_failure].size}"
-  results[:common_failure].each { |tc| puts "   - #{tc}" }
-
-  puts "\n💥 Errors: #{results[:error].size}"
-  results[:error].each { |tc| puts "   - #{tc}" }
-
-  # Save detailed results
-  output_file = "/tmp/false_positive_investigation.txt"
-  File.open(output_file, "w") do |f|
-    f.puts "FALSE POSITIVE INVESTIGATION RESULTS"
-    f.puts "=" * 80
-    f.puts "\nConfirmed False Positives (#{results[:false_positive].size}):"
-    results[:false_positive].each { |tc| f.puts tc }
-    f.puts "\nAlready Fixed (#{results[:fixed].size}):"
-    results[:fixed].each { |tc| f.puts tc }
-    f.puts "\nMisclassified (#{results[:false_negative].size}):"
-    results[:false_negative].each { |tc| f.puts tc }
-    f.puts "\nCommon Failures (#{results[:common_failure].size}):"
-    results[:common_failure].each { |tc| f.puts tc }
-  end
-
-  puts "\nDetailed results saved to: #{output_file}"
-end
-
-main if __FILE__ == $PROGRAM_NAME

data/scripts/investigate_mixed_content.rb

@@ -1,125 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-require "bundler/setup"
-require "canon"
-require "nokogiri"
-
-# Test mixed content element extraction
-def test_mixed_content_extraction
-  puts "=" * 80
-  puts "Testing Mixed Content Text Extraction"
-  puts "=" * 80
-
-  # Create test XML with mixed content
-  xml_str = <<~XML
-    <root>
-      <formattedAddress>123 Main St<br/>Springfield, IL<br/>62701</formattedAddress>
-      <normalText>Just plain text</normalText>
-      <withSpaces>  Text with   spaces  </withSpaces>
-      <withNewlines>Text
-      with
-      newlines</withNewlines>
-    </root>
-  XML
-
-  doc = Nokogiri::XML(xml_str)
-
-  # Test each element
-  doc.root.element_children.each do |elem|
-    puts "\nElement: <#{elem.name}>"
-    puts "  Content: #{elem.content.inspect}"
-
-    # Extract text nodes
-    text_nodes = elem.children.select(&:text?)
-    puts "  Text nodes count: #{text_nodes.size}"
-    text_nodes.each_with_index do |node, i|
-      puts "    [#{i}]: #{node.text.inspect}"
-    end
-
-    # Join text
-    joined = text_nodes.map(&:text).join
-    puts "  Joined text: #{joined.inspect}"
-
-    # Show normalization
-    normalized = joined.gsub(/\s+/, " ").strip
-    puts "  Normalized: #{normalized.inspect}"
-
-    # Element children
-    elem_children = elem.element_children
-    puts "  Element children: #{elem_children.map(&:name).inspect}"
-  end
-end
-
-# Test with Canon adapter
-def test_with_adapter
-  puts "\n#{'=' * 80}"
-  puts "Testing with Canon XML Adapter"
-  puts "=" * 80
-
-  xml_str = <<~XML
-    <root>
-      <formattedAddress>123 Main St<br/>Springfield, IL<br/>62701</formattedAddress>
-    </root>
-  XML
-
-  doc = Nokogiri::XML(xml_str)
-
-  adapter = Canon::TreeDiff::Adapters::XMLAdapter.new
-  tree = adapter.to_tree(doc)
-
-  # Find the formattedAddress node
-  address_node = tree.children.first
-
-  puts "\nTreeNode for formattedAddress:"
-  puts "  Label: #{address_node.label}"
-  puts "  Value: #{address_node.value.inspect}"
-  puts "  Children count: #{address_node.children.size}"
-  address_node.children.each do |child|
-    puts "    Child: #{child.label} = #{child.value.inspect}"
-  end
-end
-
-# Test normalization in operation detector
-def test_normalization_comparison
-  puts "\n#{'=' * 80}"
-  puts "Testing Normalization in Comparison"
-  puts "=" * 80
-
-  # Two versions with different whitespace in mixed content
-  xml1 = <<~XML
-    <root>
-      <address>123 Main St<br/>Springfield, IL<br/>62701</address>
-    </root>
-  XML
-
-  xml2 = <<~XML
-    <root>
-      <address>123 Main St<br/>Springfield,  IL<br/>62701</address>
-    </root>
-  XML
-
-  # Compare with whitespace_sensitive: false
-  result = Canon.semantic_tree_diff(xml1, xml2,
-                                    whitespace_sensitive: false,
-                                    verbose: true)
-
-  puts "\nComparison result:"
-  puts "  Identical: #{result.identical?}"
-  puts "  Normative differences: #{result.normative_differences?}"
-  puts "  Informative differences: #{result.informative_differences?}"
-
-  if result.operations.any?
-    puts "\nOperations:"
-    result.operations.each do |op|
-      puts "  #{op.type}: #{op.path} - #{op.classification}"
-      puts "    Old: #{op.old_value.inspect}" if op.old_value
-      puts "    New: #{op.new_value.inspect}" if op.new_value
-    end
-  end
-end
-
-# Run all tests
-test_mixed_content_extraction
-test_with_adapter
-test_normalization_comparison