canon 0.1.8 → 0.1.9

data/scripts/debug_sourcecode_124.rb

@@ -1,32 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-# Debug sourcecode_spec.rb:124 to understand the false positive pattern
-# Usage: ruby scripts/debug_sourcecode_124.rb
-
-require "bundler/setup"
-require_relative "../../src/mn/isodoc/spec/spec_helper"
-
-# Run the specific test with verbose output to capture expected/actual
-puts "=" * 80
-puts "DEBUGGING: sourcecode_spec.rb:124"
-puts "=" * 80
-
-# Run test with DOM (should pass)
-puts "\n1. Running with DOM algorithm (should PASS):"
-puts "-" * 60
-ENV["CANON_HTML_DIFF_ALGORITHM"] = "dom"
-ENV["CANON_XML_DIFF_ALGORITHM"] = "dom"
-ENV["CANON_HTML_DIFF_VERBOSE"] = "true"
-ENV["CANON_XML_DIFF_VERBOSE"] = "true"
-system("cd /Users/mulgogi/src/mn/isodoc && bundle exec rspec spec/isodoc/sourcecode_spec.rb:124 --format documentation 2>&1")
-
-puts "\n#{'=' * 80}"
-# Run test with Semantic (should fail - false positive)
-puts "\n2. Running with Semantic algorithm (should FAIL):"
-puts "-" * 60
-ENV["CANON_HTML_DIFF_ALGORITHM"] = "semantic"
-ENV["CANON_XML_DIFF_ALGORITHM"] = "semantic"
-ENV["CANON_HTML_DIFF_VERBOSE"] = "true"
-ENV["CANON_XML_DIFF_VERBOSE"] = "true"
-system("cd /Users/mulgogi/src/mn/isodoc && bundle exec rspec spec/isodoc/sourcecode_spec.rb:124 --format documentation 2>&1")

data/scripts/debug_whitespace_sensitive.rb

@@ -1,192 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-require_relative "../lib/canon"
-require "nokogiri"
-
-expected = <<~HTML
-  <pre>
-
-  </pre>
-HTML
-
-actual = <<~HTML
-  <pre>    </pre>
-HTML
-
-# Parse and inspect the trees directly
-puts "=== Tree Inspection ==="
-doc1 = Nokogiri::HTML(expected)
-doc2 = Nokogiri::HTML(actual)
-
-pre1 = doc1.at_css("pre")
-pre2 = doc2.at_css("pre")
-
-puts "Pre1 text: #{pre1.text.inspect}"
-puts "Pre1 text length: #{pre1.text.length}"
-puts "Pre1 text bytes: #{pre1.text.bytes.inspect}"
-
-puts "\nPre2 text: #{pre2.text.inspect}"
-puts "Pre2 text length: #{pre2.text.length}"
-puts "Pre2 text bytes: #{pre2.text.bytes.inspect}"
-
-# Now build trees using the adapter
-adapter = Canon::TreeDiff::Adapters::HTMLAdapter.new
-tree1 = adapter.to_tree(doc1)
-tree2 = adapter.to_tree(doc2)
-
-# Check signatures
-# tree1 structure: html -> body -> pre
-body1 = tree1.children.first
-pre1_node = body1.children.find { |c| c.label == "pre" }
-body2 = tree2.children.first
-pre2_node = body2.children.find { |c| c.label == "pre" }
-
-puts "\n=== Signatures ==="
-if pre1_node && pre2_node
-  sig1 = Canon::TreeDiff::Core::NodeSignature.for(pre1_node)
-  sig2 = Canon::TreeDiff::Core::NodeSignature.for(pre2_node)
-  puts "Pre1 label: #{pre1_node.label}, value: #{pre1_node.value.inspect}"
-  puts "Pre1 signature: #{sig1}"
-  puts "Pre2 label: #{pre2_node.label}, value: #{pre2_node.value.inspect}"
-  puts "Pre2 signature: #{sig2}"
-  puts "Signatures equal: #{sig1 == sig2}"
-else
-  puts "ERROR: Could not find <pre> nodes"
-  puts "Body1 children: #{body1.children.map(&:label)}"
-  puts "Body2 children: #{body2.children.map(&:label)}"
-end
-
-def print_tree(node, indent = 0)
-  prefix = "  " * indent
-  puts "#{prefix}<#{node.label}>"
-  puts "#{prefix}  value: #{node.value.inspect}" if node.value
-  puts "#{prefix}  attrs: #{node.attributes}" unless node.attributes.empty?
-  node.children.each { |child| print_tree(child, indent + 1) }
-end
-
-puts "\n=== Tree 1 ==="
-print_tree(tree1)
-
-puts "\n=== Tree 2 ==="
-print_tree(tree2)
-
-# Now test comparison
-puts "\n=== Comparison ==="
-
-# Test using TreeDiff directly
-require_relative "../lib/canon/tree_diff/tree_diff_integrator"
-integrator = Canon::TreeDiff::TreeDiffIntegrator.new(
-  format: :html,
-  options: {},
-)
-
-puts "\n=== Direct TreeDiff Test ==="
-diff_result = integrator.diff(doc1, doc2)
-puts "Operations count: #{diff_result[:operations].size}"
-diff_result[:operations].each_with_index do |op, idx|
-  puts "\nOperation #{idx + 1}:"
-  puts "  Type: #{op.type}"
-  puts "  Node: #{begin
-    op[:node]&.label
-  rescue StandardError
-    'N/A'
-  end}"
-  if op[:node]
-    puts "  Value: #{begin
-      op[:node]&.value.inspect
-    rescue StandardError
-      'N/A'
-    end}"
-  end
-end
-
-# Convert operations to DiffNodes
-puts "\n=== Operation Conversion ==="
-converter = Canon::TreeDiff::OperationConverter.new(
-  format: :html,
-  match_options: {},
-)
-diff_nodes = converter.convert(diff_result[:operations])
-puts "Converted DiffNodes count: #{diff_nodes.size}"
-diff_nodes.each_with_index do |dn, idx|
-  puts "\nDiffNode #{idx + 1}:"
-  puts "  Dimension: #{dn.dimension}"
-  puts "  Normative: #{dn.normative?}"
-  puts "  Reason: #{dn.reason}"
-  puts "  Node1: #{begin
-    dn.node1.inspect
-  rescue StandardError
-    'nil'
-  end}"
-  puts "  Node2: #{begin
-    dn.node2.inspect
-  rescue StandardError
-    'nil'
-  end}"
-end
-
-# Now test via Canon::Comparison
-puts "\n=== Canon::Comparison Result (with :semantic) ==="
-result = Canon::Comparison.equivalent?(
-  expected,
-  actual,
-  format: :html,
-  diff_algorithm: :semantic,
-  verbose: true,
-)
-
-puts "Result class: #{result.class}"
-puts "Equivalent: #{result.equivalent?}"
-puts "Differences count: #{result.differences.size}"
-puts "Has normative diffs: #{result.has_normative_diffs?}"
-
-puts "\n=== Trying with :semantic_tree ==="
-result2 = Canon::Comparison.equivalent?(
-  expected,
-  actual,
-  format: :html,
-  diff_algorithm: :semantic_tree,
-  verbose: true,
-)
-
-puts "Result class: #{result2.class}"
-if result2.is_a?(Canon::Comparison::ComparisonResult)
-  puts "Equivalent: #{result2.equivalent?}"
-  puts "Differences count: #{result2.differences.size}"
-else
-  puts "Result: #{result2.inspect}"
-end
-
-result.differences.each_with_index do |diff, idx|
-  puts "\n--- Diff #{idx + 1} ---"
-  puts "Dimension: #{diff.dimension}"
-  puts "Normative: #{diff.normative?}"
-  puts "Reason: #{diff.reason}"
-  if diff.node1
-    puts "Node1 type: #{diff.node1.class}"
-    puts "Node1 name: #{begin
-      diff.node1.name
-    rescue StandardError
-      'N/A'
-    end}"
-    puts "Node1 text: #{begin
-      diff.node1.text.inspect
-    rescue StandardError
-      'N/A'
-    end}"
-  end
-  if diff.node2
-    puts "Node2 type: #{diff.node2.class}"
-    puts "Node2 name: #{begin
-      diff.node2.name
-    rescue StandardError
-      'N/A'
-    end}"
-    puts "Node2 text: #{begin
-      diff.node2.text.inspect
-    rescue StandardError
-      'N/A'
-    end}"
-  end
-end

data/scripts/extract_false_positives.rb

@@ -1,138 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-# Script to extract and analyze the 16 false positive test cases
-# Usage: ruby scripts/extract_false_positives.rb
-
-require "bundler/setup"
-require "canon"
-
-# Map of test files to line numbers
-FALSE_POSITIVES = {
-  "blocks_spec.rb" => [352],
-  "footnotes_spec.rb" => [740],
-  "inline_spec.rb" => [1012, 1251],
-  "postproc_spec.rb" => [948],
-  "postproc_word_spec.rb" => [372, 576],
-  "presentation_xml_numbers_override_spec.rb" => [2095],
-  "presentation_xml_spec.rb" => [1288, 1500],
-  "ref_spec.rb" => [906],
-  "sourcecode_spec.rb" => [124, 610],
-  "terms_spec.rb" => [1445],
-  "xref_format_spec.rb" => [628],
-  "xref_spec.rb" => [315],
-}.freeze
-
-ISODOC_SPEC_DIR = File.expand_path("../../mn/isodoc/spec/isodoc", __dir__)
-
-def extract_test_context(file_path, line_number)
-  return nil unless File.exist?(file_path)
-
-  lines = File.readlines(file_path)
-
-  # Find the start of the test block (looking backward for 'it "')
-  start_line = line_number - 1
-  while start_line.positive?
-    break if /^\s*it\s+["']/.match?(lines[start_line])
-
-    start_line -= 1
-  end
-
-  # Find the end of the test block (looking forward for matching 'end')
-  end_line = line_number - 1
-  depth = 0
-  while end_line < lines.length
-    line = lines[end_line]
-    depth += 1 if /\b(do|begin)\b/.match?(line)
-    depth -= 1 if /\bend\b/.match?(line)
-    break if depth <= 0 && end_line > start_line
-
-    end_line += 1
-  end
-
-  # Extract test description
-  test_desc = lines[start_line].match(/it\s+["'](.+?)["']/)&.captures&.first || "Unknown test"
-
-  {
-    file: File.basename(file_path),
-    line: line_number,
-    description: test_desc,
-    content: lines[start_line..end_line].join,
-  }
-end
-
-def analyze_test_for_patterns(test_info)
-  content = test_info[:content]
-
-  patterns = []
-
-  # Check for various patterns
-  patterns << "whitespace_in_pre" if /<pre[^>]*>.*?<\/pre>/m.match?(content)
-  patterns << "sourcecode_element" if /sourcecode/i.match?(content)
-  patterns << "metadata_elements" if /<(bookmark|span|meta|a name=)/.match?(content)
-  patterns << "mixed_content" if /<[^>]+>[^<]*<[^>]+>/.match?(content)
-  patterns << "attribute_order" if /\s+\w+=["'][^"']*["']\s+\w+=["'][^"']*["']/.match?(content)
-  patterns << "nested_formatting" if /<(strong|em|i|b|u)[^>]*>.*?<(strong|em|i|b|u)/m.match?(content)
-
-  patterns
-end
-
-puts "=" * 80
-puts "EXTRACTING FALSE POSITIVE TEST CASES"
-puts "=" * 80
-
-all_tests = []
-pattern_summary = Hash.new(0)
-
-FALSE_POSITIVES.each do |file, line_numbers|
-  file_path = File.join(ISODOC_SPEC_DIR, file)
-
-  puts "\n#{file}:"
-
-  line_numbers.each do |line|
-    test_info = extract_test_context(file_path, line)
-
-    if test_info
-      patterns = analyze_test_for_patterns(test_info)
-      test_info[:patterns] = patterns
-      all_tests << test_info
-
-      patterns.each { |p| pattern_summary[p] += 1 }
-
-      puts "  Line #{line}: #{test_info[:description]}"
-      puts "    Patterns: #{patterns.join(', ')}" unless patterns.empty?
-    else
-      puts "  Line #{line}: ⚠️  Could not extract test"
-    end
-  end
-end
-
-puts "\n#{'=' * 80}"
-puts "PATTERN SUMMARY"
-puts "=" * 80
-
-pattern_summary.sort_by { |_, count| -count }.each do |pattern, count|
-  puts "  #{pattern}: #{count} occurrences"
-end
-
-puts "\n#{'=' * 80}"
-puts "DETAILED TEST EXTRACTION"
-puts "=" * 80
-
-# Save detailed output
-output_file = "false_positive_analysis.txt"
-File.open(output_file, "w") do |f|
-  all_tests.each_with_index do |test, i|
-    f.puts "\n#{'=' * 80}"
-    f.puts "TEST #{i + 1}: #{test[:file]}:#{test[:line]}"
-    f.puts "=" * 80
-    f.puts "Description: #{test[:description]}"
-    f.puts "Patterns: #{test[:patterns].join(', ')}"
-    f.puts "\nTest Code:"
-    f.puts "-" * 80
-    f.puts test[:content]
-  end
-end
-
-puts "\nDetailed analysis saved to: #{output_file}"
-puts "\nTotal false positives analyzed: #{all_tests.length}"

data/scripts/find_actual_false_positives.rb

@@ -1,125 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-# Find the actual false positives by comparing DOM vs semantic failures
-
-require "json"
-require "set"
-
-ISODOC_DIR = File.expand_path("../../../mn/isodoc", __dir__)
-
-def run_tests(algorithm)
-  puts "Running with CANON_ALGORITHM=#{algorithm}..."
-  output_file = "/tmp/rspec_#{algorithm}_#{Process.pid}.json"
-  cmd = "cd #{ISODOC_DIR} && CANON_ALGORITHM=#{algorithm} bundle exec rspec --format json --out #{output_file} 2>&1 >/dev/null"
-  system(cmd)
-
-  if File.exist?(output_file)
-    content = File.read(output_file)
-    File.delete(output_file)
-    begin
-      JSON.parse(content)
-    rescue JSON::ParserError => e
-      puts "Failed to parse JSON for #{algorithm}: #{e.message}"
-      puts "First 200 chars: #{content[0..200]}"
-      nil
-    end
-  else
-    puts "Output file not created for #{algorithm}"
-    nil
-  end
-end
-
-def extract_failures(results)
-  return [] unless results && results["examples"]
-
-  results["examples"].select { |ex| ex["status"] == "failed" }.map do |ex|
-    # Extract file and line from id
-    # Format: "./spec/isodoc/blocks_spec.rb[1:1:1]"
-    if ex["id"] =~ %r{\./(spec/isodoc/[^\[]+)\[}
-      file = $1
-      line = ex["line_number"]
-      "#{file}:#{line}"
-    end
-  end.compact
-end
-
-puts "=" * 80
-puts "FINDING ACTUAL FALSE POSITIVES"
-puts "=" * 80
-puts
-
-# Run with both algorithms
-dom_results = run_tests("dom")
-semantic_results = run_tests("semantic")
-
-if dom_results.nil? || semantic_results.nil?
-  puts "ERROR: Failed to get test results"
-  exit 1
-end
-
-# Extract failure lists
-dom_failures = Set.new(extract_failures(dom_results))
-semantic_failures = Set.new(extract_failures(semantic_results))
-
-puts "DOM failures:      #{dom_failures.size}"
-puts "Semantic failures: #{semantic_failures.size}"
-puts
-
-# Find false positives (pass with DOM, fail with semantic)
-false_positives = semantic_failures - dom_failures
-
-puts "=" * 80
-puts "FALSE POSITIVES (#{false_positives.size})"
-puts "Tests that PASS with DOM but FAIL with semantic:"
-puts "=" * 80
-
-if false_positives.empty?
-  puts "✅ NO FALSE POSITIVES FOUND!"
-  puts "DOM and semantic algorithms have perfect parity!"
-else
-  false_positives.sort.each_with_index do |test, idx|
-    puts "#{idx + 1}. #{test}"
-  end
-end
-
-# Find false negatives (fail with DOM, pass with semantic)
-false_negatives = dom_failures - semantic_failures
-
-puts
-puts "=" * 80
-puts "FALSE NEGATIVES (#{false_negatives.size})"
-puts "Tests that FAIL with DOM but PASS with semantic:"
-puts "=" * 80
-
-if false_negatives.empty?
-  puts "✅ NO FALSE NEGATIVES FOUND!"
-else
-  false_negatives.sort.each_with_index do |test, idx|
-    puts "#{idx + 1}. #{test}"
-  end
-end
-
-# Common failures
-common_failures = dom_failures & semantic_failures
-
-puts
-puts "=" * 80
-puts "COMMON FAILURES (#{common_failures.size})"
-puts "Tests that FAIL with BOTH algorithms:"
-puts "=" * 80
-puts "#{common_failures.size} tests fail with both algorithms"
-
-# Summary
-puts
-puts "=" * 80
-puts "SUMMARY"
-puts "=" * 80
-puts "Total tests:        #{dom_results['examples'].size}"
-puts "DOM failures:       #{dom_failures.size}"
-puts "Semantic failures:  #{semantic_failures.size}"
-puts "Common failures:    #{common_failures.size}"
-puts "False positives:    #{false_positives.size} (semantic fails, DOM passes)"
-puts "False negatives:    #{false_negatives.size} (DOM fails, semantic passes)"
-puts "Gap:                #{(semantic_failures.size - dom_failures.size).abs}"
-puts "=" * 80

data/scripts/investigate_all_false_positives.rb

@@ -1,161 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-# Systematically investigate all false positive failures
-# to identify patterns in why semantic fails but DOM passes
-
-require "fileutils"
-
-FALSE_POSITIVES = [
-  "blocks_spec.rb:352",
-  "footnotes_spec.rb:740",
-  "inline_spec.rb:1012",
-  "inline_spec.rb:1251",
-  "postproc_spec.rb:948",
-  "postproc_word_spec.rb:372",
-  "postproc_word_spec.rb:576",
-  "presentation_xml_numbers_override_spec.rb:2095",
-  "presentation_xml_spec.rb:1288",
-  "presentation_xml_spec.rb:1500",
-  "ref_spec.rb:906",
-  "sourcecode_spec.rb:124",
-  "sourcecode_spec.rb:610",
-  "terms_spec.rb:1445",
-  "xref_format_spec.rb:628",
-  "xref_spec.rb:315",
-].freeze
-
-ISODOC_PATH = "/Users/mulgogi/src/mn/isodoc"
-
-def run_test(spec_file, line, algorithm)
-  cmd = "cd #{ISODOC_PATH} && CANON_ALGORITHM=#{algorithm} bundle exec rspec ./spec/isodoc/#{spec_file}:#{line} 2>&1"
-  output = `#{cmd}`
-  {
-    passed: $?.success?,
-    output: output,
-  }
-end
-
-def extract_diff_type(output)
-  # Look for dimension in diff report
-  if output =~ /Dimension:[^\n]*\n[^\n]*Location:[^\n]*([^\n]+)/
-    location = $1.strip
-  end
-
-  if output =~ /Dimension:\s*([^\n]+)/
-    dimension = $1.strip
-  end
-
-  # Look for changes description
-  changes = []
-  output.scan(/✨ Changes:\s*([^\n]+)/) do |match|
-    changes << match[0].strip
-  end
-
-  {
-    dimension: dimension,
-    location: location,
-    changes: changes,
-  }
-end
-
-def analyze_false_positive(fp)
-  file, line = fp.split(":")
-  puts "\n#{'=' * 80}"
-  puts "Analyzing: #{fp}"
-  puts "=" * 80
-
-  # Run with both algorithms
-  puts "\nRunning with DOM algorithm..."
-  dom_result = run_test(file, line, "dom")
-
-  puts "Running with SEMANTIC algorithm..."
-  semantic_result = run_test(file, line, "semantic")
-
-  # Verify it's actually a false positive
-  unless dom_result[:passed] && !semantic_result[:passed]
-    puts "⚠️  WARNING: Not a false positive!"
-    puts "  DOM passed: #{dom_result[:passed]}"
-    puts "  Semantic passed: #{semantic_result[:passed]}"
-    return nil
-  end
-
-  puts "✓ Confirmed false positive (DOM passes, Semantic fails)"
-
-  # Extract diff details from semantic output
-  diff_info = extract_diff_type(semantic_result[:output])
-
-  puts "\nDiff Details:"
-  puts "  Dimension: #{diff_info[:dimension] || 'unknown'}"
-  puts "  Location: #{diff_info[:location] || 'unknown'}"
-  puts "  Changes: #{diff_info[:changes].join(', ')}" unless diff_info[:changes].empty?
-
-  # Save full output for detailed analysis
-  output_dir = "/tmp/false_positives"
-  FileUtils.mkdir_p(output_dir)
-
-  File.write("#{output_dir}/#{file.gsub('.rb', '')}_#{line}.txt",
-             semantic_result[:output])
-  puts "\nFull output saved to: #{output_dir}/#{file.gsub('.rb',
-                                                          '')}_#{line}.txt"
-
-  diff_info.merge(spec: fp, file: file, line: line)
-end
-
-def main
-  puts "Investigating #{FALSE_POSITIVES.size} false positives..."
-  puts "This will take several minutes..."
-
-  results = []
-
-  FALSE_POSITIVES.each_with_index do |fp, idx|
-    puts "\n[#{idx + 1}/#{FALSE_POSITIVES.size}]"
-    result = analyze_false_positive(fp)
-    results << result if result
-    sleep 0.5 # Brief pause between tests
-  end
-
-  # Summarize patterns
-  puts "\n#{'=' * 80}"
-  puts "PATTERN ANALYSIS"
-  puts "=" * 80
-
-  puts "\nBy Dimension:"
-  dimension_groups = results.compact.group_by { |r| r[:dimension] }
-  dimension_groups.each do |dim, group|
-    puts "  #{dim}: #{group.size} cases"
-    group.each { |r| puts "    - #{r[:spec]}" }
-  end
-
-  puts "\nBy Changes:"
-  changes_groups = results.compact.group_by { |r| r[:changes].join(", ") }
-  changes_groups.each do |change, group|
-    puts "  #{change}: #{group.size} cases"
-    group.each { |r| puts "    - #{r[:spec]}" }
-  end
-
-  # Save summary
-  summary_file = "/tmp/false_positive_patterns.txt"
-  File.open(summary_file, "w") do |f|
-    f.puts "FALSE POSITIVE PATTERN ANALYSIS"
-    f.puts "=" * 80
-    f.puts "\nTotal: #{results.compact.size} false positives analyzed"
-
-    f.puts "\n\nBy Dimension:"
-    dimension_groups.each do |dim, group|
-      f.puts "  #{dim}: #{group.size}"
-      group.each { |r| f.puts "    #{r[:spec]}" }
-    end
-
-    f.puts "\n\nBy Changes:"
-    changes_groups.each do |change, group|
-      f.puts "  #{change}: #{group.size}"
-      group.each { |r| f.puts "    #{r[:spec]}" }
-    end
-  end
-
-  puts "\nSummary saved to: #{summary_file}"
-  puts "\nDetailed outputs in: /tmp/false_positives/"
-end
-
-main if __FILE__ == $PROGRAM_NAME