canon 0.1.8 → 0.1.9

data/scripts/analyze_current_state.rb

@@ -1,85 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-# Parse test results from a log file
-def parse_failures(log_file)
-  lines = File.readlines(log_file)
-
-  failures = []
-  lines.each do |line|
-    # Strip ANSI color codes first
-    clean_line = line.gsub(/\e\[\d+m/, "")
-
-    # Match rspec failure format
-    if clean_line =~ /^rspec (\.\/spec\/\S+:\d+)/
-      failures << $1
-    end
-  end
-
-  failures
-end
-
-# Main
-semantic_log = "/tmp/semantic_after_mixed_content_fix.log"
-dom_log = "/tmp/dom_baseline.log"
-
-puts "=" * 80
-puts "Current Semantic Tree Algorithm Status"
-puts "=" * 80
-
-if File.exist?(semantic_log)
-  semantic_failures = parse_failures(semantic_log)
-  puts "\nSemantic failures: #{semantic_failures.size}"
-
-  if File.exist?(dom_log)
-    dom_failures = parse_failures(dom_log)
-    puts "DOM failures: #{dom_failures.size}"
-
-    # Calculate differences
-    false_positives = semantic_failures - dom_failures
-    false_negatives = dom_failures - semantic_failures
-    common = semantic_failures & dom_failures
-
-    puts "\n#{'=' * 80}"
-    puts "Comparison with DOM Baseline"
-    puts "=" * 80
-    puts "False Positives (semantic fails, DOM passes): #{false_positives.size}"
-    puts "False Negatives (semantic passes, DOM fails): #{false_negatives.size}"
-    puts "Common failures (both fail): #{common.size}"
-
-    if false_positives.any?
-      puts "\n#{'-' * 80}"
-      puts "FALSE POSITIVES (Need to fix - #{false_positives.size}):"
-      puts "-" * 80
-      false_positives.sort.each { |f| puts "  #{f}" }
-    end
-
-    if false_negatives.any?
-      puts "\n#{'-' * 80}"
-      puts "FALSE NEGATIVES (Investigate - #{false_negatives.size}):"
-      puts "-" * 80
-      false_negatives.sort.each { |f| puts "  #{f}" }
-    end
-
-    # Progress tracking
-    puts "\n#{'=' * 80}"
-    puts "Progress Tracking"
-    puts "=" * 80
-    puts "Initial state:     62 failures (29 FP, 5 FN, 33 common)"
-    puts "After metadata:    56 failures (23 FP, 5 FN, 33 common)"
-    puts "After mixed content: #{semantic_failures.size} failures (#{false_positives.size} FP, #{false_negatives.size} FN, #{common.size} common)"
-    puts "Target (DOM parity): #{dom_failures.size} failures (0 FP, 0 FN, #{dom_failures.size} common)"
-
-    improvement = 56 - semantic_failures.size
-    remaining = semantic_failures.size - dom_failures.size
-    puts "\nImprovement: #{improvement} tests fixed"
-    puts "Remaining gap: #{remaining} tests"
-
-  else
-    puts "\nWarning: DOM baseline not found at #{dom_log}"
-    puts "Semantic failures:"
-    semantic_failures.sort.each { |f| puts "  #{f}" }
-  end
-else
-  puts "Error: Semantic log not found at #{semantic_log}"
-end

data/scripts/analyze_false_positives.rb

@@ -1,114 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-# Analyze false positive patterns to identify systematic issues
-# Usage: ruby scripts/analyze_false_positives.rb /tmp/semantic_failures_final.txt
-
-require "json"
-
-# Parse false positives from comparison results
-def parse_false_positives
-  file = "/tmp/failure_comparison.txt"
-  false_positives = []
-  in_section = false
-
-  File.readlines(file).each do |line|
-    if line.include?("FALSE POSITIVES")
-      in_section = true
-      next
-    elsif line.include?("FALSE NEGATIVES")
-      break
-    end
-
-    next unless in_section
-    next if line.strip.empty?
-
-    # Parse line like "blocks_spec.rb:352"
-    if line =~ /^\s*([a-z_]+_spec\.rb):(\d+)/
-      false_positives << { file: $1, line: $2.to_i }
-    end
-  end
-
-  false_positives
-end
-
-# Run a specific test to capture its output
-def run_test(spec_file, line_num)
-  cmd = "cd /Users/mulgogi/src/mn/isodoc && CANON_ALGORITHM=semantic bundle exec rspec ./spec/isodoc/#{spec_file}:#{line_num} 2>&1"
-  output = `#{cmd}`
-
-  {
-    spec: "#{spec_file}:#{line_num}",
-    output: output,
-    has_diff: output.include?("expected"),
-    diff_preview: extract_diff_preview(output),
-  }
-end
-
-def extract_diff_preview(output)
-  lines = output.lines
-  diff_start = lines.index { |l| l.include?("expected") || l.include?("Diff:") }
-  return nil unless diff_start
-
-  # Get 10 lines after the diff marker
-  preview = lines[diff_start, 10].join
-  preview.length > 500 ? "#{preview[0..500]}..." : preview
-end
-
-def main
-  puts "Analyzing false positive patterns..."
-  puts "=" * 80
-
-  false_positives = parse_false_positives
-  puts "\nFound #{false_positives.size} false positives to analyze"
-
-  # Group by spec file
-  by_file = false_positives.group_by { |fp| fp[:file] }
-
-  puts "\nBreakdown by file:"
-  by_file.sort_by { |_, v| -v.size }.each do |file, items|
-    puts "  #{file}: #{items.size} failures"
-  end
-
-  # Sample a few from each top category
-  puts "\n#{'=' * 80}"
-  puts "SAMPLING TOP FAILURES FOR PATTERN ANALYSIS"
-  puts "=" * 80
-
-  samples = []
-
-  # Take first 2 from each top category
-  by_file.sort_by { |_, v| -v.size }.take(5).each_value do |items|
-    items.take(2).each do |item|
-      puts "\n#{'-' * 80}"
-      puts "Testing: #{item[:file]}:#{item[:line]}"
-      puts "-" * 80
-
-      result = run_test(item[:file], item[:line])
-      samples << result
-
-      if result[:has_diff]
-        puts "\nDiff Preview:"
-        puts result[:diff_preview]
-      else
-        puts "\nNo diff found in output"
-      end
-    end
-  end
-
-  # Save results
-  output = {
-    total_false_positives: false_positives.size,
-    by_file: by_file.transform_values(&:size),
-    samples: samples.map do |s|
-      { spec: s[:spec], diff_preview: s[:diff_preview] }
-    end,
-  }
-
-  File.write("/tmp/false_positive_analysis.json", JSON.pretty_generate(output))
-  puts "\n#{'=' * 80}"
-  puts "Analysis saved to /tmp/false_positive_analysis.json"
-  puts "=" * 80
-end
-
-main if __FILE__ == $PROGRAM_NAME

data/scripts/analyze_remaining_failures.rb

@@ -1,105 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-# Compare semantic tree failures vs DOM diff failures
-# to identify remaining false positives
-
-# DOM diff failures (39 - the baseline/correct)
-dom_failures = [
-  "blocks_notes_spec.rb:494",
-  "blocks_provisions_spec.rb:4",
-  "blocks_spec.rb:4",
-  "blocks_spec.rb:1062",
-  "cleanup_spec.rb:180",
-  "cleanup_spec.rb:347",
-  "figures_spec.rb:5",
-  "figures_spec.rb:1662",
-  "figures_spec.rb:1764",
-  "figures_spec.rb:1815",
-  "footnotes_spec.rb:5",
-  "i18n_spec.rb:1644",
-  "inline_spec.rb:610",
-  "inline_spec.rb:726",
-  "inline_spec.rb:2114",
-  "lists_spec.rb:4",
-  "lists_spec.rb:817",
-  "postproc_spec.rb:1010",
-  "postproc_spec.rb:1084",
-  "postproc_word_spec.rb:89",
-  "presentation_xml_metadata_spec.rb:75",
-  "presentation_xml_spec.rb:454",
-  "ref_identifier_spec.rb:446",
-  "ref_identifier_spec.rb:602",
-  "ref_identifier_spec.rb:671",
-  "ref_identifier_spec.rb:872",
-  "ref_spec.rb:4",
-  "ref_spec.rb:1511",
-  "ref_spec.rb:1705",
-  "section_spec.rb:4",
-  "section_spec.rb:736",
-  "section_title_spec.rb:4",
-  "sourcecode_spec.rb:4",
-  "sourcecode_spec.rb:838",
-  "table_debug_spec.rb:4",
-  "table_spec.rb:4",
-  "table_spec.rb:811",
-  "table_spec.rb:1683",
-  "table_spec.rb:1906",
-].to_set
-
-# Read semantic failures from file
-semantic_failures = File.readlines("/tmp/semantic_failures.txt").map do |line|
-  # Extract spec file and line number from rspec output
-  if line =~ /rspec \.\/spec\/isodoc\/(.+?)# /
-    $1.strip
-  end
-end.compact.to_set
-
-puts "=" * 80
-puts "SEMANTIC TREE ALGORITHM - ANALYSIS AFTER FIX"
-puts "=" * 80
-puts
-puts "Statistics:"
-puts "  DOM diff failures (baseline):     #{dom_failures.size}"
-puts "  Semantic tree failures (current):  #{semantic_failures.size}"
-puts "  False positives (semantic only):   #{(semantic_failures - dom_failures).size}"
-puts "  False negatives (DOM only):        #{(dom_failures - semantic_failures).size}"
-puts
-
-# False positives (in semantic but not in DOM)
-false_positives = semantic_failures - dom_failures
-if false_positives.any?
-  puts "FALSE POSITIVES (#{false_positives.size} remaining):"
-  puts "-" * 80
-  false_positives.sort.each do |failure|
-    puts "  • #{failure}"
-  end
-  puts
-end
-
-# False negatives (in DOM but not in semantic)
-false_negatives = dom_failures - semantic_failures
-if false_negatives.any?
-  puts "FALSE NEGATIVES (#{false_negatives.size} tests):"
-  puts "-" * 80
-  false_negatives.sort.each do |failure|
-    puts "  • #{failure}"
-  end
-  puts
-end
-
-# Real failures (both agree)
-real_failures = dom_failures & semantic_failures
-puts "REAL FAILURES (#{real_failures.size} tests - both algorithms agree):"
-puts "-" * 80
-real_failures.sort.each do |failure|
-  puts "  • #{failure}"
-end
-puts
-
-puts "=" * 80
-puts "SUMMARY:"
-puts "  ✅ Fixed false positives: #{46 - false_positives.size} tests"
-puts "  ⚠️  Remaining false positives: #{false_positives.size} tests"
-puts "  ⚠️  False negatives: #{false_negatives.size} tests"
-puts "=" * 80

data/scripts/compare_current_failures.rb

@@ -1,95 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-# Direct comparison of current semantic vs DOM failures
-# Usage: ruby scripts/compare_current_failures.rb
-
-require "set"
-
-def parse_failures(file)
-  failures = Set.new
-  File.readlines(file).each do |line|
-    # Format: "rspec ./spec/isodoc/blocks_notes_spec.rb:494 # ..."
-    if line =~ %r{rspec \./spec/isodoc/([a-z_0-9]+_spec\.rb):(\d+)}
-      failures << "#{$1}:#{$2}"
-    end
-  end
-  failures
-end
-
-semantic_file = "/tmp/semantic_fresh.txt"
-dom_file = "/tmp/dom_fresh.txt"
-
-puts "Parsing failures..."
-semantic = parse_failures(semantic_file)
-dom = parse_failures(dom_file)
-
-puts "\n#{'=' * 80}"
-puts "CURRENT FAILURE STATE"
-puts "=" * 80
-
-puts "\nTotal failures:"
-puts "  Semantic: #{semantic.size}"
-puts "  DOM:      #{dom.size}"
-puts "  Gap:      #{(semantic.size - dom.size).abs}"
-
-false_positives = semantic - dom # Semantic fails, DOM passes
-false_negatives = dom - semantic   # DOM fails, Semantic passes
-common = semantic & dom            # Both fail
-
-puts "\n#{'-' * 80}"
-puts "FALSE POSITIVES (Semantic fails, DOM passes): #{false_positives.size}"
-puts "-" * 80
-false_positives.sort.each { |f| puts "  #{f}" }
-
-puts "\n#{'-' * 80}"
-puts "FALSE NEGATIVES (DOM fails, Semantic passes): #{false_negatives.size}"
-puts "-" * 80
-false_negatives.sort.each { |f| puts "  #{f}" }
-
-puts "\n#{'-' * 80}"
-puts "COMMON FAILURES (Both fail): #{common.size}"
-puts "-" * 80
-puts "(Not listing #{common.size} common failures for brevity)"
-
-puts "\n#{'=' * 80}"
-puts "ANALYSIS"
-puts "=" * 80
-
-if false_positives.empty? && false_negatives.size == 1
-  puts "\n✅ PERFECT PARITY ACHIEVED!"
-  puts "  - No false positives (semantic not too strict)"
-  puts "  - Only 1 false negative (acceptable difference)"
-  puts "  - This is the target state!"
-elsif false_positives.empty? && false_negatives.empty?
-  puts "\n🎉 EXACT PARITY ACHIEVED!"
-  puts "  - Both algorithms have identical failures"
-  puts "  - #{common.size} common failures"
-elsif false_positives.size == 1 && false_negatives.empty?
-  puts "\n⚠️  ONE FALSE POSITIVE AWAY FROM PARITY"
-  puts "  - Need to fix 1 case where semantic is too strict"
-  puts "  - Target: #{dom.size} failures for both algorithms"
-else
-  puts "\n📊 Current Status:"
-  puts "  - #{false_positives.size} false positives to fix (semantic too strict)"
-  puts "  - #{false_negatives.size} false negatives to address (semantic too lenient)"
-  puts "  - Gap from DOM: #{(semantic.size - dom.size).abs} failures"
-end
-
-# Save detailed results
-output_file = "/tmp/current_failure_analysis.txt"
-File.open(output_file, "w") do |f|
-  f.puts "CURRENT FAILURE ANALYSIS"
-  f.puts "=" * 80
-  f.puts "\nSemantic: #{semantic.size} failures"
-  f.puts "DOM:      #{dom.size} failures"
-  f.puts "Gap:      #{(semantic.size - dom.size).abs}"
-  f.puts "\nFALSE POSITIVES (#{false_positives.size}):"
-  false_positives.sort.each { |fp| f.puts fp }
-  f.puts "\nFALSE NEGATIVES (#{false_negatives.size}):"
-  false_negatives.sort.each { |fn| f.puts fn }
-  f.puts "\nCOMMON FAILURES (#{common.size}):"
-  common.sort.each { |cf| f.puts cf }
-end
-
-puts "\nDetailed results saved to: #{output_file}"

data/scripts/compare_dom_tree_diff.rb

@@ -1,158 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-require_relative "../lib/canon"
-
-# Sample IsoDoc-style XML with differences
-xml1 = <<~XML
-  <iso-standard xmlns="http://riboseinc.com/isoxml" type="presentation">
-     <preface>
-        <clause type="toc" id="_" displayorder="1">
-           <fmt-title id="_" depth="1">Table of contents</fmt-title>
-        </clause>
-        <foreword id="fwd" displayorder="2">
-           <title id="_">Foreword</title>
-           <fmt-title id="_" depth="1">
-              <semx element="title" source="_">Foreword</semx>
-           </fmt-title>
-           <figure id="F" autonum="1">
-              <fmt-name id="_">
-                 <span class="fmt-caption-label">
-                    <span class="fmt-element-name">Figure</span>
-                    <semx element="autonum" source="F">1</semx>
-                 </span>
-              </fmt-name>
-              <note id="FB" autonum="">
-                 <fmt-name id="_">
-                    <span class="fmt-caption-label">
-                       <span class="fmt-element-name">NOTE</span>
-                    </span>
-                 </fmt-name>
-                 <p>XYZ</p>
-              </note>
-           </figure>
-        </foreword>
-     </preface>
-  </iso-standard>
-XML
-
-xml2 = <<~XML
-  <iso-standard xmlns="http://riboseinc.com/isoxml" type="presentation">
-     <preface>
-        <clause type="toc" id="_" displayorder="1">
-           <fmt-title id="_" depth="1">Table of contents</fmt-title>
-        </clause>
-        <foreword id="fwd" displayorder="2">
-           <title id="_">Introduction</title>
-           <fmt-title id="_" depth="1">
-              <semx element="title" source="_">Introduction</semx>
-           </fmt-title>
-           <figure id="F" autonum="2">
-              <fmt-name id="_">
-                 <span class="fmt-caption-label">
-                    <span class="fmt-element-name">Figure</span>
-                    <semx element="autonum" source="F">2</semx>
-                 </span>
-              </fmt-name>
-              <note id="FB" autonum="">
-                 <fmt-name id="_">
-                    <span class="fmt-caption-label">
-                       <span class="fmt-element-name">NOTE</span>
-                    </span>
-                 </fmt-name>
-                 <p>ABC</p>
-              </note>
-           </figure>
-           <p id="new-para">This is a new paragraph.</p>
-        </foreword>
-     </preface>
-  </iso-standard>
-XML
-
-puts "=" * 80
-puts "DOM DIFF ALGORITHM COMPARISON"
-puts "=" * 80
-puts
-
-dom_result = Canon::Comparison.equivalent?(
-  xml1, xml2,
-  diff_algorithm: :dom,
-  verbose: true
-)
-
-puts "Algorithm: DOM DIFF"
-puts "Differences count: #{dom_result.differences.length}"
-puts "Operations count: #{dom_result.operations.length}"
-puts
-puts "Sample differences:"
-dom_result.differences.first(5).each_with_index do |diff, i|
-  puts "  #{i + 1}. Dimension: #{diff.dimension}"
-  puts "     Expected: #{diff.value1.to_s[0..100]}" if diff.respond_to?(:value1) && diff.value1
-  puts "     Actual: #{diff.value2.to_s[0..100]}" if diff.respond_to?(:value2) && diff.value2
-  puts
-end
-
-puts "\n#{'=' * 80}"
-puts "TREE (SEMANTIC) DIFF ALGORITHM COMPARISON"
-puts "=" * 80
-puts
-
-tree_result = Canon::Comparison.equivalent?(
-  xml1, xml2,
-  diff_algorithm: :semantic,
-  verbose: true
-)
-
-puts "Algorithm: SEMANTIC TREE DIFF"
-puts "Differences count: #{tree_result.differences.length}"
-puts "Operations count: #{tree_result.operations.length}"
-puts
-
-if tree_result.match_options[:tree_diff_statistics]
-  stats = tree_result.match_options[:tree_diff_statistics]
-  puts "Tree Statistics:"
-  puts "  - Tree1 nodes: #{stats[:tree1_nodes]}"
-  puts "  - Tree2 nodes: #{stats[:tree2_nodes]}"
-  puts "  - Total matches: #{stats[:total_matches]}"
-  puts "  - Match ratio (tree1): #{(stats[:match_ratio_tree1] * 100).round(1)}%"
-  puts "  - Match ratio (tree2): #{(stats[:match_ratio_tree2] * 100).round(1)}%"
-  puts
-end
-
-puts "Sample differences:"
-tree_result.differences.first(5).each_with_index do |diff, i|
-  puts "  #{i + 1}. Dimension: #{diff.dimension}"
-  puts "     Expected: #{diff.value1.to_s[0..100]}" if diff.respond_to?(:value1) && diff.value1
-  puts "     Actual: #{diff.value2.to_s[0..100]}" if diff.respond_to?(:value2) && diff.value2
-  puts
-end
-
-puts "\n#{'=' * 80}"
-puts "TREE DIFF OPERATIONS"
-puts "=" * 80
-puts
-
-tree_result.operations.first(10).each_with_index do |op, i|
-  puts "  #{i + 1}. #{op.type.to_s.upcase}"
-  puts "     Node: #{op.node.label}" if op.node
-  puts "     Details: #{op.inspect[0..150]}"
-  puts
-end
-
-puts "\n#{'=' * 80}"
-puts "COMPARISON SUMMARY"
-puts "=" * 80
-puts
-
-puts "DOM Diff:"
-puts "  - Differences count: #{dom_result.differences.length}"
-puts "  - Operations: #{dom_result.operations.length}"
-puts "  - Has detailed diff information: #{!dom_result.differences.empty?}"
-puts
-
-puts "Tree Diff:"
-puts "  - Differences count: #{tree_result.differences.length}"
-puts "  - Operations: #{tree_result.operations.length}"
-puts "  - Has detailed diff information: #{!tree_result.differences.empty?}"
-puts "  - Has tree diff operations: #{!tree_result.operations.empty?}"
-puts "  - Has tree statistics: #{!tree_result.match_options[:tree_diff_statistics].nil?}"