wp2txt 2.3.2 → 2.3.3

data/scripts/benchmark_regex.rb

@@ -1,161 +0,0 @@
-# frozen_string_literal: true
-
-# Benchmark script for wp2txt regex performance
-# Compares pre-compiled regex patterns vs inline compilation
-#
-# Usage: ruby scripts/benchmark_regex.rb
-
-require "benchmark"
-begin
-  require "benchmark/ips"
-rescue LoadError
-  # benchmark-ips is optional
-end
-
-# Add lib to load path
-$LOAD_PATH.unshift(File.expand_path("../lib", __dir__))
-require "wp2txt"
-require "wp2txt/article"
-
-# Sample Wikipedia-like content for benchmarking
-SAMPLE_TEXT = <<~WIKI
-  {{Infobox person
-  | name = Test Person
-  | birth_date = 1980-01-01
-  | occupation = Writer
-  }}
-  '''Test Article''' is a [[test]] article with various [[wiki markup|markup]].
-
-  == Section 1 ==
-  This section has some {{cite web|url=http://example.com|title=Example}} references.
-  There are also [[Category:Test]] links and [[File:Image.jpg|thumb|A caption]].
-
-  === Subsection ===
-  More content with '''bold''' and ''italic'' text.
-  * List item 1
-  * List item 2
-  # Numbered item
-
-  == Section 2 ==
-  {| class="wikitable"
-  |-
-  ! Header 1 !! Header 2
-  |-
-  | Cell 1 || Cell 2
-  |}
-
-  Some text with &nbsp; entities and &#x266A; characters.
-  Also has <ref name="test">Reference content</ref> and <nowiki>[[preserved]]</nowiki>.
-
-  {{DEFAULTSORT:Test Article}}
-  [[Category:Articles]]
-  [[Category:Tests]]
-WIKI
-
-# Create multiple copies for more realistic benchmarking
-LARGE_TEXT = (SAMPLE_TEXT * 100).freeze
-
-class BenchmarkRunner
-  include Wp2txt
-
-  def initialize
-    @nowikis = {}
-  end
-
-  def run_cleanup(text)
-    cleanup(text.dup)
-  end
-
-  def run_full_format(text)
-    format_wiki(text.dup)
-  end
-end
-
-def run_benchmarks
-  puts "=" * 60
-  puts "wp2txt Regex Performance Benchmark"
-  puts "=" * 60
-  puts
-  puts "Ruby version: #{RUBY_VERSION}"
-  puts "Sample text size: #{SAMPLE_TEXT.bytesize} bytes"
-  puts "Large text size: #{LARGE_TEXT.bytesize} bytes"
-  puts
-
-  runner = BenchmarkRunner.new
-
-  puts "-" * 60
-  puts "Warmup (JIT compilation, method caching)"
-  puts "-" * 60
-  5.times { runner.run_cleanup(SAMPLE_TEXT) }
-  5.times { runner.run_full_format(SAMPLE_TEXT) }
-  puts "Done."
-  puts
-
-  puts "-" * 60
-  puts "Benchmark: cleanup() method"
-  puts "-" * 60
-
-  Benchmark.bm(20) do |x|
-    x.report("cleanup (small):") do
-      1000.times { runner.run_cleanup(SAMPLE_TEXT) }
-    end
-
-    x.report("cleanup (large):") do
-      10.times { runner.run_cleanup(LARGE_TEXT) }
-    end
-  end
-
-  puts
-  puts "-" * 60
-  puts "Benchmark: format_wiki() method (full pipeline)"
-  puts "-" * 60
-
-  Benchmark.bm(20) do |x|
-    x.report("format_wiki (small):") do
-      1000.times { runner.run_full_format(SAMPLE_TEXT) }
-    end
-
-    x.report("format_wiki (large):") do
-      10.times { runner.run_full_format(LARGE_TEXT) }
-    end
-  end
-
-  # If benchmark-ips is available, run IPS benchmarks
-  if defined?(Benchmark::IPS)
-    puts
-    puts "-" * 60
-    puts "IPS Benchmark (iterations per second)"
-    puts "-" * 60
-
-    Benchmark.ips do |x|
-      x.report("cleanup") { runner.run_cleanup(SAMPLE_TEXT) }
-      x.report("format_wiki") { runner.run_full_format(SAMPLE_TEXT) }
-      x.compare!
-    end
-  end
-
-  puts
-  puts "-" * 60
-  puts "Memory profile (approximate)"
-  puts "-" * 60
-
-  # Simple memory measurement
-  GC.start
-  before = GC.stat[:total_allocated_objects]
-  100.times { runner.run_cleanup(SAMPLE_TEXT) }
-  after = GC.stat[:total_allocated_objects]
-  puts "cleanup() allocations per call: ~#{(after - before) / 100}"
-
-  GC.start
-  before = GC.stat[:total_allocated_objects]
-  100.times { runner.run_full_format(SAMPLE_TEXT) }
-  after = GC.stat[:total_allocated_objects]
-  puts "format_wiki() allocations per call: ~#{(after - before) / 100}"
-
-  puts
-  puts "=" * 60
-  puts "Benchmark complete"
-  puts "=" * 60
-end
-
-run_benchmarks

data/scripts/fetch_html_entities.rb

@@ -1,94 +0,0 @@
-# frozen_string_literal: true
-
-# Fetches HTML named character references from WHATWG HTML specification
-# Usage: ruby scripts/fetch_html_entities.rb
-#
-# This script downloads the official entities.json from WHATWG and converts
-# it into a format suitable for wp2txt text processing.
-
-require "net/http"
-require "json"
-require "fileutils"
-
-WHATWG_ENTITIES_URL = "https://html.spec.whatwg.org/entities.json"
-
-def fetch_whatwg_entities
-  puts "Fetching entities from WHATWG HTML specification..."
-  uri = URI(WHATWG_ENTITIES_URL)
-
-  response = Net::HTTP.get_response(uri)
-  unless response.is_a?(Net::HTTPSuccess)
-    warn "Failed to fetch entities: HTTP #{response.code}"
-    return nil
-  end
-
-  JSON.parse(response.body)
-rescue StandardError => e
-  warn "Error fetching entities: #{e.message}"
-  nil
-end
-
-def convert_entities(raw_data)
-  entities = {}
-
-  raw_data.each do |name, info|
-    # Only include entries with semicolon (standard form)
-    # Skip legacy forms without semicolon like "&nbsp"
-    next unless name.end_with?(";")
-
-    # Extract entity name without & and ;
-    # e.g., "α" -> "alpha"
-    key = name
-
-    # Get the character(s)
-    characters = info["characters"]
-    next if characters.nil? || characters.empty?
-
-    entities[key] = characters
-  end
-
-  entities
-end
-
-def main
-  raw_data = fetch_whatwg_entities
-  if raw_data.nil?
-    warn "Failed to fetch entities. Aborting."
-    exit 1
-  end
-
-  puts "Processing #{raw_data.size} raw entries..."
-
-  entities = convert_entities(raw_data)
-  puts "Converted to #{entities.size} standard entities (with semicolon)"
-
-  result = {
-    "meta" => {
-      "generated_at" => Time.now.utc.iso8601,
-      "source" => WHATWG_ENTITIES_URL,
-      "description" => "HTML named character references from WHATWG HTML specification",
-      "total_entities" => entities.size
-    },
-    "entities" => entities.sort.to_h
-  }
-
-  # Write output
-  output_path = File.join(__dir__, "..", "lib", "wp2txt", "data", "html_entities.json")
-  FileUtils.mkdir_p(File.dirname(output_path))
-
-  File.write(output_path, JSON.pretty_generate(result))
-  puts "\nData written to: #{output_path}"
-
-  # Summary - show some categories
-  greek = entities.keys.select { |k| k.match?(/&(alpha|beta|gamma|delta|epsilon|zeta|eta|theta|iota|kappa|lambda|mu|nu|xi|omicron|pi|rho|sigma|tau|upsilon|phi|chi|psi|omega);/i) }
-  math = entities.keys.select { |k| k.match?(/&(sum|prod|int|infin|nabla|part|forall|exist|empty|isin|notin|cap|cup|sub|sup|oplus|otimes);/i) }
-  arrows = entities.keys.select { |k| k.match?(/arr;$/i) }
-
-  puts "\n=== Summary ==="
-  puts "Total entities: #{entities.size}"
-  puts "Greek letters: #{greek.size}"
-  puts "Math symbols: #{math.size}"
-  puts "Arrows: #{arrows.size}"
-end
-
-main if __FILE__ == $PROGRAM_NAME

data/scripts/fetch_language_metadata.rb

@@ -1,180 +0,0 @@
-# frozen_string_literal: true
-
-# Fetches Wikipedia language metadata from Wikimedia APIs
-# Usage: ruby scripts/fetch_language_metadata.rb
-#
-# This script queries the Wikimedia sitematrix API to get all Wikipedia
-# language editions and their statistics (article counts, etc.)
-
-require "net/http"
-require "json"
-require "fileutils"
-
-# Fetch all Wikipedia languages with statistics from sitematrix API
-def fetch_wikipedia_languages
-  uri = URI("https://meta.wikimedia.org/w/api.php")
-  params = {
-    action: "sitematrix",
-    smtype: "language",
-    format: "json"
-  }
-  uri.query = URI.encode_www_form(params)
-
-  response = Net::HTTP.get_response(uri)
-  return {} unless response.is_a?(Net::HTTPSuccess)
-
-  data = JSON.parse(response.body)
-  languages = {}
-
-  data["sitematrix"].each do |key, val|
-    next unless key.match?(/^\d+$/) && val.is_a?(Hash) && val["site"]
-
-    # Find Wikipedia site info
-    wiki_site = val["site"].find { |site| site["code"] == "wiki" }
-    next unless wiki_site
-
-    lang_code = val["code"]
-    languages[lang_code] = {
-      "name" => val["name"],
-      "localname" => val["localname"],
-      "url" => wiki_site["url"],
-      "dbname" => wiki_site["dbname"],
-      "closed" => wiki_site["closed"] || false,
-      "private" => wiki_site["private"] || false
-    }
-  end
-
-  languages
-rescue StandardError => e
-  warn "Error fetching sitematrix: #{e.message}"
-  {}
-end
-
-# Fetch article statistics for a specific Wikipedia
-def fetch_wiki_statistics(lang_code)
-  uri = URI("https://#{lang_code}.wikipedia.org/w/api.php")
-  params = {
-    action: "query",
-    meta: "siteinfo",
-    siprop: "statistics",
-    format: "json"
-  }
-  uri.query = URI.encode_www_form(params)
-
-  response = Net::HTTP.get_response(uri)
-  return nil unless response.is_a?(Net::HTTPSuccess)
-
-  data = JSON.parse(response.body)
-  stats = data.dig("query", "statistics")
-  return nil unless stats
-
-  {
-    "articles" => stats["articles"],
-    "pages" => stats["pages"],
-    "edits" => stats["edits"],
-    "users" => stats["users"],
-    "activeusers" => stats["activeusers"]
-  }
-rescue StandardError
-  nil
-end
-
-def main
-  puts "Fetching Wikipedia language list..."
-  languages = fetch_wikipedia_languages
-
-  if languages.empty?
-    warn "Failed to fetch language list. Aborting."
-    exit 1
-  end
-
-  # Filter out closed/private wikis
-  active_languages = languages.reject { |_, info| info["closed"] || info["private"] }
-  puts "Found #{active_languages.size} active Wikipedia editions."
-
-  puts "Fetching statistics for each Wikipedia (this may take a few minutes)..."
-  successful = 0
-  failed = []
-
-  active_languages.each_with_index do |(lang_code, info), idx|
-    print "\r  Processing: #{lang_code.ljust(10)} (#{idx + 1}/#{active_languages.size})"
-    $stdout.flush
-
-    stats = fetch_wiki_statistics(lang_code)
-    if stats
-      info.merge!(stats)
-      successful += 1
-    else
-      failed << lang_code
-    end
-
-    sleep 0.05 # Rate limiting
-  end
-
-  puts "\n  Successfully fetched: #{successful}/#{active_languages.size}"
-  puts "  Failed: #{failed.size} (#{failed.first(10).join(', ')}#{failed.size > 10 ? '...' : ''})" if failed.any?
-
-  # Categorize by size
-  size_categories = {
-    "large" => [],    # 1M+ articles
-    "medium" => [],   # 100K-1M articles
-    "small" => [],    # 10K-100K articles
-    "mini" => []      # <10K articles
-  }
-
-  active_languages.each do |lang_code, info|
-    articles = info["articles"] || 0
-    category = if articles >= 1_000_000
-                 "large"
-               elsif articles >= 100_000
-                 "medium"
-               elsif articles >= 10_000
-                 "small"
-               else
-                 "mini"
-               end
-    size_categories[category] << lang_code
-    info["size_category"] = category
-  end
-
-  # Build result
-  result = {
-    "meta" => {
-      "generated_at" => Time.now.utc.iso8601,
-      "source" => "Wikimedia sitematrix + siteinfo APIs",
-      "total_languages" => active_languages.size,
-      "statistics_fetched" => successful
-    },
-    "size_summary" => {
-      "large" => size_categories["large"].size,
-      "medium" => size_categories["medium"].size,
-      "small" => size_categories["small"].size,
-      "mini" => size_categories["mini"].size
-    },
-    "languages" => active_languages.sort_by { |_, info| -(info["articles"] || 0) }.to_h
-  }
-
-  # Write output
-  output_path = File.join(__dir__, "..", "lib", "wp2txt", "data", "language_metadata.json")
-  FileUtils.mkdir_p(File.dirname(output_path))
-
-  File.write(output_path, JSON.pretty_generate(result))
-  puts "\nData written to: #{output_path}"
-
-  # Summary
-  puts "\n=== Summary ==="
-  puts "Total active Wikipedias: #{active_languages.size}"
-  puts "Size categories:"
-  puts "  Large (1M+ articles): #{size_categories['large'].size} - #{size_categories['large'].first(5).join(', ')}..."
-  puts "  Medium (100K-1M): #{size_categories['medium'].size}"
-  puts "  Small (10K-100K): #{size_categories['small'].size}"
-  puts "  Mini (<10K): #{size_categories['mini'].size}"
-
-  # Top 20 by article count
-  puts "\nTop 20 Wikipedias by article count:"
-  active_languages.sort_by { |_, info| -(info["articles"] || 0) }.first(20).each_with_index do |(code, info), idx|
-    puts "  #{(idx + 1).to_s.rjust(2)}. #{code.ljust(5)} #{info['name'].to_s.ljust(20)} #{(info['articles'] || 0).to_s.rjust(10)} articles"
-  end
-end
-
-main if __FILE__ == $PROGRAM_NAME