wp2txt 1.1.3 → 2.1.0

data/scripts/fetch_mediawiki_data.rb

@@ -0,0 +1,334 @@
+# frozen_string_literal: true
+
+# Fetches magic words, namespace data, and interwiki info from Wikipedia APIs
+# Usage: ruby scripts/fetch_mediawiki_data.rb
+#
+# This script queries the MediaWiki API for all Wikipedia language editions
+# and extracts:
+# - Magic words (redirect, defaultsort, displaytitle, image options, etc.)
+# - Double-underscore behavior switches (__NOTOC__, __TOC__, etc.)
+# - Namespace names and aliases (Category, File, Template, etc.)
+# - Interwiki map (for sister projects)
+
+require "net/http"
+require "json"
+require "fileutils"
+
+# Fetch all Wikipedia language codes from Wikimedia sitematrix API
+def fetch_all_wikipedia_languages
+  uri = URI("https://meta.wikimedia.org/w/api.php")
+  params = {
+    action: "sitematrix",
+    smtype: "language",
+    format: "json"
+  }
+  uri.query = URI.encode_www_form(params)
+
+  response = Net::HTTP.get_response(uri)
+  return [] unless response.is_a?(Net::HTTPSuccess)
+
+  data = JSON.parse(response.body)
+  languages = []
+
+  data["sitematrix"].each do |key, val|
+    next unless key.match?(/^\d+$/) && val.is_a?(Hash) && val["site"]
+
+    # Check if this language has a Wikipedia (code: 'wiki')
+    has_wikipedia = val["site"].any? { |site| site["code"] == "wiki" }
+    languages << val["code"] if has_wikipedia
+  end
+
+  languages.sort
+rescue StandardError => e
+  warn "Error fetching language list: #{e.message}"
+  []
+end
+
+# Magic word types we care about for text processing
+RELEVANT_MAGIC_WORDS = %w[
+  redirect
+  notoc noeditsection nogallery forcetoc toc nocontentconvert nocc
+  notitleconvert notc displaytitle defaultsort
+  img_thumbnail img_manualthumb img_right img_left img_none img_center
+  img_framed img_frameless img_page img_upright img_border img_baseline
+  img_sub img_super img_top img_text_top img_middle img_bottom img_text_bottom
+  img_link img_alt img_class img_lang
+].freeze
+
+# All namespace IDs (negative = special, 0 = main article, positive = other)
+# We want to collect all non-article namespaces for filtering
+# ID 0 is main namespace (articles), we want everything else
+NON_ARTICLE_NAMESPACE_IDS = [
+  -2,  # Media
+  -1,  # Special
+  # 0 is main article namespace - skip
+  1,   # Talk
+  2,   # User
+  3,   # User talk
+  4,   # Project (Wikipedia)
+  5,   # Project talk
+  6,   # File
+  7,   # File talk
+  8,   # MediaWiki
+  9,   # MediaWiki talk
+  10,  # Template
+  11,  # Template talk
+  12,  # Help
+  13,  # Help talk
+  14,  # Category
+  15,  # Category talk
+  # 100+ are custom namespaces per wiki (Portal, WikiProject, Module, etc.)
+].freeze
+
+# We still need specific namespace collections for targeted operations
+SPECIFIC_NAMESPACE_IDS = {
+  "file" => 6,
+  "category" => 14,
+  "template" => 10,
+  "wikipedia" => 4,
+  "help" => 12,
+  "portal" => 100, # May not exist in all wikis
+  "module" => 828  # May not exist in all wikis
+}.freeze
+
+def fetch_siteinfo(lang)
+  uri = URI("https://#{lang}.wikipedia.org/w/api.php")
+  params = {
+    action: "query",
+    meta: "siteinfo",
+    siprop: "magicwords|namespaces|namespacealiases|interwikimap|extensiontags",
+    format: "json"
+  }
+  uri.query = URI.encode_www_form(params)
+
+  response = Net::HTTP.get_response(uri)
+  return nil unless response.is_a?(Net::HTTPSuccess)
+
+  JSON.parse(response.body)
+rescue StandardError => e
+  warn "  Error fetching #{lang}: #{e.message}"
+  nil
+end
+
+def extract_extension_tags(data)
+  return [] unless data&.dig("query", "extensiontags")
+
+  # Extension tags come as ["<tag>", ...], extract just the tag name
+  data["query"]["extensiontags"].map do |tag|
+    tag.gsub(/^<|>$/, "")
+  end.compact.uniq
+end
+
+def extract_magic_words(data)
+  return {} unless data&.dig("query", "magicwords")
+
+  result = {}
+  double_underscore = []
+
+  data["query"]["magicwords"].each do |mw|
+    name = mw["name"]
+    aliases = mw["aliases"] || []
+
+    # Collect double-underscore magic words (behavior switches)
+    double_underscore_aliases = aliases.select { |a| a.start_with?("__") && a.end_with?("__") }
+    double_underscore.concat(double_underscore_aliases) unless double_underscore_aliases.empty?
+
+    # Only keep specific magic words we care about
+    next unless RELEVANT_MAGIC_WORDS.include?(name)
+
+    # Remove # prefix for redirect (we'll add it in regex)
+    if name == "redirect"
+      aliases = aliases.map { |a| a.sub(/^[#＃]/, "") }
+    end
+    result[name] = aliases.uniq
+  end
+
+  # Add double-underscore as a special category
+  result["double_underscore"] = double_underscore.uniq unless double_underscore.empty?
+
+  result
+end
+
+def extract_namespaces(data)
+  return {} unless data&.dig("query", "namespaces")
+
+  result = {}
+  all_non_article = []
+
+  # Get main namespace names
+  data["query"]["namespaces"].each do |id, ns|
+    id_int = id.to_i
+
+    # Collect all non-article namespace names
+    if id_int != 0 && ns["*"] && !ns["*"].empty?
+      all_non_article << ns["*"]
+      all_non_article << ns["canonical"] if ns["canonical"] && !ns["canonical"].empty?
+    end
+
+    # Also collect specific namespaces by ID
+    SPECIFIC_NAMESPACE_IDS.each do |key, target_id|
+      next unless id_int == target_id
+
+      result[key] ||= []
+      result[key] << ns["canonical"] if ns["canonical"] && !ns["canonical"].empty?
+      result[key] << ns["*"] if ns["*"] && !ns["*"].empty? && ns["*"] != ns["canonical"]
+    end
+  end
+
+  # Get namespace aliases
+  (data["query"]["namespacealiases"] || []).each do |alias_info|
+    id = alias_info["id"]
+    alias_name = alias_info["*"]
+    next unless alias_name && !alias_name.empty?
+
+    # Add to all non-article namespaces
+    all_non_article << alias_name if id != 0
+
+    # Add to specific namespace collections
+    SPECIFIC_NAMESPACE_IDS.each do |key, target_id|
+      next unless id == target_id
+
+      result[key] ||= []
+      result[key] << alias_name
+    end
+  end
+
+  # Store all non-article namespace names
+  result["non_article"] = all_non_article.uniq
+
+  # Deduplicate all collections
+  result.transform_values!(&:uniq)
+  result
+end
+
+def extract_interwiki(data)
+  return {} unless data&.dig("query", "interwikimap")
+
+  result = {}
+  sister_projects = []
+
+  # Known Wikimedia sister project prefixes
+  wikimedia_projects = %w[
+    commons wikibooks wikinews wikiquote wikisource
+    wikiversity wikivoyage wiktionary wikidata wikispecies
+    meta mediawiki mediawikiwiki species
+  ]
+
+  data["query"]["interwikimap"].each do |iw|
+    prefix = iw["prefix"]
+    url = iw["url"] || ""
+
+    # Check if this is a Wikimedia project
+    is_wikimedia = wikimedia_projects.include?(prefix) ||
+                   url.include?("wikimedia.org") ||
+                   url.include?("wikipedia.org") ||
+                   url.include?("wikibooks.org") ||
+                   url.include?("wikinews.org") ||
+                   url.include?("wikiquote.org") ||
+                   url.include?("wikisource.org") ||
+                   url.include?("wikiversity.org") ||
+                   url.include?("wikivoyage.org") ||
+                   url.include?("wiktionary.org") ||
+                   url.include?("wikidata.org") ||
+                   url.include?("wikispecies.org") ||
+                   url.include?("mediawiki.org")
+
+    sister_projects << prefix if is_wikimedia
+  end
+
+  result["sister_projects"] = sister_projects.uniq
+  result
+end
+
+def main
+  puts "Fetching list of all Wikipedia languages..."
+  languages = fetch_all_wikipedia_languages
+
+  if languages.empty?
+    warn "Failed to fetch language list. Aborting."
+    exit 1
+  end
+
+  puts "Found #{languages.size} Wikipedia editions. Fetching data..."
+
+  all_magic_words = Hash.new { |h, k| h[k] = Set.new }
+  all_namespaces = Hash.new { |h, k| h[k] = Set.new }
+  all_interwiki = Hash.new { |h, k| h[k] = Set.new }
+  all_extension_tags = Set.new
+  successful = 0
+  failed = []
+
+  languages.each_with_index do |lang, idx|
+    print "\r  Processing: #{lang.ljust(10)} (#{idx + 1}/#{languages.size})"
+    $stdout.flush
+
+    data = fetch_siteinfo(lang)
+    unless data
+      failed << lang
+      next
+    end
+
+    # Merge magic words
+    extract_magic_words(data).each do |name, aliases|
+      aliases.each { |a| all_magic_words[name] << a }
+    end
+
+    # Merge namespaces
+    extract_namespaces(data).each do |name, aliases|
+      aliases.each { |a| all_namespaces[name] << a }
+    end
+
+    # Merge interwiki
+    extract_interwiki(data).each do |name, prefixes|
+      prefixes.each { |p| all_interwiki[name] << p }
+    end
+
+    # Merge extension tags (consistent across all wikis, but collect from all to be safe)
+    extract_extension_tags(data).each { |tag| all_extension_tags << tag }
+
+    successful += 1
+    sleep 0.05 # Rate limiting (faster since we have many languages)
+  end
+
+  puts "\n  Successfully fetched: #{successful}/#{languages.size}"
+  puts "  Failed: #{failed.size} (#{failed.first(10).join(', ')}#{failed.size > 10 ? '...' : ''})" if failed.any?
+
+  # Convert Sets to sorted Arrays
+  result = {
+    "meta" => {
+      "generated_at" => Time.now.utc.iso8601,
+      "source" => "MediaWiki API (siteinfo via Wikimedia sitematrix)",
+      "languages_queried" => languages.size,
+      "languages_successful" => successful
+    },
+    "magic_words" => all_magic_words.transform_values { |v| v.to_a.sort },
+    "namespaces" => all_namespaces.transform_values { |v| v.to_a.sort },
+    "interwiki" => all_interwiki.transform_values { |v| v.to_a.sort },
+    "extension_tags" => all_extension_tags.to_a.sort
+  }
+
+  # Write output
+  output_path = File.join(__dir__, "..", "lib", "wp2txt", "data", "mediawiki_aliases.json")
+  FileUtils.mkdir_p(File.dirname(output_path))
+
+  File.write(output_path, JSON.pretty_generate(result))
+  puts "\nData written to: #{output_path}"
+
+  # Summary
+  puts "\n=== Summary ==="
+  puts "Magic Words:"
+  result["magic_words"].each do |name, aliases|
+    puts "  #{name}: #{aliases.size} aliases"
+  end
+  puts "\nNamespaces:"
+  result["namespaces"].each do |name, aliases|
+    puts "  #{name}: #{aliases.size} aliases"
+  end
+  puts "\nInterwiki:"
+  result["interwiki"].each do |name, prefixes|
+    puts "  #{name}: #{prefixes.size} prefixes"
+  end
+  puts "\nExtension Tags: #{result["extension_tags"].size} tags"
+end
+
+main if __FILE__ == $PROGRAM_NAME

data/scripts/fetch_template_data.rb

@@ -0,0 +1,186 @@
+# frozen_string_literal: true
+
+# Fetches template aliases and redirects from Wikipedia APIs
+# Usage: ruby scripts/fetch_template_data.rb
+#
+# This script augments the existing template_aliases.json by:
+# - Fetching redirects for known templates (to discover aliases)
+# - Validating that templates exist
+# - Merging new aliases from multiple Wikipedia language editions
+#
+# Note: Unlike magic words and namespaces (which come from MediaWiki siteinfo),
+# templates are wiki-specific pages. This script queries a subset of major
+# Wikipedia editions to collect common template aliases.
+
+require "net/http"
+require "json"
+require "fileutils"
+require "set"
+
+# Languages to query for template data
+TEMPLATE_LANGUAGES = %w[en ja de fr es it ru zh pt nl pl ar ko].freeze
+
+# Base template names to look up redirects for (English)
+# These are canonical names; we'll find their translations and aliases
+BASE_TEMPLATES = {
+  "remove_templates" => %w[
+    Reflist Notelist Sfn Efn Main See_also Further About Portal
+  ],
+  "authority_control" => %w[
+    Authority_control Normdaten Persondata
+  ],
+  "citation_templates" => %w[
+    Cite_web Cite_book Cite_news Cite_journal Citation
+  ],
+  "sister_project_templates" => %w[
+    Commons Commons_category Wiktionary Wikiquote Wikisource
+  ]
+}.freeze
+
+# Fetch template redirects from a specific Wikipedia
+def fetch_template_redirects(lang, template_name)
+  uri = URI("https://#{lang}.wikipedia.org/w/api.php")
+  params = {
+    action: "query",
+    titles: "Template:#{template_name}",
+    prop: "redirects",
+    rdlimit: "max",
+    format: "json"
+  }
+  uri.query = URI.encode_www_form(params)
+
+  response = Net::HTTP.get_response(uri)
+  return [] unless response.is_a?(Net::HTTPSuccess)
+
+  data = JSON.parse(response.body)
+  pages = data.dig("query", "pages") || {}
+
+  redirects = []
+  pages.each_value do |page|
+    next if page["missing"]
+
+    # Add the page title itself (normalized)
+    page_title = page["title"]&.sub(/^Template:/, "")
+    redirects << page_title if page_title
+
+    # Add all redirects
+    (page["redirects"] || []).each do |rd|
+      rd_title = rd["title"]&.sub(/^Template:/, "")
+      redirects << rd_title if rd_title
+    end
+  end
+
+  redirects.compact.uniq
+rescue StandardError => e
+  warn "  Error fetching #{lang}:Template:#{template_name}: #{e.message}"
+  []
+end
+
+# Fetch all templates in a category
+def fetch_category_members(lang, category_name, limit: 100)
+  uri = URI("https://#{lang}.wikipedia.org/w/api.php")
+  params = {
+    action: "query",
+    list: "categorymembers",
+    cmtitle: "Category:#{category_name}",
+    cmtype: "page",
+    cmnamespace: "10",  # Template namespace
+    cmlimit: limit.to_s,
+    format: "json"
+  }
+  uri.query = URI.encode_www_form(params)
+
+  response = Net::HTTP.get_response(uri)
+  return [] unless response.is_a?(Net::HTTPSuccess)
+
+  data = JSON.parse(response.body)
+  members = data.dig("query", "categorymembers") || []
+
+  members.map { |m| m["title"]&.sub(/^Template:/, "") }.compact
+rescue StandardError => e
+  warn "  Error fetching category #{category_name}: #{e.message}"
+  []
+end
+
+def main
+  puts "Template Data Fetcher"
+  puts "=" * 50
+
+  # Load existing data
+  data_path = File.join(__dir__, "..", "lib", "wp2txt", "data", "template_aliases.json")
+  existing_data = if File.exist?(data_path)
+    JSON.parse(File.read(data_path))
+  else
+    { "meta" => {} }
+  end
+
+  # Convert arrays to sets for efficient merging
+  categories = {}
+  existing_data.each do |key, value|
+    next if key == "meta"
+    categories[key] = Set.new(value) if value.is_a?(Array)
+  end
+
+  puts "\nFetching template redirects from #{TEMPLATE_LANGUAGES.size} Wikipedia editions..."
+
+  # Fetch redirects for base templates
+  BASE_TEMPLATES.each do |category, templates|
+    puts "\n#{category}:"
+    templates.each do |template|
+      TEMPLATE_LANGUAGES.each do |lang|
+        print "  #{lang}:#{template}..."
+        aliases = fetch_template_redirects(lang, template)
+        if aliases.any?
+          categories[category] ||= Set.new
+          aliases.each { |a| categories[category] << a }
+          puts " #{aliases.size} aliases"
+        else
+          puts " not found"
+        end
+        sleep 0.1  # Rate limiting
+      end
+    end
+  end
+
+  # Fetch citation templates from category (English)
+  puts "\nFetching citation templates from category..."
+  citation_members = fetch_category_members("en", "Citation templates")
+  if citation_members.any?
+    categories["citation_templates"] ||= Set.new
+    citation_members.each { |t| categories["citation_templates"] << t }
+    puts "  Found #{citation_members.size} citation templates"
+  end
+
+  # Fetch hatnote templates
+  puts "\nFetching hatnote templates from category..."
+  hatnote_members = fetch_category_members("en", "Hatnote templates")
+  if hatnote_members.any?
+    categories["remove_templates"] ||= Set.new
+    hatnote_members.each { |t| categories["remove_templates"] << t }
+    puts "  Found #{hatnote_members.size} hatnote templates"
+  end
+
+  # Convert sets back to sorted arrays
+  result = { "meta" => existing_data["meta"] || {} }
+  result["meta"]["generated_at"] = Time.now.utc.iso8601
+  result["meta"]["source"] = "Manual curation + MediaWiki API (templates)"
+  result["meta"]["languages_queried"] = TEMPLATE_LANGUAGES
+
+  categories.each do |key, set|
+    result[key] = set.to_a.sort_by(&:downcase)
+  end
+
+  # Write output
+  File.write(data_path, JSON.pretty_generate(result))
+  puts "\n" + "=" * 50
+  puts "Data written to: #{data_path}"
+
+  # Summary
+  puts "\n=== Summary ==="
+  result.each do |key, value|
+    next if key == "meta"
+    puts "  #{key}: #{value.size} templates"
+  end
+end
+
+main if __FILE__ == $PROGRAM_NAME

data/scripts/profile_memory.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+# Memory profiling script for wp2txt streaming operations
+# Demonstrates the MemoryMonitor module and adaptive buffer sizing
+#
+# Usage: ruby scripts/profile_memory.rb [input_file.xml]
+
+$LOAD_PATH.unshift(File.expand_path("../lib", __dir__))
+require "wp2txt"
+require "wp2txt/memory_monitor"
+require "wp2txt/stream_processor"
+
+def print_separator(title = nil)
+  puts
+  puts "=" * 60
+  puts title if title
+  puts "=" * 60
+end
+
+def profile_memory_monitor
+  print_separator("MemoryMonitor Module Test")
+
+  puts "System Memory Information:"
+  puts "-" * 40
+
+  stats = Wp2txt::MemoryMonitor.memory_stats
+  stats.each do |key, value|
+    formatted_key = key.to_s.gsub("_", " ").capitalize
+    puts "  #{formatted_key}: #{value}"
+  end
+
+  puts
+  puts "Raw values:"
+  puts "  Current usage: #{Wp2txt::MemoryMonitor.format_memory(Wp2txt::MemoryMonitor.current_memory_usage)}"
+  puts "  Total system:  #{Wp2txt::MemoryMonitor.format_memory(Wp2txt::MemoryMonitor.total_system_memory)}"
+  puts "  Available:     #{Wp2txt::MemoryMonitor.format_memory(Wp2txt::MemoryMonitor.available_memory)}"
+  puts "  Optimal buffer: #{Wp2txt::MemoryMonitor.format_memory(Wp2txt::MemoryMonitor.optimal_buffer_size)}"
+end
+
+def profile_stream_processor(input_file)
+  print_separator("StreamProcessor Memory Profile")
+
+  unless File.exist?(input_file)
+    puts "File not found: #{input_file}"
+    puts "Creating a sample XML file for testing..."
+
+    # Create a sample file for testing
+    sample_xml = <<~XML
+      <mediawiki>
+      #{(1..100).map { |i| <<~PAGE
+        <page>
+          <title>Test Article #{i}</title>
+          <revision>
+            <text>This is test content for article #{i}. #{"Lorem ipsum " * 100}</text>
+          </revision>
+        </page>
+        PAGE
+      }.join}
+      </mediawiki>
+    XML
+
+    input_file = "/tmp/wp2txt_test_sample.xml"
+    File.write(input_file, sample_xml)
+    puts "Created sample file: #{input_file} (#{File.size(input_file)} bytes)"
+  end
+
+  puts
+  puts "Processing: #{input_file}"
+  puts "File size: #{Wp2txt::MemoryMonitor.format_memory(File.size(input_file))}"
+  puts
+
+  # Test with adaptive buffer
+  puts "Testing with adaptive buffer sizing:"
+  puts "-" * 40
+
+  processor = Wp2txt::StreamProcessor.new(input_file, adaptive_buffer: true)
+  puts "Initial buffer size: #{Wp2txt::MemoryMonitor.format_memory(processor.buffer_size)}"
+
+  start_time = Time.now
+  start_memory = Wp2txt::MemoryMonitor.current_memory_usage
+
+  page_count = 0
+  processor.each_page do |title, _text|
+    page_count += 1
+    if page_count % 10 == 0
+      stats = processor.stats
+      puts "  Processed #{page_count} pages, buffer: #{Wp2txt::MemoryMonitor.format_memory(stats[:buffer_size])}"
+    end
+  end
+
+  end_time = Time.now
+  end_memory = Wp2txt::MemoryMonitor.current_memory_usage
+
+  puts
+  puts "Final Statistics:"
+  final_stats = processor.stats
+  puts "  Pages processed: #{final_stats[:pages_processed]}"
+  puts "  Bytes read: #{Wp2txt::MemoryMonitor.format_memory(final_stats[:bytes_read])}"
+  puts "  Final buffer size: #{Wp2txt::MemoryMonitor.format_memory(final_stats[:buffer_size])}"
+  puts "  Processing time: #{(end_time - start_time).round(3)}s"
+  puts "  Memory delta: #{Wp2txt::MemoryMonitor.format_memory(end_memory - start_memory)}"
+
+  # Test without adaptive buffer
+  puts
+  puts "Testing with fixed buffer sizing (10 MB):"
+  puts "-" * 40
+
+  processor2 = Wp2txt::StreamProcessor.new(input_file, adaptive_buffer: false)
+  puts "Fixed buffer size: #{Wp2txt::MemoryMonitor.format_memory(processor2.buffer_size)}"
+
+  start_time = Time.now
+  start_memory = Wp2txt::MemoryMonitor.current_memory_usage
+
+  page_count = 0
+  processor2.each_page do |_title, _text|
+    page_count += 1
+  end
+
+  end_time = Time.now
+  end_memory = Wp2txt::MemoryMonitor.current_memory_usage
+
+  final_stats = processor2.stats
+  puts "  Pages processed: #{final_stats[:pages_processed]}"
+  puts "  Processing time: #{(end_time - start_time).round(3)}s"
+  puts "  Memory delta: #{Wp2txt::MemoryMonitor.format_memory(end_memory - start_memory)}"
+end
+
+def main
+  profile_memory_monitor
+
+  input_file = ARGV[0] || "/tmp/wp2txt_test_sample.xml"
+  profile_stream_processor(input_file)
+
+  print_separator("Memory Profiling Complete")
+  puts "Current memory: #{Wp2txt::MemoryMonitor.format_memory(Wp2txt::MemoryMonitor.current_memory_usage)}"
+  puts "Memory low?: #{Wp2txt::MemoryMonitor.memory_low?}"
+end
+
+main