wp2txt 1.1.3 → 2.1.0

data/lib/wp2txt/ractor_worker.rb

@@ -0,0 +1,215 @@
+# frozen_string_literal: true
+
+require "etc"
+
+module Wp2txt
+  # Ractor-based parallel processing for Wikipedia article conversion
+  #
+  # Ractor allows true parallelism by bypassing Ruby's GVL (Global VM Lock),
+  # enabling significant speedups for CPU-intensive text processing.
+  #
+  # REQUIREMENTS: Ruby 4.0+ (Ractor API stabilized in Ruby 4.0)
+  # For Ruby 3.x, the Parallel gem is used instead (process-based parallelism).
+  #
+  # Performance: Typically 1.5-2x speedup with 4 workers on multi-core systems.
+  #
+  # Usage:
+  #   pages = [["Title1", "wiki text..."], ["Title2", "wiki text..."]]
+  #   results = RactorWorker.process_articles(pages, config: config)
+  #
+  module RactorWorker
+    # Minimum Ruby version required for stable Ractor support
+    MINIMUM_RUBY_VERSION = "4.0"
+
+    # Registry of available operations
+    OPERATIONS = %i[process_article double fib].freeze
+
+    module_function
+
+    # Check if Ractor is available and usable
+    # Requires Ruby 4.0+ for stable Ractor support
+    # @return [Boolean] true if Ractor can be used
+    def available?
+      return @available if defined?(@available)
+
+      @available = check_ractor_available
+    end
+
+    # Internal method to check Ractor availability
+    # @return [Boolean] true if Ractor can be used
+    def check_ractor_available
+      return false unless ruby_version_sufficient?
+      return false unless defined?(Ractor)
+
+      # Test basic Ractor functionality with Ruby 4.0 API
+      r = Ractor.new { 1 + 1 }
+      r.join
+      r.value == 2
+    rescue StandardError
+      false
+    end
+
+    # Check if Ruby version meets minimum requirement
+    # @return [Boolean] true if Ruby version is 4.0 or higher
+    def ruby_version_sufficient?
+      Gem::Version.new(RUBY_VERSION) >= Gem::Version.new(MINIMUM_RUBY_VERSION)
+    end
+
+    # Process articles in parallel using Ractor (main entry point)
+    # @param pages [Array<Array>] Array of [title, text] pairs
+    # @param config [Hash] Configuration options for formatting
+    # @param strip_tmarker [Boolean] Whether to strip list markers
+    # @param num_workers [Integer] Number of parallel Ractors (optional)
+    # @return [Array<String>] Formatted article results
+    def process_articles(pages, config:, strip_tmarker: false, num_workers: nil)
+      items = pages.map { |title, text| [title, text, strip_tmarker] }
+
+      parallel_process(
+        items,
+        operation: :process_article,
+        config: config,
+        num_workers: num_workers
+      )
+    end
+
+    # Process items in parallel using map-join-value pattern (Ruby 4.0+)
+    # @param items [Array] Items to process
+    # @param operation [Symbol] Operation to perform (:process_article, :double, :fib)
+    # @param config [Hash] Configuration to pass to each operation
+    # @param num_workers [Integer] Max concurrent Ractors (default: optimal_workers)
+    # @return [Array] Results from processing (in original order)
+    def parallel_process(items, operation:, config: {}, num_workers: nil)
+      batch_size = num_workers || optimal_workers
+      batch_size = [batch_size, 1].max
+
+      # Fall back to sequential if Ractor not available or single item
+      unless available? && items.size > 1
+        return items.map { |item| process_single(item, operation, config) }
+      end
+
+      # Freeze config for sharing across Ractors
+      frozen_config = deep_freeze(config.dup)
+
+      # Process in batches to limit concurrent Ractors
+      results = []
+      items.each_slice(batch_size) do |batch|
+        batch_results = process_batch(batch, operation, frozen_config)
+        results.concat(batch_results)
+      end
+
+      results
+    rescue Ractor::Error => e
+      warn "Ractor error (#{e.message}), falling back to sequential processing"
+      items.map { |item| process_single(item, operation, config) }
+    end
+
+    # Process a batch using map-join-value pattern (Ruby 4.0 API)
+    # @param items [Array] Items to process in this batch
+    # @param operation [Symbol] Operation to perform
+    # @param frozen_config [Hash] Frozen configuration hash
+    # @return [Array] Results in original order
+    def process_batch(items, operation, frozen_config)
+      # Create one Ractor per item
+      ractors = items.map.with_index do |item, idx|
+        Ractor.new(item, frozen_config, operation, idx) do |it, cfg, op, i|
+          result = begin
+            case op
+            when :process_article
+              require_relative "utils"
+              require_relative "regex"
+              require_relative "article"
+              require_relative "formatter"
+
+              title, text, strip_tmarker = it
+              formatter = Object.new
+              formatter.extend(Wp2txt)
+              formatter.extend(Wp2txt::Formatter)
+              article = Wp2txt::Article.new(text, title, strip_tmarker)
+              formatter.format_article(article, cfg)
+            when :double
+              it * 2
+            when :fib
+              fib = ->(n) { n <= 1 ? n : fib.call(n - 1) + fib.call(n - 2) }
+              fib.call(it)
+            else
+              raise "Unknown operation: #{op}"
+            end
+          rescue StandardError
+            nil # Return nil on error
+          end
+          [i, result] # Return index and result for ordering
+        end
+      end
+
+      # Wait for all Ractors to complete and collect results
+      collected = Array.new(items.size)
+      ractors.each do |r|
+        r.join
+        idx, result = r.value
+        collected[idx] = result
+      end
+
+      collected
+    end
+
+    # Process a single item (for fallback/sequential processing)
+    # @param item [Object] Item to process
+    # @param operation [Symbol] Operation to perform
+    # @param config [Hash] Configuration options
+    # @return [Object] Processing result
+    def process_single(item, operation, config)
+      case operation
+      when :process_article
+        require_relative "utils"
+        require_relative "regex"
+        require_relative "article"
+        require_relative "formatter"
+
+        title, text, strip_tmarker = item
+        formatter = Object.new
+        formatter.extend(Wp2txt)
+        formatter.extend(Wp2txt::Formatter)
+        article = Wp2txt::Article.new(text, title, strip_tmarker)
+        formatter.format_article(article, config)
+      when :double
+        item * 2
+      when :fib
+        fib = ->(n) { n <= 1 ? n : fib.call(n - 1) + fib.call(n - 2) }
+        fib.call(item)
+      else
+        raise "Unknown operation: #{operation}"
+      end
+    end
+
+    # Calculate optimal number of workers based on CPU cores
+    # @return [Integer] Recommended concurrency level
+    def optimal_workers
+      cores = Etc.nprocessors
+      case cores
+      when 1..4 then cores
+      when 5..8 then cores - 1
+      else (cores * 0.8).to_i
+      end
+    end
+
+    # Deep freeze an object for Ractor sharing
+    # @param obj [Object] Object to freeze
+    # @return [Object] The frozen object
+    def deep_freeze(obj)
+      case obj
+      when Hash
+        obj.transform_keys { |k| deep_freeze(k) }
+           .transform_values { |v| deep_freeze(v) }
+           .freeze
+      when Array
+        obj.map { |v| deep_freeze(v) }.freeze
+      when String
+        obj.frozen? ? obj : obj.dup.freeze
+      when Symbol, Integer, Float, TrueClass, FalseClass, NilClass
+        obj
+      else
+        obj.freeze rescue obj
+      end
+    end
+  end
+end

data/lib/wp2txt/regex.rb

@@ -1,8 +1,145 @@
 # frozen_string_literal: true
 
 require "htmlentities"
+require "json"
+require_relative "global_data_cache"
+
+# Make HTMLEntities gem Ractor-shareable
+# This must be done before any Ractor tries to use HTMLEntities
+if defined?(Ractor) && Ractor.respond_to?(:make_shareable)
+  begin
+    HTMLEntities.constants.each do |const_name|
+      const = HTMLEntities.const_get(const_name)
+      next if Ractor.shareable?(const)
+
+      Ractor.make_shareable(const)
+    rescue Ractor::IsolationError, FrozenError, TypeError
+      # Skip if can't be made shareable
+    end
+  rescue StandardError
+    # Ignore errors during shareable setup
+  end
+end
 
 module Wp2txt
+  # Data file paths
+  MEDIAWIKI_DATA_PATH = File.join(__dir__, "data", "mediawiki_aliases.json")
+  HTML_ENTITIES_PATH = File.join(__dir__, "data", "html_entities.json")
+  WIKIPEDIA_ENTITIES_PATH = File.join(__dir__, "data", "wikipedia_entities.json")
+  TEMPLATE_DATA_PATH = File.join(__dir__, "data", "template_aliases.json")
+
+  # Load MediaWiki aliases from data file (generated by scripts/fetch_mediawiki_data.rb)
+  # Uses SQLite cache for faster subsequent loads
+  def self.load_mediawiki_data
+    return @mediawiki_data if @mediawiki_data
+
+    # Try SQLite cache first
+    cached = GlobalDataCache.load(:mediawiki)
+    if cached
+      @mediawiki_data = cached
+      return @mediawiki_data
+    end
+
+    # Load from JSON file
+    if File.exist?(MEDIAWIKI_DATA_PATH)
+      @mediawiki_data = JSON.parse(File.read(MEDIAWIKI_DATA_PATH))
+      # Save to cache for future use
+      GlobalDataCache.save(:mediawiki, @mediawiki_data)
+    else
+      # Fallback to minimal defaults if data file is missing
+      @mediawiki_data = {
+        "magic_words" => { "redirect" => ["REDIRECT"] },
+        "namespaces" => { "category" => ["Category"], "file" => ["File", "Image"] }
+      }
+    end
+    @mediawiki_data
+  end
+
+  # Load template aliases from data file (generated by scripts/fetch_template_data.rb)
+  # Uses SQLite cache for faster subsequent loads
+  def self.load_template_data
+    return @template_data if @template_data
+
+    # Try SQLite cache first
+    cached = GlobalDataCache.load(:template)
+    if cached
+      @template_data = cached
+      return @template_data
+    end
+
+    # Load from JSON file
+    if File.exist?(TEMPLATE_DATA_PATH)
+      @template_data = JSON.parse(File.read(TEMPLATE_DATA_PATH))
+      # Save to cache for future use
+      GlobalDataCache.save(:template, @template_data)
+    else
+      # Fallback to minimal defaults if data file is missing
+      @template_data = {
+        "remove_templates" => %w[reflist notelist main see\ also portal],
+        "authority_control" => %w[authority\ control normdaten],
+        "cleanup_remnants" => %w[clear clearleft clearright]
+      }
+    end
+    @template_data
+  end
+
+  # Build regex pattern from template list (escapes special chars, joins with |)
+  def self.build_template_regex(templates, options = {})
+    return nil if templates.nil? || templates.empty?
+
+    pattern = templates.map { |t| Regexp.escape(t) }.join("|")
+    if options[:anchor_start]
+      pattern = '\A\s*(?:' + pattern + ')'
+    else
+      pattern = '(?:' + pattern + ')'
+    end
+    if options[:require_pipe_or_end]
+      pattern += '\s*(?:\||$)'
+    end
+    Regexp.new(pattern, Regexp::IGNORECASE)
+  end
+
+  # Load HTML entities from WHATWG data file (generated by scripts/fetch_html_entities.rb)
+  # Uses SQLite cache for faster subsequent loads
+  def self.load_html_entities
+    return @html_entities if @html_entities
+
+    # Try SQLite cache first (combined entities)
+    cached = GlobalDataCache.load(:html_entities_combined)
+    if cached
+      @html_entities = cached
+      return @html_entities
+    end
+
+    @html_entities = {}
+
+    # Load WHATWG HTML entities
+    if File.exist?(HTML_ENTITIES_PATH)
+      data = JSON.parse(File.read(HTML_ENTITIES_PATH))
+      @html_entities.merge!(data["entities"] || {})
+    end
+
+    # Load Wikipedia-specific entities (override/supplement WHATWG)
+    if File.exist?(WIKIPEDIA_ENTITIES_PATH)
+      data = JSON.parse(File.read(WIKIPEDIA_ENTITIES_PATH))
+      @html_entities.merge!(data["entities"] || {})
+    end
+
+    # Save combined entities to cache
+    GlobalDataCache.save(:html_entities_combined, @html_entities) unless @html_entities.empty?
+
+    @html_entities
+  end
+
+  # Build regex for extra entities not handled by HTMLEntities gem
+  def self.build_extra_entities_regex
+    entities = load_html_entities
+    return nil if entities.empty?
+
+    # Build regex pattern for all entity keys
+    pattern = "(" + entities.keys.map { |k| Regexp.escape(k) }.join("|") + ")"
+    Regexp.new(pattern)
+  end
   ###################################################
   # variables to save resource for generating regexps
   # those with a trailing number 1 represent opening tag/markup
@@ -14,9 +151,22 @@ module Wp2txt
   ENTITIES = ['&nbsp;', '&lt;', '&gt;', '&amp;', '&quot;'].zip([' ', '<', '>', '&', '"'])
   HTML_HASH = Hash[*ENTITIES.flatten]
   HTML_REGEX = Regexp.new("(" + HTML_HASH.keys.join("|") + ")")
+
+  # Extra HTML entities loaded from JSON files (WHATWG + Wikipedia-specific)
+  # These supplement the HTMLEntities gem which only covers ~250 basic entities
+  # Data sources:
+  #   - lib/wp2txt/data/html_entities.json (2000+ WHATWG entities)
+  #   - lib/wp2txt/data/wikipedia_entities.json (Wikipedia-specific)
+  EXTRA_ENTITIES = load_html_entities.freeze
+  EXTRA_ENTITIES_REGEX = build_extra_entities_regex
+
+  # Legacy alias for backward compatibility
+  MATH_ENTITIES = EXTRA_ENTITIES
+  MATH_ENTITIES_REGEX = EXTRA_ENTITIES_REGEX
   ML_TEMPLATE_ONSET_REGEX = Regexp.new('^\{\{[^\}]*$')
   ML_TEMPLATE_END_REGEX = Regexp.new('\}\}\s*$')
-  ML_LINK_ONSET_REGEX = Regexp.new('^\[\[[^\]]*$')
+  # Match lines starting with [[ that don't end with ]] (handles inner links)
+  ML_LINK_ONSET_REGEX = Regexp.new('^\[\[(?!.*\]\]\s*$).*$')
   ML_LINK_END_REGEX = Regexp.new('\]\]\s*$')
   ISOLATED_TEMPLATE_REGEX = Regexp.new('^\s*\{\{.+\}\}\s*$')
   ISOLATED_TAG_REGEX = Regexp.new('^\s*\<[^\<\>]+\>.+\<[^\<\>]+\>\s*$')
@@ -30,10 +180,12 @@ module Wp2txt
   IN_MATH_REGEX = Regexp.new('<math.*?>.*?<\/math>')
   IN_MATH_REGEX1 = Regexp.new('<math.*?>')
   IN_MATH_REGEX2 = Regexp.new('<\/math>')
-  IN_HEADING_REGEX = Regexp.new('^=+.*?=+$')
+  IN_HEADING_REGEX = Regexp.new('^=+.*?=+\s*$')
   IN_HTML_TABLE_REGEX = Regexp.new("<table.*?><\/table>")
   IN_HTML_TABLE_REGEX1 = Regexp.new('<table\b')
   IN_HTML_TABLE_REGEX2 = Regexp.new('<\/\s*table>')
+  # HTML comments (can span multiple lines)
+  HTML_COMMENT_REGEX = Regexp.new('<!--.*?-->', Regexp::MULTILINE)
   IN_TABLE_REGEX1 = Regexp.new('^\s*\{\|')
   IN_TABLE_REGEX2 = Regexp.new('^\|\}.*?$')
   IN_UNORDERED_REGEX = Regexp.new('^\*')
@@ -41,17 +193,25 @@ module Wp2txt
   IN_PRE_REGEX = Regexp.new('^ ')
   IN_DEFINITION_REGEX = Regexp.new('^[\;\:]')
   BLANK_LINE_REGEX = Regexp.new('^\s*$')
-  REDIRECT_REGEX = Regexp.new('#(?:REDIRECT|転送)\s+\[\[(.+)\]\]', Regexp::IGNORECASE)
+
+  # Multilingual redirect keyword support
+  # Data source: MediaWiki API (siteinfo) via scripts/fetch_mediawiki_data.rb
+  REDIRECT_KEYWORDS = load_mediawiki_data.dig("magic_words", "redirect")&.join("|") || "REDIRECT"
+  REDIRECT_REGEX = Regexp.new('[#＃](?:' + REDIRECT_KEYWORDS + ')\s*:?\s*\[\[([^\]]+)\]\]', Regexp::IGNORECASE)
   REMOVE_TAG_REGEX = Regexp.new("\<[^\<\>]*\>")
-  REMOVE_DIRECTIVES_REGEX = Regexp.new("\_\_[^\_]*\_\_")
+
+  # Legacy generic pattern for double-underscore directives
+  # Note: Data-driven REMOVE_DIRECTIVES_REGEX is defined later (after DOUBLE_UNDERSCORE_MAGIC_REGEX)
+  REMOVE_DIRECTIVES_REGEX_GENERIC = Regexp.new("\_\_[^\_]*\_\_")
+
   REMOVE_EMPHASIS_REGEX = Regexp.new('(' + Regexp.escape("''") + '+)(.+?)\1')
   CHRREF_TO_UTF_REGEX = Regexp.new('&#(x?)([0-9a-fA-F]+);')
   MNDASH_REGEX = Regexp.new('\{(mdash|ndash|–)\}')
-  REMOVE_HR_REGEX = Regexp.new('^\s*\-+\s*$')
-  MAKE_REFERENCE_REGEX_A = Regexp.new('<br ?\/>')
-  MAKE_REFERENCE_REGEX_B = Regexp.new('<ref[^>]*\/>')
-  MAKE_REFERENCE_REGEX_C = Regexp.new('<ref[^>]*>')
-  MAKE_REFERENCE_REGEX_D = Regexp.new('<\/ref>')
+  REMOVE_HR_REGEX = Regexp.new('^\s*\-{4,}\s*$')
+  MAKE_REFERENCE_REGEX_A = Regexp.new('<br ?\/>', Regexp::IGNORECASE)
+  MAKE_REFERENCE_REGEX_B = Regexp.new('<ref[^>]*\/>', Regexp::IGNORECASE)
+  MAKE_REFERENCE_REGEX_C = Regexp.new('<ref[^>]*>', Regexp::IGNORECASE)
+  MAKE_REFERENCE_REGEX_D = Regexp.new('<\/ref>', Regexp::IGNORECASE)
   FORMAT_REF_REGEX = Regexp.new('\[ref\](.*?)\[\/ref\]', Regexp::MULTILINE)
   HEADING_ONSET_REGEX = Regexp.new('^(\=+)\s+')
   HEADING_CODA_REGEX = Regexp.new('\s+(\=+)$')
@@ -60,15 +220,44 @@ module Wp2txt
   DEF_MARKS_REGEX = Regexp.new('\A[\;\:\ ]+')
   ONSET_BAR_REGEX = Regexp.new('\A[^\|]+\z')
 
-  CATEGORY_PATTERNS = ["Category", "Categoria"].join("|")
-  CATEGORY_REGEX = Regexp.new('[\{\[\|\b](?:' + CATEGORY_PATTERNS + ')\:(.*?)[\}\]\|\b]', Regexp::IGNORECASE)
+  # Multilingual category namespace support
+  # Data source: MediaWiki API (siteinfo) via scripts/fetch_mediawiki_data.rb
+  CATEGORY_NAMESPACES = load_mediawiki_data.dig("namespaces", "category")&.join("|") || "Category"
+
+  # Multilingual file namespace support (for image/file links)
+  FILE_NAMESPACES = load_mediawiki_data.dig("namespaces", "file")&.join("|") || "File|Image"
+  FILE_NAMESPACES_REGEX = Regexp.new('\A\s*(?:' + FILE_NAMESPACES + ')\s*:', Regexp::IGNORECASE)
+
+  # Multilingual category namespace support (for filtering out category links from body text)
+  CATEGORY_NAMESPACE_REGEX = Regexp.new('\A\s*(?:' + CATEGORY_NAMESPACES + ')\s*:', Regexp::IGNORECASE)
+
+  # Image parameters (multilingual) - used for filtering out non-caption parts of File/Image links
+  # Combines: img_thumbnail, img_framed, img_frameless, img_border, img_right, img_left, img_center, img_none,
+  # img_upright, img_baseline, img_sub, img_super, img_top, img_text_top, img_middle, img_bottom, img_text_bottom
+  IMAGE_PARAM_KEYS = %w[img_thumbnail img_framed img_frameless img_border img_right img_left img_center img_none
+                        img_upright img_baseline img_sub img_super img_top img_text_top img_middle img_bottom img_text_bottom].freeze
+  IMAGE_PARAMS = IMAGE_PARAM_KEYS.flat_map { |k| load_mediawiki_data.dig("magic_words", k) || [] }.uniq.join("|")
+  IMAGE_PARAMS_REGEX = IMAGE_PARAMS.empty? ? nil : Regexp.new('\A(' + IMAGE_PARAMS + ')\z', Regexp::IGNORECASE)
+
+  # Cleanup regex patterns using dynamic file namespaces
+  # For lines like "Image:file.jpg|thumb|...|caption" (gallery/imagemap remnants)
+  CLEANUP_FILE_LINE_REGEX = Regexp.new('^(?:' + FILE_NAMESPACES + '):[^\n]+\|[^\n]+$', Regexp::IGNORECASE | Regexp::MULTILINE)
+  # For incomplete File/Image links (opened but not closed)
+  CLEANUP_FILE_INCOMPLETE_REGEX = Regexp.new('\[\[(?:' + FILE_NAMESPACES + '):[^\]]*\|?\s*$', Regexp::IGNORECASE | Regexp::MULTILINE)
+
+  # Category regex - captures category name without sortkey
+  # [[Category:Name|sortkey]] -> captures only "Name" (not "Name|sortkey")
+  # The (?:[^\|\]\}]*) captures the category name up to | or ] or }
+  CATEGORY_REGEX = Regexp.new('[\{\[\|\b](?:' + CATEGORY_NAMESPACES + ')\s*:([^\|\]\}]+)[\|\]\}]', Regexp::IGNORECASE)
 
   ESCAPE_NOWIKI_REGEX = Regexp.new('<nowiki>(.*?)<\/nowiki>', Regexp::MULTILINE)
   UNESCAPE_NOWIKI_REGEX = Regexp.new('<nowiki\-(\d+?)>')
 
   REMOVE_ISOLATED_REGEX = Regexp.new('^\s*\{\{(.*?)\}\}\s*$')
   REMOVE_INLINE_REGEX = Regexp.new('\{\{(.*?)\}\}')
-  TYPE_CODE_REGEX = Regexp.new('\A(?:lang*|\AIPA|IEP|SEP|indent|audio|small|dmoz|pron|unicode|note label|nowrap|ArabDIN|trans|Nihongo|Polytonic)', Regexp::IGNORECASE)
+
+  # Note: TYPE_CODE_REGEX removed (was unused dead code)
+  # Template type detection is now handled by data-driven patterns in template_aliases.json
 
   SINGLE_SQUARE_BRACKET_REGEX = Regexp.new("(#{Regexp.escape("[")}|#{Regexp.escape("]")})", Regexp::MULTILINE)
   DOUBLE_SQUARE_BRACKET_REGEX = Regexp.new("(#{Regexp.escape("[[")}|#{Regexp.escape("]]")})", Regexp::MULTILINE)
@@ -76,6 +265,7 @@ module Wp2txt
   DOUBLE_CURLY_BRACKET_REGEX = Regexp.new("(#{Regexp.escape("{{")}|#{Regexp.escape("}}")})", Regexp::MULTILINE)
   CURLY_SQUARE_BRACKET_REGEX = Regexp.new("(#{Regexp.escape("{|")}|#{Regexp.escape("|}")})", Regexp::MULTILINE)
 
+  SELF_CLOSING_TAG_REGEX = Regexp.new('<[^<>]+/>')
   COMPLEX_REGEX_01 = Regexp.new('\<\<([^<>]++)\>\>\s?')
   COMPLEX_REGEX_02 = Regexp.new('\[\[File\:((?:[^\[\]]++|\[\[\g<1>\]\])++)\]\]', Regexp::MULTILINE | Regexp::IGNORECASE)
   COMPLEX_REGEX_03 = Regexp.new('^\[\[((?:[^\[\]]++|\[\[\g<1>\]\])++)^\]\]', Regexp::MULTILINE)
@@ -90,4 +280,198 @@ module Wp2txt
   CLEANUP_REGEX_06 = Regexp.new('\{\|.*$')
   CLEANUP_REGEX_07 = Regexp.new('^.*\|\}')
   CLEANUP_REGEX_08 = Regexp.new('\n\n\n+', Regexp::MULTILINE)
+
+  # Additional cleanup patterns (pre-compiled for performance)
+  # Mixed whitespace between newlines: \n<spaces/tabs>\n<spaces/tabs>\n+ -> \n\n
+  CLEANUP_MIXED_WHITESPACE_REGEX = Regexp.new('\n[ \t]*\n[ \t]*\n+')
+  # Multiple consecutive spaces (not at line start) -> single space
+  CLEANUP_MULTIPLE_SPACES_REGEX = Regexp.new('([^\n]) {2,}')
+  # Empty parentheses (ASCII and Japanese) - combined for single-pass
+  CLEANUP_EMPTY_PARENS_REGEX = Regexp.new('\(\s*\)|（\s*）')
+  # Multiple pipes (table remnants)
+  CLEANUP_MULTIPLE_PIPES_REGEX = Regexp.new('\|\|+')
+  # Trailing pipe at end of line
+  CLEANUP_TRAILING_PIPE_REGEX = Regexp.new('\|\s*$')
+  # Lines that are just pipe + content (table rows)
+  CLEANUP_PIPE_LINE_REGEX = Regexp.new('^\s*\|[^|]*$\n?', Regexp::MULTILINE)
+  # Lines with multiple pipe-separated key=value pairs (infobox remnants)
+  CLEANUP_KEY_VALUE_LINE_REGEX = Regexp.new('^\s*\|?\w+=[\w\s-]+(?:\|\w+=[\w\s-]+)+\s*$', Regexp::MULTILINE)
+  # Orphaned closing brackets (]] at start of line or after whitespace)
+  CLEANUP_ORPHANED_CLOSE_REGEX = Regexp.new('(?:^|(?<=\s))([^|\[\]\n]+)\]\]')
+  # Orphaned opening wiki brackets not closed on same line
+  CLEANUP_ORPHANED_OPEN_REGEX = Regexp.new('\[\[[^\[\]\n]*$')
+  # Standalone ]] on its own line
+  CLEANUP_STANDALONE_CLOSE_REGEX = Regexp.new('^\s*\]\]\s*$', Regexp::MULTILINE)
+  # Combined pattern for orphaned brackets (both open and standalone close) - single pass removal
+  CLEANUP_ORPHANED_BRACKETS_REGEX = Regexp.new('\[\[[^\[\]\n]*$|^\s*\]\]\s*$', Regexp::MULTILINE)
+  # ]] preceded by pipe without matching [[ (orphaned from broken links)
+  CLEANUP_PIPE_CLOSE_REGEX = Regexp.new('([^|\[\]\n])\|([^|\[\]\n]+)\]\](?!\])')
+  # Multiple blank lines (final cleanup)
+  CLEANUP_MULTI_BLANK_REGEX = Regexp.new('\n{3,}')
+  # Imagemap coordinate remnants (rect, poly, circle, default with coordinates)
+  IMAGEMAP_COORD_REGEX = Regexp.new('^(?:rect|poly|circle|default)\s+[\d\s]+.*$', Regexp::IGNORECASE)
+
+  # =========================================================================
+  # Multilingual cleanup patterns (language-agnostic)
+  # =========================================================================
+
+  # MediaWiki magic words (universal across all wikis)
+  # DEFAULTSORT, DISPLAYTITLE, etc. - loaded from mediawiki_aliases.json for multilingual support
+  DEFAULTSORT_KEYWORDS = load_mediawiki_data.dig("magic_words", "defaultsort")&.join("|") || "DEFAULTSORT"
+  DISPLAYTITLE_KEYWORDS = load_mediawiki_data.dig("magic_words", "displaytitle")&.join("|") || "DISPLAYTITLE"
+
+  # Match bare magic words on their own line: DEFAULTSORT:value or デフォルトソート:value
+  MAGIC_WORD_LINE_REGEX = Regexp.new('^(?:' + DEFAULTSORT_KEYWORDS + '|' + DISPLAYTITLE_KEYWORDS + ')[^\n]*$', Regexp::IGNORECASE)
+
+  # Match magic word template format: {{DEFAULTSORT:value}} or {{デフォルトソート:value}}
+  MAGIC_WORD_TEMPLATE_REGEX = Regexp.new('\{\{\s*(?:' + DEFAULTSORT_KEYWORDS + '|' + DISPLAYTITLE_KEYWORDS + ')[^\}]*\}\}', Regexp::IGNORECASE)
+
+  # Double-underscore magic words: __NOTOC__, __TOC__, __FORCETOC__, __NOEDITSECTION__, etc.
+  # Data source: MediaWiki API (siteinfo) via scripts/fetch_mediawiki_data.rb
+  # Contains 1198 multilingual aliases for behavior switches
+  DOUBLE_UNDERSCORE_PATTERNS = load_mediawiki_data.dig("magic_words", "double_underscore") || []
+  DOUBLE_UNDERSCORE_MAGIC_REGEX = if DOUBLE_UNDERSCORE_PATTERNS.empty?
+    Regexp.new('__[A-Z]+__')  # Fallback to basic pattern
+  else
+    # Build alternation pattern from actual magic word aliases
+    pattern = DOUBLE_UNDERSCORE_PATTERNS.map { |p| Regexp.escape(p) }.join("|")
+    Regexp.new('(?:' + pattern + ')', Regexp::IGNORECASE)
+  end
+
+  # Data-driven pattern for removing double-underscore behavior switches from text
+  # Uses the comprehensive multilingual magic word list (1198 aliases)
+  # Falls back to generic pattern if data file is empty
+  REMOVE_DIRECTIVES_REGEX = DOUBLE_UNDERSCORE_PATTERNS.empty? ?
+    REMOVE_DIRECTIVES_REGEX_GENERIC :
+    DOUBLE_UNDERSCORE_MAGIC_REGEX
+
+  # Interwiki links: :en:Article, :fr:Article, :de:Article, etc.
+  # Removes the prefix but keeps the article name
+  INTERWIKI_PREFIX_REGEX = Regexp.new(':([a-z]{2,3}):(?=[^\s\]]+)')
+
+  # Authority control and metadata templates (standalone lines)
+  # These are template names that appear alone on a line after processing
+  # Data source: template_aliases.json (authority_control category)
+  AUTHORITY_CONTROL_TEMPLATES = load_template_data["authority_control"] || []
+  AUTHORITY_CONTROL_REGEX = if AUTHORITY_CONTROL_TEMPLATES.empty?
+    # Fallback to basic pattern
+    Regexp.new(
+      '^\s*(Normdaten|Authority\s*control|Persondata|VIAF|LCCN|GND)\s*$',
+      Regexp::MULTILINE | Regexp::IGNORECASE
+    )
+  else
+    pattern = AUTHORITY_CONTROL_TEMPLATES.map { |t| Regexp.escape(t) }.join("|")
+    Regexp.new('^\s*(' + pattern + ')\s*$', Regexp::MULTILINE | Regexp::IGNORECASE)
+  end
+
+  # Cleanup remnants - template names that appear as artifacts after processing
+  # Data source: template_aliases.json (cleanup_remnants category)
+  CLEANUP_REMNANTS_TEMPLATES = load_template_data["cleanup_remnants"] || []
+  CLEANUP_REMNANTS_REGEX = if CLEANUP_REMNANTS_TEMPLATES.empty?
+    # Fallback to basic pattern
+    Regexp.new('^\s*(Clear|Clearleft|Clearright|notelist\d*)\s*$', Regexp::MULTILINE | Regexp::IGNORECASE)
+  else
+    pattern = CLEANUP_REMNANTS_TEMPLATES.map { |t| Regexp.escape(t) }.join("|")
+    # Also match notelist with numbers (notelist2, notelist3, etc.)
+    pattern += '|notelist\d+'
+    Regexp.new('^\s*(' + pattern + ')\s*$', Regexp::MULTILINE | Regexp::IGNORECASE)
+  end
+
+  # Category line patterns for all Wikipedia languages
+  # Loaded from mediawiki_aliases.json for complete multilingual support (230+ languages)
+  # Note: Must NOT match "CATEGORIES:" (our summary line)
+  CATEGORY_LINE_REGEX = Regexp.new(
+    '^\s*\*?\s*(?!CATEGORIES)(?:' + CATEGORY_NAMESPACES + '):[^\n]+$',
+    Regexp::MULTILINE | Regexp::IGNORECASE
+  )
+
+  # Wikimedia sister project markers (standalone lines)
+  # Data source: MediaWiki API (siteinfo interwikimap) via scripts/fetch_mediawiki_data.rb
+  # Contains 546 sister project prefixes from all Wikipedia language editions
+  SISTER_PROJECTS = load_mediawiki_data.dig("interwiki", "sister_projects") || []
+  # Filter to only keep known Wikimedia project names (not language codes)
+  WIKIMEDIA_PROJECT_NAMES = %w[
+    wikibooks wikiversity wikisource wikiquote wikinews wiktionary
+    wikivoyage wikispecies wikidata commons meta mediawiki
+    mediawikiwiki species oldwikisource wikifunctions school
+  ].freeze
+  WIKIMEDIA_PROJECT_REGEX = begin
+    # Combine known project names with any from data
+    projects_from_data = SISTER_PROJECTS.select { |p| WIKIMEDIA_PROJECT_NAMES.include?(p.downcase) }
+    # Always include all known project names (ensures complete coverage)
+    all_projects = (WIKIMEDIA_PROJECT_NAMES + projects_from_data).uniq
+    # Add common variations
+    pattern_parts = all_projects.map { |p| Regexp.escape(p) }
+    pattern_parts << 'Wikimedia\s*Commons'  # Common alternate form
+    pattern_parts << 'Commons\s*cat(?:egory)?'  # Commons category template
+    Regexp.new(
+      '^\s*(' + pattern_parts.join("|") + ')(?::|$)',
+      Regexp::MULTILINE | Regexp::IGNORECASE
+    )
+  end
+
+  # Lines that are just a single asterisk (list marker without content)
+  LONE_ASTERISK_REGEX = Regexp.new('^\s*\*\s*$', Regexp::MULTILINE)
+
+  # =========================================================================
+  # Non-article namespace prefixes (for validation filtering)
+  # =========================================================================
+  # These are namespaces that should be excluded from article validation
+  # as they contain templates, portals, help pages, etc. not encyclopedia content
+  #
+  # Data source: MediaWiki API (siteinfo) via scripts/fetch_mediawiki_data.rb
+  # Contains 6083 namespace aliases from 351 Wikipedia language editions
+  NON_ARTICLE_NAMESPACES = (load_mediawiki_data.dig("namespaces", "non_article") || []).freeze
+
+  # Build regex for matching non-article titles
+  # Matches "Namespace:Title" where Namespace is in the list
+  NON_ARTICLE_NAMESPACE_REGEX = if NON_ARTICLE_NAMESPACES.empty?
+    # Fallback to basic English namespaces
+    Regexp.new(
+      '\A\s*(Wikipedia|Template|Portal|Help|Category|File|Image|User|Talk|Module|Draft|MediaWiki)\s*:',
+      Regexp::IGNORECASE
+    )
+  else
+    Regexp.new(
+      '\A\s*(' + NON_ARTICLE_NAMESPACES.map { |ns| Regexp.escape(ns) }.join("|") + ')\s*:',
+      Regexp::IGNORECASE
+    )
+  end
+
+  # Helper method to check if a title is an article page (not a special namespace)
+  def self.article_page?(title)
+    return true if title.nil? || title.empty?
+    !(title =~ NON_ARTICLE_NAMESPACE_REGEX)
+  end
+
+  # =========================================================================
+  # Make constants Ractor-shareable for parallel processing
+  # =========================================================================
+  # This allows Ractor workers to access these constants without isolation errors.
+  # All Regexp and frozen String/Array constants are made shareable.
+
+  # Constants that should NOT be made Ractor-shareable
+  # (they require mutable state or are already shareable)
+  RACTOR_SHAREABLE_EXCLUDES = %i[HTML_DECODER RACTOR_SHAREABLE_EXCLUDES].freeze
+
+  def self.make_constants_ractor_shareable!
+    return unless defined?(Ractor) && Ractor.respond_to?(:make_shareable)
+
+    constants(false).each do |const_name|
+      next if RACTOR_SHAREABLE_EXCLUDES.include?(const_name)
+
+      const = const_get(const_name)
+      next if Ractor.shareable?(const)
+
+      begin
+        Ractor.make_shareable(const)
+      rescue Ractor::IsolationError, FrozenError, TypeError
+        # Some constants can't be made shareable, skip them
+      end
+    end
+  end
+
+  # Make constants shareable when this module is loaded
+  # Excludes constants that require mutable state (like HTML_DECODER)
+  make_constants_ractor_shareable!
 end