wp2txt 1.1.3 → 2.1.0

data/lib/wp2txt/section_extractor.rb

@@ -0,0 +1,354 @@
+# frozen_string_literal: true
+
+require "yaml"
+require_relative "constants"
+
+module Wp2txt
+  # SectionExtractor handles extraction of sections from Wikipedia articles
+  # Supports both metadata extraction (headings only) and content extraction
+  class SectionExtractor
+    # Reserved keyword for the lead section (text before first heading)
+    SUMMARY_KEY = "summary"
+
+    # Default section aliases (canonical name => array of aliases)
+    DEFAULT_ALIASES = {
+      "Plot" => ["Synopsis"],
+      "Reception" => ["Critical reception"]
+    }.freeze
+
+    # Track which actual headings matched which requested sections
+    attr_reader :matched_sections
+
+    # @param target_sections [Array<String>, nil] List of section names to extract (nil = all)
+    # @param options [Hash] Extraction options
+    # @option options [Integer] :min_length Minimum section length (default: 0)
+    # @option options [Boolean] :skip_empty Skip articles with no matching sections (default: false)
+    # @option options [Hash] :aliases Custom section aliases (merged with defaults)
+    # @option options [String] :alias_file Path to YAML file with custom aliases
+    # @option options [Boolean] :use_aliases Enable alias matching (default: true)
+    # @option options [Boolean] :track_matches Track which headings matched (default: false)
+    def initialize(target_sections = nil, options = {})
+      @targets = normalize_targets(target_sections)
+      @min_length = options[:min_length] || 0
+      @skip_empty = options[:skip_empty] || false
+      @use_aliases = options.fetch(:use_aliases, true)
+      @track_matches = options[:track_matches] || false
+      @matched_sections = {}
+      @aliases = build_aliases(options[:aliases], options[:alias_file])
+    end
+
+    # Load aliases from YAML file
+    # @param file_path [String] Path to YAML file
+    # @return [Hash] Aliases hash (canonical => [aliases])
+    def self.load_aliases_from_file(file_path)
+      return {} unless file_path && File.exist?(file_path)
+
+      data = YAML.safe_load(File.read(file_path), permitted_classes: [Symbol])
+      return {} unless data.is_a?(Hash)
+
+      # Normalize: ensure values are arrays
+      data.transform_values { |v| Array(v) }
+    rescue Psych::SyntaxError, Errno::ENOENT
+      {}
+    end
+
+    # Extract section headings from article (for --metadata-only)
+    # @param article [Article] The article to extract from
+    # @return [Array<String>] List of section heading names
+    def extract_headings(article)
+      headings = []
+      article.elements.each do |element|
+        next unless element[0] == :mw_heading
+
+        heading_text = clean_heading_text(element[1])
+        headings << heading_text unless heading_text.empty?
+      end
+      headings
+    end
+
+    # Extract section headings with levels (for detailed analysis)
+    # @param article [Article] The article to extract from
+    # @return [Array<Hash>] List of {name:, level:} hashes
+    def extract_headings_with_levels(article)
+      headings = []
+      article.elements.each do |element|
+        next unless element[0] == :mw_heading
+
+        heading_text = clean_heading_text(element[1])
+        level = element[2] || 2
+        headings << { name: heading_text, level: level } unless heading_text.empty?
+      end
+      headings
+    end
+
+    # Extract summary (lead section) from article
+    # @param article [Article] The article to extract from
+    # @param config [Hash] Formatting configuration
+    # @return [String, nil] The summary text or nil if empty
+    def extract_summary(article, config = {})
+      contents = +""
+      article.elements.each do |element|
+        # Stop at first heading
+        break if element[0] == :mw_heading
+
+        # Skip non-content elements
+        next if %i[mw_blank mw_redirect mw_comment].include?(element[0])
+
+        content = element[1].to_s
+        contents << content
+      end
+
+      result = contents.strip
+      result.empty? ? nil : result
+    end
+
+    # Extract specified sections from article
+    # @param article [Article] The article to extract from
+    # @param config [Hash] Formatting configuration
+    # @return [Hash] Section name => content (nil if not found)
+    def extract_sections(article, config = {})
+      return {} if @targets.nil? || @targets.empty?
+
+      # Reset matched sections for this article
+      @matched_sections = {}
+
+      result = {}
+
+      # Initialize all targets with nil
+      @targets.each { |t| result[t] = nil }
+
+      # Handle summary separately
+      if @targets.include?(SUMMARY_KEY)
+        summary = extract_summary(article, config)
+        result[SUMMARY_KEY] = apply_min_length_filter(summary)
+      end
+
+      # Extract other sections
+      current_section = nil
+      current_level = nil
+      buffer = +""
+
+      article.elements.each do |element|
+        type = element[0]
+        content = element[1]
+        level = element[2]
+
+        if type == :mw_heading
+          # Save previous section if it was a target
+          if current_section
+            canonical = find_canonical_name(current_section)
+            if canonical && canonical != SUMMARY_KEY
+              result[canonical] = apply_min_length_filter(buffer.strip)
+            end
+          end
+
+          # Check if this heading is a target
+          heading_text = clean_heading_text(content)
+          canonical = find_canonical_name(heading_text)
+
+          if canonical && canonical != SUMMARY_KEY
+            current_section = heading_text
+            current_level = level || 2
+            buffer = +""
+          elsif current_level && (level.nil? || level <= current_level)
+            # Same or higher level heading ends current section
+            current_section = nil
+            current_level = nil
+            buffer = +""
+          end
+        elsif current_section
+          # Accumulate content for current section
+          buffer << content.to_s
+        end
+      end
+
+      # Save final section
+      if current_section
+        canonical = find_canonical_name(current_section)
+        if canonical && canonical != SUMMARY_KEY
+          result[canonical] = apply_min_length_filter(buffer.strip)
+        end
+      end
+
+      result
+    end
+
+    # Check if article has any matching sections
+    # @param article [Article] The article to check
+    # @return [Boolean] true if at least one target section exists
+    def has_matching_sections?(article)
+      return true if @targets.nil? || @targets.empty?
+
+      # Check summary
+      if @targets.include?(SUMMARY_KEY)
+        summary = extract_summary(article)
+        return true if summary && !summary.empty?
+      end
+
+      # Check headings (don't record matches during check)
+      headings = extract_headings(article)
+      headings.any? { |h| find_canonical_name(h, record_match: false) }
+    end
+
+    # Check if extraction should be skipped for this article
+    # @param article [Article] The article to check
+    # @return [Boolean] true if article should be skipped
+    def should_skip?(article)
+      return false unless @skip_empty
+      !has_matching_sections?(article)
+    end
+
+    private
+
+    # Normalize target section names
+    def normalize_targets(targets)
+      return nil if targets.nil?
+
+      Array(targets).map { |t| t.to_s.strip }.reject(&:empty?)
+    end
+
+    # Build aliases hash from options, file, and defaults
+    def build_aliases(custom_aliases, alias_file = nil)
+      return {} unless @use_aliases
+
+      aliases = DEFAULT_ALIASES.dup
+
+      # Load from file if specified
+      if alias_file
+        file_aliases = self.class.load_aliases_from_file(alias_file)
+        aliases.merge!(file_aliases)
+      end
+
+      # Merge inline custom aliases
+      aliases.merge!(custom_aliases) if custom_aliases.is_a?(Hash)
+      aliases
+    end
+
+    # Clean heading text by removing = markers and whitespace
+    def clean_heading_text(text)
+      text.to_s.gsub(/^[\s\n]*=+\s*/, "").gsub(/\s*=+[\s\n]*$/, "").strip
+    end
+
+    # Find canonical name for a heading (handles aliases)
+    # @param heading [String] The actual heading text from the article
+    # @param record_match [Boolean] Whether to record the match for tracking
+    # @return [String, nil] The canonical (requested) section name, or nil
+    def find_canonical_name(heading, record_match: true)
+      return nil if heading.nil? || heading.empty?
+      return nil if @targets.nil?
+
+      heading_lower = heading.downcase.strip
+
+      # Direct match
+      @targets.each do |target|
+        if target.downcase == heading_lower
+          # Record direct match (only if heading differs in case)
+          if @track_matches && record_match && target != heading
+            @matched_sections[target] = heading
+          end
+          return target
+        end
+      end
+
+      # Alias match
+      return nil unless @use_aliases
+
+      @aliases.each do |canonical, alias_list|
+        next unless @targets.any? { |t| t.downcase == canonical.downcase }
+
+        if alias_list.any? { |a| a.downcase == heading_lower }
+          # Return the target that matches canonical
+          target = @targets.find { |t| t.downcase == canonical.downcase }
+          # Record alias match
+          if @track_matches && record_match && target
+            @matched_sections[target] = heading
+          end
+          return target
+        end
+      end
+
+      nil
+    end
+
+    # Apply minimum length filter
+    def apply_min_length_filter(text)
+      return nil if text.nil?
+      return nil if @min_length > 0 && text.length < @min_length
+
+      text
+    end
+  end
+
+  # Collects section heading statistics across multiple articles
+  # Used for --section-stats mode
+  class SectionStatsCollector
+    attr_reader :total_articles, :section_counts
+
+    def initialize
+      @total_articles = 0
+      @section_counts = Hash.new(0)
+      @extractor = SectionExtractor.new
+    end
+
+    # Process an article and collect section heading statistics
+    # @param article [Article] The article to process
+    def process(article)
+      @total_articles += 1
+      headings = @extractor.extract_headings(article)
+      headings.each { |h| @section_counts[h] += 1 }
+    end
+
+    # Get top N sections by count
+    # @param n [Integer] Number of sections to return (default: 50)
+    # @return [Array<Hash>] Array of {name:, count:} hashes
+    def top_sections(n = 50)
+      @section_counts
+        .sort_by { |_name, count| -count }
+        .first(n)
+        .map { |name, count| { "name" => name, "count" => count } }
+    end
+
+    # Generate statistics output as a hash
+    # @param top_n [Integer] Number of top sections to include
+    # @return [Hash] Statistics hash
+    def to_hash(top_n: DEFAULT_TOP_N_SECTIONS)
+      {
+        "total_articles" => @total_articles,
+        "section_counts" => @section_counts.sort_by { |_k, v| -v }.to_h,
+        "top_sections" => top_sections(top_n)
+      }
+    end
+
+    # Generate JSON output
+    # @param top_n [Integer] Number of top sections to include
+    # @return [String] JSON string
+    def to_json(top_n: DEFAULT_TOP_N_SECTIONS)
+      require "json"
+      JSON.pretty_generate(to_hash(top_n: top_n))
+    end
+
+    # Merge another collector's results into this one
+    # Used for combining results from parallel processing
+    # @param other [SectionStatsCollector, Hash] Another collector or hash with results
+    def merge(other)
+      if other.is_a?(SectionStatsCollector)
+        @total_articles += other.total_articles
+        other.section_counts.each { |name, count| @section_counts[name] += count }
+      elsif other.is_a?(Hash)
+        @total_articles += other[:total_articles] || other["total_articles"] || 0
+        counts = other[:section_counts] || other["section_counts"] || {}
+        counts.each { |name, count| @section_counts[name] += count }
+      end
+      self
+    end
+
+    # Export current state as a hash (for parallel processing)
+    # @return [Hash] Hash with total_articles and section_counts
+    def to_mergeable_hash
+      {
+        total_articles: @total_articles,
+        section_counts: @section_counts.dup
+      }
+    end
+  end
+end

data/lib/wp2txt/stream_processor.rb

@@ -0,0 +1,271 @@
+# frozen_string_literal: true
+
+require "nokogiri"
+require "stringio"
+require_relative "constants"
+require_relative "memory_monitor"
+require_relative "bz2_validator"
+
+module Wp2txt
+  # StreamProcessor handles streaming decompression and XML parsing
+  # without creating intermediate files
+  class StreamProcessor
+    include Wp2txt
+
+    # Buffer size bounds from constants
+    MIN_BUFFER_SIZE = Wp2txt::MIN_BUFFER_SIZE
+    MAX_BUFFER_SIZE = Wp2txt::MAX_BUFFER_SIZE
+    DEFAULT_BUFFER_SIZE = Wp2txt::DEFAULT_BUFFER_SIZE
+
+    attr_reader :buffer_size, :pages_processed, :bytes_read, :redirects_skipped
+
+    def initialize(input_path, bz2_gem: false, adaptive_buffer: true, validate_bz2: true, skip_redirects: true)
+      @input_path = input_path
+      @bz2_gem = bz2_gem
+      @buffer = +""
+      @file_pointer = nil
+      @adaptive_buffer = adaptive_buffer
+      @buffer_size = adaptive_buffer ? calculate_optimal_buffer_size : DEFAULT_BUFFER_SIZE
+      @pages_processed = 0
+      @bytes_read = 0
+      @validate_bz2 = validate_bz2
+      @skip_redirects = skip_redirects
+      @redirects_skipped = 0
+    end
+
+    # Validate bz2 file before processing
+    # @param quick [Boolean] Use quick validation (header only) vs full validation
+    # @return [Bz2Validator::ValidationResult] Validation result
+    # @raise [Wp2txt::FileIOError] If validation fails and raise_on_error is true
+    def validate_input(quick: false, raise_on_error: false)
+      return nil unless @input_path.end_with?(".bz2")
+
+      result = quick ? Bz2Validator.validate_quick(@input_path) : Bz2Validator.validate(@input_path)
+
+      if !result.valid? && raise_on_error
+        raise Wp2txt::FileIOError, "Invalid bz2 file: #{result.message}"
+      end
+
+      result
+    end
+
+    # Calculate optimal buffer size based on available memory
+    def calculate_optimal_buffer_size
+      MemoryMonitor.optimal_buffer_size
+    rescue StandardError
+      DEFAULT_BUFFER_SIZE
+    end
+
+    # Get current memory statistics
+    def memory_stats
+      MemoryMonitor.memory_stats
+    end
+
+    # Iterate over each page in the input
+    # Yields [title, text] for each page
+    def each_page
+      return enum_for(:each_page) unless block_given?
+
+      if File.directory?(@input_path)
+        # Process XML files in directory
+        Dir.glob(File.join(@input_path, "*.xml")).sort.each do |xml_file|
+          process_xml_file(xml_file) { |title, text| yield title, text }
+        end
+      elsif @input_path.end_with?(".bz2")
+        # Process bz2 compressed file with streaming
+        process_bz2_stream { |title, text| yield title, text }
+      elsif @input_path.end_with?(".xml")
+        # Process single XML file
+        process_xml_file(@input_path) { |title, text| yield title, text }
+      else
+        raise ArgumentError, "Unsupported input format: #{@input_path}"
+      end
+    end
+
+    # Get processing statistics (public API for monitoring)
+    def stats
+      {
+        pages_processed: @pages_processed,
+        redirects_skipped: @redirects_skipped,
+        bytes_read: @bytes_read,
+        buffer_size: @buffer_size,
+        current_buffer_length: @buffer.bytesize,
+        memory: memory_stats
+      }
+    end
+
+    private
+
+    # Process a single XML file
+    def process_xml_file(xml_file)
+      @buffer = +""
+      @file_pointer = File.open(xml_file, "r:UTF-8")
+
+      while (page = extract_next_page)
+        result = parse_page_xml(page)
+        yield result if result
+      end
+
+      @file_pointer.close
+    end
+
+    # Process bz2 stream directly without intermediate files
+    def process_bz2_stream
+      # Validate bz2 file before processing (if enabled)
+      if @validate_bz2
+        validation = validate_input(quick: false)
+        unless validation.nil? || validation.valid?
+          raise Wp2txt::FileIOError, "Cannot process corrupted bz2 file: #{validation.message}"
+        end
+      end
+
+      @buffer = +""
+      @file_pointer = open_bz2_stream
+
+      while (page = extract_next_page)
+        result = parse_page_xml(page)
+        yield result if result
+      end
+
+      @file_pointer.close
+    rescue Errno::EPIPE
+      # Ignore broken pipe (can happen if we stop reading early)
+    end
+
+    # Open bz2 stream using external command or gem
+    def open_bz2_stream
+      if @bz2_gem
+        require "bzip2-ruby"
+        Bzip2::Reader.new(File.open(@input_path, "rb"))
+      elsif Gem.win_platform?
+        IO.popen(["bunzip2.exe", "-c", @input_path], "rb")
+      else
+        bzpath = find_bzip2_command
+        raise "No bzip2 decompression command found" unless bzpath
+        IO.popen([bzpath, "-c", "-d", @input_path], "rb")
+      end
+    end
+
+    # Find available bzip2 command
+    def find_bzip2_command
+      %w[lbzip2 pbzip2 bzip2].each do |cmd|
+        path = IO.popen(["which", cmd], err: File::NULL, &:read).strip
+        return path unless path.empty?
+      end
+      nil
+    end
+
+    # Fill buffer from file pointer
+    def fill_buffer
+      chunk = @file_pointer.read(@buffer_size)
+      return false unless chunk
+
+      @bytes_read += chunk.bytesize
+
+      # Handle encoding for bz2 streams
+      chunk = chunk.force_encoding("UTF-8")
+      chunk = chunk.scrub("")
+      @buffer << chunk
+
+      # Adaptive buffer adjustment: if memory is low, reduce buffer size
+      if @adaptive_buffer && MemoryMonitor.memory_low?
+        new_size = [@buffer_size / 2, MIN_BUFFER_SIZE].max
+        @buffer_size = new_size if new_size != @buffer_size
+      end
+
+      true
+    end
+
+    # Extract next <page>...</page> from buffer
+    def extract_next_page
+      loop do
+        # Look for complete page in buffer
+        start_idx = @buffer.index("<page>")
+        if start_idx
+          end_idx = @buffer.index("</page>", start_idx)
+          if end_idx
+            # Extract the complete page
+            page_end = end_idx + "</page>".length
+            page = @buffer[start_idx...page_end]
+            @buffer = @buffer[page_end..]
+            return page
+          end
+        end
+
+        # Need more data
+        break unless fill_buffer
+      end
+
+      # Check for remaining page in buffer (end of file)
+      start_idx = @buffer.index("<page>")
+      return nil unless start_idx
+
+      end_idx = @buffer.index("</page>", start_idx)
+      return nil unless end_idx
+
+      page_end = end_idx + "</page>".length
+      page = @buffer[start_idx...page_end]
+      @buffer = @buffer[page_end..]
+      page
+    end
+
+    # Parse page XML and extract title and text
+    def parse_page_xml(page_xml)
+      # Wrap in minimal mediawiki element for parsing
+      xmlns = '<mediawiki xmlns="http://www.mediawiki.org/xml/export-0.5/">'
+      xml = xmlns + page_xml + "</mediawiki>"
+
+      doc = Nokogiri::XML(xml, nil, "UTF-8")
+      text_node = doc.xpath("//xmlns:text").first
+      return nil unless text_node
+
+      title_node = text_node.parent.parent.at_css("title")
+      return nil unless title_node
+
+      title = title_node.content
+      # Skip special pages (containing colon in title like "Wikipedia:", "File:", etc.)
+      return nil if title.include?(":")
+
+      text = text_node.content
+
+      # Early redirect detection and skip (before expensive processing)
+      # Redirects start with # or ＃ followed by redirect keyword and [[target]]
+      if @skip_redirects && redirect_page?(text)
+        @redirects_skipped += 1
+        return nil
+      end
+
+      # Remove HTML comments while preserving newline count
+      text = text.gsub(/<!--(.*?)-->/m) do |content|
+        num_newlines = content.count("\n")
+        num_newlines.zero? ? "" : "\n" * num_newlines
+      end
+
+      @pages_processed += 1
+      [title, text]
+    rescue Nokogiri::XML::SyntaxError
+      # Skip malformed XML
+      nil
+    end
+
+    # Fast redirect detection using heuristic check
+    # Checks if text starts with redirect pattern without full regex evaluation
+    # @param text [String] The page text content
+    # @return [Boolean] true if page appears to be a redirect
+    def redirect_page?(text)
+      return false if text.nil? || text.empty?
+
+      # Check first 200 characters for redirect pattern
+      # Redirects are always at the start: #REDIRECT [[Target]] or #転送 [[ターゲット]]
+      first_part = text[0, 200]
+      return false unless first_part
+
+      # Quick check: must start with # or ＃ (after optional whitespace)
+      stripped = first_part.lstrip
+      return false unless stripped.start_with?("#", "＃")
+
+      # Must contain [[ which indicates the redirect target
+      stripped.include?("[[")
+    end
+  end
+end