wp2txt 1.1.3 → 2.1.0

data/lib/wp2txt/article.rb

@@ -35,7 +35,6 @@ module Wp2txt
       text = text.gsub(/\|\n\n+/m) { "|\n" }
       text = remove_html(text)
       text = make_reference(text)
-      text = remove_ref(text)
       parse text
     end
 
@@ -43,23 +42,97 @@ module Wp2txt
       [tpx, text]
     end
 
+    # Create a heading element with level information
+    # @param text [String] The heading text (with or without = markers)
+    # @param level [Integer] The heading level (2 for ==, 3 for ===, etc.)
+    # @return [Array] [:mw_heading, text, level]
+    def create_heading_element(text, level)
+      [:mw_heading, text, level]
+    end
+
+    # Extract heading level from line with = markers
+    # @param line [String] The heading line (e.g., "== Heading ==")
+    # @return [Integer] The heading level (count of = signs)
+    def extract_heading_level(line)
+      match = line.match(/^(=+)/)
+      match ? match[1].length : 2
+    end
+
+    # Extract clean heading text without = markers
+    # @param line [String] The heading line
+    # @return [String] The heading text without = markers
+    def extract_heading_text(line)
+      line.gsub(/^=+\s*/, "").gsub(/\s*=+$/, "").strip
+    end
+
+    # Check if a line has unbalanced [[ ]] brackets
+    # Returns true if there are more [[ than ]] (indicating multi-line link)
+    def has_unbalanced_link_brackets?(line)
+      open_count = line.scan(/\[\[/).size
+      close_count = line.scan(/\]\]/).size
+      open_count > close_count
+    end
+
+    # Process a line in multi-line template mode, tracking brace depth
+    # Updates @brace_depth and returns remaining content after }} if template closed, nil otherwise
+    def process_ml_template_line(line)
+      pos = 0
+      close_pos = nil
+
+      while pos < line.length
+        open_idx = line.index("{{", pos)
+        close_idx = line.index("}}", pos)
+
+        if open_idx && (!close_idx || open_idx < close_idx)
+          @brace_depth += 1
+          pos = open_idx + 2
+        elsif close_idx
+          @brace_depth -= 1
+          pos = close_idx + 2
+          if @brace_depth == 0
+            close_pos = close_idx + 2
+            break
+          end
+        else
+          break
+        end
+      end
+
+      if close_pos
+        # Template closed at close_pos
+        template_part = line[0...close_pos]
+        remaining = line[close_pos..]
+        @elements.last.last << template_part
+        remaining
+      else
+        # Template continues
+        @elements.last.last << line
+        nil
+      end
+    end
+
     def parse(source)
       @elements = []
       @categories = []
       mode = nil
+      @brace_depth = 0
       source.each_line do |line|
+        # Collect categories without deduplicating on each line (O(n²) → O(n))
         matched = line.scan(CATEGORY_REGEX)
-        if matched && !matched.empty?
-          @categories += matched
-          @categories.uniq!
-        end
+        @categories.concat(matched) if matched && !matched.empty?
 
         case mode
         when :mw_ml_template
-          scanner = StringScanner.new(line)
-          str = process_nested_structure(scanner, "{{", "}}") { "" }
-          mode = nil if ML_TEMPLATE_END_REGEX =~ str
-          @elements.last.last << line
+          # Track brace depth to find where template actually ends
+          remaining = process_ml_template_line(line)
+          if remaining
+            # Template closed, remaining content needs to be processed
+            mode = nil
+            # Process remaining content if any
+            unless remaining.strip.empty?
+              @elements << create_element(:mw_paragraph, "\n" + remaining)
+            end
+          end
           next
         when :mw_ml_link
           scanner = StringScanner.new(line)
@@ -99,16 +172,26 @@ module Wp2txt
         when REDIRECT_REGEX
           @elements << create_element(:mw_redirect, line)
         when IN_HEADING_REGEX
-          line = line.sub(HEADING_ONSET_REGEX) { $1 }.sub(HEADING_CODA_REGEX) { $1 }
-          @elements << create_element(:mw_heading, "\n" + line + "\n")
+          level = extract_heading_level(line)
+          # Keep original format for backward compatibility, but also store level
+          formatted_line = line.sub(HEADING_ONSET_REGEX) { $1 }.sub(HEADING_CODA_REGEX) { $1 }
+          @elements << create_heading_element("\n" + formatted_line + "\n", level)
         when IN_INPUTBOX_REGEX
           @elements << create_element(:mw_inputbox, line)
         when ML_TEMPLATE_ONSET_REGEX
           @elements << create_element(:mw_ml_template, line)
           mode = :mw_ml_template
+          # Count initial braces: count {{ minus }} in this line
+          @brace_depth = line.scan(/\{\{/).size - line.scan(/\}\}/).size
         when ML_LINK_ONSET_REGEX
-          @elements << create_element(:mw_ml_link, line)
-          mode = :mw_ml_link
+          # Only treat as multi-line link if brackets are actually unbalanced
+          if has_unbalanced_link_brackets?(line)
+            @elements << create_element(:mw_ml_link, line)
+            mode = :mw_ml_link
+          else
+            # Brackets are balanced, treat as paragraph
+            @elements << create_element(:mw_paragraph, "\n" + line)
+          end
         when IN_INPUTBOX_REGEX1
           mode = :mw_inputbox
           @elements << create_element(:mw_inputbox, line)
@@ -148,6 +231,8 @@ module Wp2txt
           @elements << create_element(:mw_paragraph, "\n" + line)
         end
       end
+      # Deduplicate categories once at the end (O(n) instead of O(n²))
+      @categories.uniq!
       @elements
     end
   end

data/lib/wp2txt/bz2_validator.rb

@@ -0,0 +1,239 @@
+# frozen_string_literal: true
+
+require_relative "constants"
+
+module Wp2txt
+  # Validates bz2 files for corruption and integrity
+  # Provides early detection of corrupt files before processing
+  module Bz2Validator
+    # Bz2 magic bytes: "BZ" followed by version ('h') and block size ('1'-'9')
+    BZ2_MAGIC = "BZ".freeze
+    BZ2_VERSION = "h".freeze
+    BZ2_BLOCK_SIZES = ("1".."9").to_a.freeze
+
+    # Minimum valid bz2 file size (header + minimal compressed data)
+    MIN_BZ2_SIZE = 14
+
+    # Test chunk size for decompression validation
+    TEST_CHUNK_SIZE = 1_048_576 # 1 MB
+
+    # Validation result structure
+    ValidationResult = Struct.new(:valid, :error_type, :message, :details, keyword_init: true) do
+      def valid?
+        valid
+      end
+
+      def to_s
+        valid ? "Valid bz2 file" : "Invalid: #{message}"
+      end
+    end
+
+    module_function
+
+    # Perform full validation of a bz2 file
+    # @param path [String] Path to bz2 file
+    # @param test_decompress [Boolean] Whether to test decompression (slower but more thorough)
+    # @return [ValidationResult] Validation result
+    def validate(path, test_decompress: true)
+      # Check file exists
+      unless File.exist?(path)
+        return ValidationResult.new(
+          valid: false,
+          error_type: :not_found,
+          message: "File not found",
+          details: { path: path }
+        )
+      end
+
+      # Check file size
+      file_size = File.size(path)
+      if file_size < MIN_BZ2_SIZE
+        return ValidationResult.new(
+          valid: false,
+          error_type: :too_small,
+          message: "File too small to be valid bz2 (#{file_size} bytes)",
+          details: { size: file_size, minimum: MIN_BZ2_SIZE }
+        )
+      end
+
+      # Check magic bytes
+      magic_result = validate_magic_bytes(path)
+      return magic_result unless magic_result.valid?
+
+      # Test decompression if requested
+      if test_decompress
+        decompress_result = test_decompression(path)
+        return decompress_result unless decompress_result.valid?
+      end
+
+      ValidationResult.new(
+        valid: true,
+        error_type: nil,
+        message: "Valid bz2 file",
+        details: { size: file_size, path: path }
+      )
+    end
+
+    # Quick validation (magic bytes only, no decompression test)
+    # @param path [String] Path to bz2 file
+    # @return [ValidationResult] Validation result
+    def validate_quick(path)
+      validate(path, test_decompress: false)
+    end
+
+    # Validate bz2 magic bytes
+    # @param path [String] Path to bz2 file
+    # @return [ValidationResult] Validation result
+    def validate_magic_bytes(path)
+      header = File.binread(path, 4)
+
+      # Check "BZ" signature
+      unless header[0, 2] == BZ2_MAGIC
+        return ValidationResult.new(
+          valid: false,
+          error_type: :invalid_magic,
+          message: "Invalid bz2 header (expected 'BZ', got '#{header[0, 2].inspect}')",
+          details: { expected: BZ2_MAGIC, actual: header[0, 2] }
+        )
+      end
+
+      # Check version byte ('h' for bzip2)
+      unless header[2] == BZ2_VERSION
+        return ValidationResult.new(
+          valid: false,
+          error_type: :invalid_version,
+          message: "Invalid bz2 version byte (expected 'h', got '#{header[2].inspect}')",
+          details: { expected: BZ2_VERSION, actual: header[2] }
+        )
+      end
+
+      # Check block size byte ('1'-'9')
+      unless BZ2_BLOCK_SIZES.include?(header[3])
+        return ValidationResult.new(
+          valid: false,
+          error_type: :invalid_block_size,
+          message: "Invalid bz2 block size (expected '1'-'9', got '#{header[3].inspect}')",
+          details: { expected: BZ2_BLOCK_SIZES, actual: header[3] }
+        )
+      end
+
+      ValidationResult.new(
+        valid: true,
+        error_type: nil,
+        message: "Valid bz2 header",
+        details: { version: header[2], block_size: header[3].to_i }
+      )
+    rescue IOError, Errno::ENOENT, Errno::EACCES => e
+      ValidationResult.new(
+        valid: false,
+        error_type: :read_error,
+        message: "Cannot read file: #{e.message}",
+        details: { error: e.class.name }
+      )
+    end
+
+    # Test decompression of first chunk
+    # @param path [String] Path to bz2 file
+    # @return [ValidationResult] Validation result
+    def test_decompression(path)
+      bzcat_cmd = find_bzip2_command
+      unless bzcat_cmd
+        # Skip decompression test if no command available
+        return ValidationResult.new(
+          valid: true,
+          error_type: nil,
+          message: "Skipped decompression test (no bzip2 command)",
+          details: { skipped: true }
+        )
+      end
+
+      # Try to decompress first chunk
+      begin
+        # Use head to limit output and timeout to prevent hanging on large files
+        output = nil
+        error = nil
+
+        IO.popen([bzcat_cmd, "-c", "-d", path], "rb", err: [:child, :out]) do |io|
+          output = io.read(TEST_CHUNK_SIZE)
+        end
+
+        exit_status = $?.exitstatus
+
+        if exit_status != 0 && (output.nil? || output.empty?)
+          return ValidationResult.new(
+            valid: false,
+            error_type: :decompression_failed,
+            message: "Decompression failed (corrupted data or truncated file)",
+            details: { exit_status: exit_status }
+          )
+        end
+
+        # Check if output looks like XML (Wikipedia dumps are XML)
+        if output && output.bytesize > 0
+          # Simple check for XML-like content
+          sample = output[0, 1000].to_s.scrub("")
+          unless sample.include?("<") && sample.include?(">")
+            return ValidationResult.new(
+              valid: false,
+              error_type: :invalid_content,
+              message: "Decompressed content does not appear to be XML",
+              details: { sample_size: output.bytesize }
+            )
+          end
+        end
+
+        ValidationResult.new(
+          valid: true,
+          error_type: nil,
+          message: "Decompression test passed",
+          details: { bytes_tested: output&.bytesize || 0 }
+        )
+      rescue Errno::EPIPE
+        # Broken pipe is OK - we only read partial output
+        ValidationResult.new(
+          valid: true,
+          error_type: nil,
+          message: "Decompression test passed (partial read)",
+          details: {}
+        )
+      rescue IOError, Errno::ENOENT, Errno::EACCES => e
+        ValidationResult.new(
+          valid: false,
+          error_type: :decompression_error,
+          message: "Decompression error: #{e.message}",
+          details: { error: e.class.name }
+        )
+      end
+    end
+
+    # Find available bzip2 decompression command
+    # @return [String, nil] Path to command or nil
+    def find_bzip2_command
+      %w[lbzip2 pbzip2 bzip2 bzcat].each do |cmd|
+        path = IO.popen(["which", cmd], err: File::NULL, &:read).strip
+        return path unless path.empty?
+      end
+      nil
+    end
+
+    # Get bz2 file information
+    # @param path [String] Path to bz2 file
+    # @return [Hash] File information
+    def file_info(path)
+      return nil unless File.exist?(path)
+
+      header = File.binread(path, 4)
+      {
+        path: path,
+        size: File.size(path),
+        size_formatted: Wp2txt.format_file_size(File.size(path)),
+        valid_header: header[0, 2] == BZ2_MAGIC,
+        version: header[2],
+        block_size: header[3]&.to_i,
+        mtime: File.mtime(path)
+      }
+    rescue IOError, Errno::ENOENT, Errno::EACCES
+      nil
+    end
+  end
+end

data/lib/wp2txt/category_cache.rb

@@ -0,0 +1,313 @@
+# frozen_string_literal: true
+
+require "sqlite3"
+require "json"
+require "fileutils"
+require_relative "constants"
+
+module Wp2txt
+  # SQLite-based cache for Wikipedia category hierarchy and members
+  # Dramatically speeds up repeated category extraction operations
+  class CategoryCache
+    CACHE_VERSION = 1
+    DEFAULT_CACHE_DIR = File.expand_path("~/.wp2txt/cache")
+
+    attr_reader :lang, :cache_path, :expiry_days
+
+    def initialize(lang, cache_dir: nil, expiry_days: nil)
+      @lang = lang.to_s
+      @cache_dir = cache_dir || DEFAULT_CACHE_DIR
+      @expiry_days = expiry_days || DEFAULT_CATEGORY_CACHE_EXPIRY_DAYS
+      @cache_path = File.join(@cache_dir, "categories_#{@lang}.sqlite3")
+      @db = nil
+      ensure_schema
+    end
+
+    # Check if a category is cached and fresh
+    # @param category_name [String] Category name (without "Category:" prefix)
+    # @return [Boolean]
+    def cached?(category_name)
+      open_db
+      row = @db.get_first_row(
+        "SELECT cached_at FROM categories WHERE name = ?",
+        [normalize_name(category_name)]
+      )
+      return false unless row
+
+      cached_at = row[0]
+      return false unless cached_at
+
+      # Check freshness
+      Time.at(cached_at) > Time.now - (@expiry_days * SECONDS_PER_DAY)
+    rescue SQLite3::Exception
+      false
+    end
+
+    # Get category data from cache
+    # @param category_name [String] Category name
+    # @return [Hash, nil] { pages: [...], subcats: [...] } or nil if not cached
+    def get(category_name)
+      return nil unless cached?(category_name)
+
+      name = normalize_name(category_name)
+      open_db
+
+      pages = []
+      subcats = []
+
+      # Get pages
+      @db.execute(
+        "SELECT page_title FROM category_pages WHERE category_name = ?",
+        [name]
+      ) do |row|
+        pages << row[0]
+      end
+
+      # Get subcategories
+      @db.execute(
+        "SELECT child_name FROM category_hierarchy WHERE parent_name = ?",
+        [name]
+      ) do |row|
+        subcats << row[0]
+      end
+
+      { pages: pages, subcats: subcats }
+    rescue SQLite3::Exception
+      nil
+    end
+
+    # Save category data to cache
+    # @param category_name [String] Category name
+    # @param pages [Array<String>] Article titles in this category
+    # @param subcats [Array<String>] Subcategory names
+    def save(category_name, pages, subcats)
+      name = normalize_name(category_name)
+      open_db
+
+      @db.execute("BEGIN TRANSACTION")
+
+      # Update or insert category
+      @db.execute(
+        "INSERT OR REPLACE INTO categories (name, page_count, subcat_count, cached_at) VALUES (?, ?, ?, ?)",
+        [name, pages.size, subcats.size, Time.now.to_i]
+      )
+
+      # Clear old pages and hierarchy
+      @db.execute("DELETE FROM category_pages WHERE category_name = ?", [name])
+      @db.execute("DELETE FROM category_hierarchy WHERE parent_name = ?", [name])
+
+      # Insert pages
+      unless pages.empty?
+        stmt = @db.prepare("INSERT INTO category_pages (category_name, page_title) VALUES (?, ?)")
+        pages.each { |page| stmt.execute([name, page]) }
+        stmt.close
+      end
+
+      # Insert subcategories
+      unless subcats.empty?
+        stmt = @db.prepare("INSERT INTO category_hierarchy (parent_name, child_name) VALUES (?, ?)")
+        subcats.each { |subcat| stmt.execute([name, normalize_name(subcat)]) }
+        stmt.close
+      end
+
+      @db.execute("COMMIT")
+    rescue SQLite3::Exception => e
+      @db&.execute("ROLLBACK") rescue nil
+      warn "CategoryCache: Failed to save #{category_name}: #{e.message}"
+    end
+
+    # Get all pages in a category tree (recursive)
+    # @param category_name [String] Root category name
+    # @param max_depth [Integer] Maximum recursion depth (0 = no recursion)
+    # @param visited [Set] Already visited categories (for cycle detection)
+    # @return [Array<String>] All article titles
+    def get_all_pages(category_name, max_depth: 0, visited: nil)
+      visited ||= Set.new
+      name = normalize_name(category_name)
+      return [] if visited.include?(name)
+
+      visited << name
+      data = get(name)
+      return [] unless data
+
+      pages = data[:pages].dup
+
+      if max_depth > 0
+        data[:subcats].each do |subcat|
+          pages.concat(get_all_pages(subcat, max_depth: max_depth - 1, visited: visited))
+        end
+      end
+
+      pages.uniq
+    end
+
+    # Get category tree structure
+    # @param category_name [String] Root category name
+    # @param max_depth [Integer] Maximum recursion depth
+    # @return [Hash] Tree structure with category info
+    def get_tree(category_name, max_depth: 0)
+      build_tree(category_name, max_depth, Set.new)
+    end
+
+    # Get statistics for all cached categories
+    # @return [Hash] Statistics
+    def stats
+      open_db
+
+      total_categories = @db.get_first_value("SELECT COUNT(*) FROM categories")
+      total_pages = @db.get_first_value("SELECT COUNT(*) FROM category_pages")
+      total_relations = @db.get_first_value("SELECT COUNT(*) FROM category_hierarchy")
+
+      oldest_cache = @db.get_first_value("SELECT MIN(cached_at) FROM categories")
+      newest_cache = @db.get_first_value("SELECT MAX(cached_at) FROM categories")
+
+      {
+        lang: @lang,
+        cache_path: @cache_path,
+        cache_size: File.exist?(@cache_path) ? File.size(@cache_path) : 0,
+        total_categories: total_categories || 0,
+        total_pages: total_pages || 0,
+        total_relations: total_relations || 0,
+        oldest_cache: oldest_cache ? Time.at(oldest_cache) : nil,
+        newest_cache: newest_cache ? Time.at(newest_cache) : nil,
+        expiry_days: @expiry_days
+      }
+    rescue SQLite3::Exception
+      { lang: @lang, error: "Failed to read stats" }
+    end
+
+    # Clear all cached data
+    def clear!
+      close_db
+      FileUtils.rm_f(@cache_path)
+      ensure_schema
+    end
+
+    # Clear expired entries
+    def cleanup_expired!
+      open_db
+      cutoff = Time.now.to_i - (@expiry_days * SECONDS_PER_DAY)
+
+      @db.execute("BEGIN TRANSACTION")
+
+      # Get expired categories
+      expired = []
+      @db.execute("SELECT name FROM categories WHERE cached_at < ?", [cutoff]) do |row|
+        expired << row[0]
+      end
+
+      # Delete expired data
+      expired.each do |name|
+        @db.execute("DELETE FROM category_pages WHERE category_name = ?", [name])
+        @db.execute("DELETE FROM category_hierarchy WHERE parent_name = ?", [name])
+        @db.execute("DELETE FROM categories WHERE name = ?", [name])
+      end
+
+      @db.execute("COMMIT")
+
+      expired.size
+    rescue SQLite3::Exception
+      @db&.execute("ROLLBACK") rescue nil
+      0
+    end
+
+    # Close database connection
+    def close
+      close_db
+    end
+
+    private
+
+    def normalize_name(name)
+      name.to_s.sub(/^[Cc]ategory:/, "").strip
+    end
+
+    def build_tree(category_name, max_depth, visited)
+      name = normalize_name(category_name)
+      return nil if visited.include?(name)
+
+      visited << name
+      data = get(name)
+
+      result = {
+        name: name,
+        cached: !data.nil?,
+        page_count: data ? data[:pages].size : 0,
+        children: []
+      }
+
+      if data && max_depth > 0
+        data[:subcats].each do |subcat|
+          child = build_tree(subcat, max_depth - 1, visited)
+          result[:children] << child if child
+        end
+      end
+
+      result
+    end
+
+    def open_db
+      return if @db
+
+      FileUtils.mkdir_p(File.dirname(@cache_path))
+      @db = SQLite3::Database.new(@cache_path)
+      @db.execute("PRAGMA journal_mode = WAL")
+      @db.execute("PRAGMA synchronous = NORMAL")
+      @db.execute("PRAGMA cache_size = -16000") # 16MB cache
+    end
+
+    def close_db
+      @db&.close
+      @db = nil
+    end
+
+    def ensure_schema
+      open_db
+
+      @db.execute(<<~SQL)
+        CREATE TABLE IF NOT EXISTS categories (
+          name TEXT PRIMARY KEY,
+          page_count INTEGER DEFAULT 0,
+          subcat_count INTEGER DEFAULT 0,
+          cached_at INTEGER
+        )
+      SQL
+
+      @db.execute(<<~SQL)
+        CREATE TABLE IF NOT EXISTS category_pages (
+          category_name TEXT NOT NULL,
+          page_title TEXT NOT NULL,
+          PRIMARY KEY (category_name, page_title)
+        )
+      SQL
+
+      @db.execute("CREATE INDEX IF NOT EXISTS idx_category_pages_category ON category_pages(category_name)")
+
+      @db.execute(<<~SQL)
+        CREATE TABLE IF NOT EXISTS category_hierarchy (
+          parent_name TEXT NOT NULL,
+          child_name TEXT NOT NULL,
+          PRIMARY KEY (parent_name, child_name)
+        )
+      SQL
+
+      @db.execute("CREATE INDEX IF NOT EXISTS idx_hierarchy_parent ON category_hierarchy(parent_name)")
+      @db.execute("CREATE INDEX IF NOT EXISTS idx_hierarchy_child ON category_hierarchy(child_name)")
+
+      @db.execute(<<~SQL)
+        CREATE TABLE IF NOT EXISTS metadata (
+          key TEXT PRIMARY KEY,
+          value TEXT
+        )
+      SQL
+
+      # Store cache version
+      @db.execute(
+        "INSERT OR REPLACE INTO metadata (key, value) VALUES ('cache_version', ?)",
+        [CACHE_VERSION.to_s]
+      )
+    rescue SQLite3::Exception => e
+      warn "CategoryCache: Failed to create schema: #{e.message}"
+    end
+  end
+end