wp2txt 1.1.3 → 2.1.0

data/lib/wp2txt/index_cache.rb

@@ -0,0 +1,258 @@
+# frozen_string_literal: true
+
+require "sqlite3"
+require "fileutils"
+require "digest"
+
+module Wp2txt
+  # SQLite-based cache for multistream index
+  # Dramatically speeds up repeated index access by storing parsed entries in SQLite
+  class IndexCache
+    CACHE_VERSION = 1
+    CACHE_SUFFIX = ".sqlite3"
+
+    attr_reader :cache_path, :source_path
+
+    def initialize(source_path, cache_dir: nil)
+      @source_path = source_path
+      @cache_dir = cache_dir || default_cache_dir
+      @cache_path = build_cache_path
+      @db = nil
+    end
+
+    # Check if cache exists and is valid for the current source file
+    # @return [Boolean] true if cache is usable
+    def valid?
+      return false unless File.exist?(@cache_path)
+      return false unless File.exist?(@source_path)
+
+      begin
+        open_db
+        meta = load_metadata
+        return false unless meta
+
+        # Check cache version
+        return false if meta[:cache_version].to_i != CACHE_VERSION
+
+        # Check source file hasn't changed
+        source_stat = File.stat(@source_path)
+        return false if meta[:source_mtime].to_i != source_stat.mtime.to_i
+        return false if meta[:source_size].to_i != source_stat.size
+
+        true
+      rescue SQLite3::Exception
+        false
+      ensure
+        close_db
+      end
+    end
+
+    # Load index entries from cache
+    # @return [Hash] { entries_by_title: {}, entries_by_id: {}, stream_offsets: [] }
+    def load
+      return nil unless valid?
+
+      entries_by_title = {}
+      entries_by_id = {}
+      stream_offsets = []
+
+      open_db
+      begin
+        # Load all entries
+        @db.execute("SELECT title, page_id, byte_offset FROM index_entries") do |row|
+          title, page_id, offset = row
+          entry = { offset: offset, page_id: page_id, title: title }
+          entries_by_title[title] = entry
+          entries_by_id[page_id] = entry
+        end
+
+        # Load stream offsets
+        @db.execute("SELECT byte_offset FROM stream_offsets ORDER BY byte_offset") do |row|
+          stream_offsets << row[0]
+        end
+
+        { entries_by_title: entries_by_title, entries_by_id: entries_by_id, stream_offsets: stream_offsets }
+      ensure
+        close_db
+      end
+    end
+
+    # Save index entries to cache
+    # @param entries_by_title [Hash] title => entry hash
+    # @param stream_offsets [Array<Integer>] sorted stream offsets
+    def save(entries_by_title, stream_offsets)
+      FileUtils.mkdir_p(File.dirname(@cache_path))
+
+      # Remove old cache if exists
+      FileUtils.rm_f(@cache_path)
+
+      open_db
+      begin
+        create_schema
+
+        # Use transaction for better performance
+        @db.execute("BEGIN TRANSACTION")
+
+        # Save metadata
+        source_stat = File.stat(@source_path)
+        save_metadata(
+          source_path: @source_path,
+          source_mtime: source_stat.mtime.to_i,
+          source_size: source_stat.size,
+          cache_version: CACHE_VERSION,
+          entry_count: entries_by_title.size
+        )
+
+        # Save entries in batches for performance
+        stmt = @db.prepare("INSERT INTO index_entries (title, page_id, byte_offset) VALUES (?, ?, ?)")
+        entries_by_title.each do |title, entry|
+          stmt.execute([title, entry[:page_id], entry[:offset]])
+        end
+        stmt.close
+
+        # Save stream offsets
+        stmt = @db.prepare("INSERT INTO stream_offsets (byte_offset) VALUES (?)")
+        stream_offsets.each do |offset|
+          stmt.execute([offset])
+        end
+        stmt.close
+
+        @db.execute("COMMIT")
+
+        true
+      rescue SQLite3::Exception => e
+        @db.execute("ROLLBACK") rescue nil
+        FileUtils.rm_f(@cache_path)
+        raise e
+      ensure
+        close_db
+      end
+    end
+
+    # Find entries by titles (efficient batch lookup)
+    # @param titles [Array<String>] titles to look up
+    # @return [Hash] title => entry or nil
+    def find_by_titles(titles)
+      return {} if titles.empty?
+      return {} unless valid?
+
+      results = {}
+      open_db
+      begin
+        # Use IN clause with placeholders for batch lookup
+        placeholders = titles.map { "?" }.join(",")
+        sql = "SELECT title, page_id, byte_offset FROM index_entries WHERE title IN (#{placeholders})"
+
+        # SQLite3 2.x requires bind variables as an array
+        @db.execute(sql, titles) do |row|
+          title, page_id, offset = row
+          results[title] = { offset: offset, page_id: page_id, title: title }
+        end
+
+        results
+      ensure
+        close_db
+      end
+    end
+
+    # Get cache statistics
+    def stats
+      return nil unless File.exist?(@cache_path)
+
+      open_db
+      begin
+        meta = load_metadata
+        entry_count = @db.get_first_value("SELECT COUNT(*) FROM index_entries")
+        stream_count = @db.get_first_value("SELECT COUNT(*) FROM stream_offsets")
+
+        {
+          cache_path: @cache_path,
+          cache_size: File.size(@cache_path),
+          entry_count: entry_count,
+          stream_count: stream_count,
+          source_path: meta[:source_path],
+          source_mtime: meta[:source_mtime] ? Time.at(meta[:source_mtime].to_i) : nil,
+          cache_version: meta[:cache_version]
+        }
+      ensure
+        close_db
+      end
+    end
+
+    # Delete cache file
+    def clear!
+      FileUtils.rm_f(@cache_path)
+    end
+
+    private
+
+    def default_cache_dir
+      File.expand_path("~/.wp2txt/cache")
+    end
+
+    def build_cache_path
+      # Use source file basename + hash of full path for uniqueness
+      basename = File.basename(@source_path, ".*").sub(/-index$/, "")
+      path_hash = Digest::MD5.hexdigest(@source_path)[0, 8]
+      File.join(@cache_dir, "#{basename}_#{path_hash}#{CACHE_SUFFIX}")
+    end
+
+    def open_db
+      @db ||= SQLite3::Database.new(@cache_path)
+      # Performance optimizations
+      @db.execute("PRAGMA journal_mode = WAL")
+      @db.execute("PRAGMA synchronous = NORMAL")
+      @db.execute("PRAGMA cache_size = -64000") # 64MB cache
+    end
+
+    def close_db
+      @db&.close
+      @db = nil
+    end
+
+    def create_schema
+      @db.execute(<<~SQL)
+        CREATE TABLE IF NOT EXISTS metadata (
+          key TEXT PRIMARY KEY,
+          value TEXT
+        )
+      SQL
+
+      @db.execute(<<~SQL)
+        CREATE TABLE IF NOT EXISTS index_entries (
+          title TEXT PRIMARY KEY,
+          page_id INTEGER,
+          byte_offset INTEGER
+        )
+      SQL
+
+      @db.execute("CREATE INDEX IF NOT EXISTS idx_page_id ON index_entries(page_id)")
+      @db.execute("CREATE INDEX IF NOT EXISTS idx_byte_offset ON index_entries(byte_offset)")
+
+      @db.execute(<<~SQL)
+        CREATE TABLE IF NOT EXISTS stream_offsets (
+          byte_offset INTEGER PRIMARY KEY
+        )
+      SQL
+    end
+
+    def save_metadata(hash)
+      stmt = @db.prepare("INSERT OR REPLACE INTO metadata (key, value) VALUES (?, ?)")
+      hash.each do |key, value|
+        stmt.execute([key.to_s, value.to_s])
+      end
+      stmt.close
+    end
+
+    def load_metadata
+      result = {}
+      @db.execute("SELECT key, value FROM metadata") do |row|
+        key, value = row
+        result[key.to_sym] = value
+      end
+      result
+    rescue SQLite3::Exception
+      nil
+    end
+  end
+end

data/lib/wp2txt/magic_words.rb

@@ -0,0 +1,353 @@
+# frozen_string_literal: true
+
+require_relative "regex"
+
+module Wp2txt
+  # Expands MediaWiki magic words to their actual values
+  # Supports: page context variables, date/time variables, string functions
+  class MagicWordExpander
+    # Page context magic words (case-insensitive)
+    PAGE_CONTEXT_WORDS = {
+      "PAGENAME" => :pagename,
+      "PAGENAMEE" => :pagename_encoded,
+      "FULLPAGENAME" => :fullpagename,
+      "FULLPAGENAMEE" => :fullpagename_encoded,
+      "BASEPAGENAME" => :basepagename,
+      "BASEPAGENAMEE" => :basepagename_encoded,
+      "ROOTPAGENAME" => :rootpagename,
+      "ROOTPAGENAMEE" => :rootpagename_encoded,
+      "SUBPAGENAME" => :subpagename,
+      "SUBPAGENAMEE" => :subpagename_encoded,
+      "TALKPAGENAME" => :talkpagename,
+      "TALKPAGENAMEE" => :talkpagename_encoded,
+      "SUBJECTPAGENAME" => :subjectpagename,
+      "SUBJECTPAGENAMEE" => :subjectpagename_encoded,
+      "ARTICLEPAGENAME" => :subjectpagename,
+      "ARTICLEPAGENAMEE" => :subjectpagename_encoded,
+      "NAMESPACE" => :namespace,
+      "NAMESPACEE" => :namespace_encoded,
+      "NAMESPACENUMBER" => :namespace_number,
+      "TALKSPACE" => :talkspace,
+      "TALKSPACEE" => :talkspace_encoded,
+      "SUBJECTSPACE" => :subjectspace,
+      "SUBJECTSPACEE" => :subjectspace_encoded,
+      "ARTICLESPACE" => :subjectspace,
+      "ARTICLESPACEE" => :subjectspace_encoded
+    }.freeze
+
+    # Date/time magic words
+    DATETIME_WORDS = {
+      "CURRENTYEAR" => :current_year,
+      "CURRENTMONTH" => :current_month,
+      "CURRENTMONTH1" => :current_month1,
+      "CURRENTMONTHNAME" => :current_month_name,
+      "CURRENTMONTHNAMEGEN" => :current_month_name,
+      "CURRENTMONTHABBREV" => :current_month_abbrev,
+      "CURRENTDAY" => :current_day,
+      "CURRENTDAY2" => :current_day2,
+      "CURRENTDOW" => :current_dow,
+      "CURRENTDAYNAME" => :current_day_name,
+      "CURRENTTIME" => :current_time,
+      "CURRENTHOUR" => :current_hour,
+      "CURRENTWEEK" => :current_week,
+      "CURRENTTIMESTAMP" => :current_timestamp,
+      # Local variants (same as current for our purposes)
+      "LOCALYEAR" => :current_year,
+      "LOCALMONTH" => :current_month,
+      "LOCALMONTH1" => :current_month1,
+      "LOCALMONTHNAME" => :current_month_name,
+      "LOCALMONTHNAMEGEN" => :current_month_name,
+      "LOCALMONTHABBREV" => :current_month_abbrev,
+      "LOCALDAY" => :current_day,
+      "LOCALDAY2" => :current_day2,
+      "LOCALDOW" => :current_dow,
+      "LOCALDAYNAME" => :current_day_name,
+      "LOCALTIME" => :current_time,
+      "LOCALHOUR" => :current_hour,
+      "LOCALWEEK" => :current_week,
+      "LOCALTIMESTAMP" => :current_timestamp
+    }.freeze
+
+    # String function magic words (with arguments)
+    STRING_FUNCTIONS = %w[
+      lc uc lcfirst ucfirst
+      padleft padright
+      anchorencode urlencode
+      plural grammar gender
+      int formatnum
+    ].freeze
+
+    # Month names for expansion
+    MONTH_NAMES = %w[
+      January February March April May June
+      July August September October November December
+    ].freeze
+
+    MONTH_ABBREVS = %w[
+      Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec
+    ].freeze
+
+    DAY_NAMES = %w[
+      Sunday Monday Tuesday Wednesday Thursday Friday Saturday
+    ].freeze
+
+    def initialize(title, namespace: "", dump_date: nil)
+      @title = title || ""
+      @namespace = namespace || ""
+      @dump_date = dump_date || Time.now
+    end
+
+    # Main expansion method - expands all supported magic words in text
+    def expand(text)
+      return text if text.nil? || text.empty?
+
+      # Early exit: no templates to process
+      return text unless text.include?("{{")
+
+      result = text.dup
+
+      # Expand simple magic words: {{PAGENAME}}, {{CURRENTYEAR}}, etc.
+      result = expand_simple_magic_words(result)
+
+      # Expand string functions: {{lc:Text}}, {{uc:Text}}, etc.
+      result = expand_string_functions(result)
+
+      # Expand #titleparts parser function
+      result = expand_titleparts(result)
+
+      result
+    end
+
+    private
+
+    # Expand simple magic words without arguments
+    def expand_simple_magic_words(text)
+      # Match {{WORD}} pattern (case-insensitive for the word)
+      text.gsub(/\{\{\s*([A-Z][A-Z0-9]*)\s*\}\}/i) do |match|
+        word = $1.upcase
+        if PAGE_CONTEXT_WORDS.key?(word)
+          expand_page_context(PAGE_CONTEXT_WORDS[word])
+        elsif DATETIME_WORDS.key?(word)
+          expand_datetime(DATETIME_WORDS[word])
+        else
+          match # Return unchanged if not recognized
+        end
+      end
+    end
+
+    # Expand page context magic words
+    def expand_page_context(type)
+      case type
+      when :pagename
+        @title
+      when :pagename_encoded
+        url_encode(@title)
+      when :fullpagename
+        @namespace.empty? ? @title : "#{@namespace}:#{@title}"
+      when :fullpagename_encoded
+        url_encode(@namespace.empty? ? @title : "#{@namespace}:#{@title}")
+      when :basepagename
+        # Remove subpage part (after last /)
+        @title.sub(%r{/[^/]*$}, "")
+      when :basepagename_encoded
+        url_encode(@title.sub(%r{/[^/]*$}, ""))
+      when :rootpagename
+        # Get root page (before first /)
+        @title.split("/").first || @title
+      when :rootpagename_encoded
+        url_encode(@title.split("/").first || @title)
+      when :subpagename
+        # Get subpage part (after last /)
+        @title.include?("/") ? @title.split("/").last : @title
+      when :subpagename_encoded
+        part = @title.include?("/") ? @title.split("/").last : @title
+        url_encode(part)
+      when :talkpagename
+        ns = @namespace.empty? ? "Talk" : "#{@namespace} talk"
+        "#{ns}:#{@title}"
+      when :talkpagename_encoded
+        ns = @namespace.empty? ? "Talk" : "#{@namespace}_talk"
+        url_encode("#{ns}:#{@title}")
+      when :subjectpagename
+        @namespace.empty? ? @title : "#{@namespace}:#{@title}"
+      when :subjectpagename_encoded
+        url_encode(@namespace.empty? ? @title : "#{@namespace}:#{@title}")
+      when :namespace
+        @namespace
+      when :namespace_encoded
+        url_encode(@namespace)
+      when :namespace_number
+        # Main namespace = 0, others would need a lookup table
+        @namespace.empty? ? "0" : ""
+      when :talkspace
+        @namespace.empty? ? "Talk" : "#{@namespace} talk"
+      when :talkspace_encoded
+        url_encode(@namespace.empty? ? "Talk" : "#{@namespace} talk")
+      when :subjectspace
+        @namespace
+      when :subjectspace_encoded
+        url_encode(@namespace)
+      else
+        ""
+      end
+    end
+
+    # Expand date/time magic words
+    def expand_datetime(type)
+      case type
+      when :current_year
+        @dump_date.year.to_s
+      when :current_month
+        @dump_date.month.to_s.rjust(2, "0")
+      when :current_month1
+        @dump_date.month.to_s
+      when :current_month_name
+        MONTH_NAMES[@dump_date.month - 1]
+      when :current_month_abbrev
+        MONTH_ABBREVS[@dump_date.month - 1]
+      when :current_day
+        @dump_date.day.to_s
+      when :current_day2
+        @dump_date.day.to_s.rjust(2, "0")
+      when :current_dow
+        @dump_date.wday.to_s
+      when :current_day_name
+        DAY_NAMES[@dump_date.wday]
+      when :current_time
+        @dump_date.strftime("%H:%M")
+      when :current_hour
+        @dump_date.hour.to_s.rjust(2, "0")
+      when :current_week
+        @dump_date.strftime("%V")
+      when :current_timestamp
+        @dump_date.strftime("%Y%m%d%H%M%S")
+      else
+        ""
+      end
+    end
+
+    # Expand string functions: {{lc:Text}}, {{uc:Text}}, etc.
+    def expand_string_functions(text)
+      # Match {{function:argument}} pattern
+      # Need to handle nested braces carefully
+      result = text.dup
+
+      # Simple string case functions (no nesting issues)
+      result.gsub!(/\{\{\s*lc\s*:\s*([^}]*)\}\}/i) { $1.downcase }
+      result.gsub!(/\{\{\s*uc\s*:\s*([^}]*)\}\}/i) { $1.upcase }
+      result.gsub!(/\{\{\s*lcfirst\s*:\s*([^}]*)\}\}/i) do
+        s = $1
+        s.empty? ? s : s[0].downcase + s[1..]
+      end
+      result.gsub!(/\{\{\s*ucfirst\s*:\s*([^}]*)\}\}/i) do
+        s = $1
+        s.empty? ? s : s[0].upcase + s[1..]
+      end
+
+      # URL encoding
+      result.gsub!(/\{\{\s*urlencode\s*:\s*([^}|]*?)(?:\s*\|\s*[^}]*)?\}\}/i) do
+        url_encode($1.strip)
+      end
+      result.gsub!(/\{\{\s*anchorencode\s*:\s*([^}]*)\}\}/i) do
+        anchor_encode($1.strip)
+      end
+
+      # Padding functions: {{padleft:string|length|pad}}
+      result.gsub!(/\{\{\s*padleft\s*:\s*([^}|]*)\s*\|\s*(\d+)(?:\s*\|\s*([^}]*))?\}\}/i) do
+        str, len, pad = $1, $2.to_i, ($3 || "0")
+        pad = "0" if pad.empty?
+        str.rjust(len, pad)
+      end
+      result.gsub!(/\{\{\s*padright\s*:\s*([^}|]*)\s*\|\s*(\d+)(?:\s*\|\s*([^}]*))?\}\}/i) do
+        str, len, pad = $1, $2.to_i, ($3 || "0")
+        pad = "0" if pad.empty?
+        str.ljust(len, pad)
+      end
+
+      # formatnum - format number with thousand separators
+      result.gsub!(/\{\{\s*formatnum\s*:\s*([^}|]*?)(?:\s*\|([^}]*))?\}\}/i) do
+        num_str = $1.strip
+        option = $2&.strip&.downcase
+
+        if option == "r"
+          # R option: remove formatting (return raw number)
+          num_str.gsub(/[,\s]/, "")
+        else
+          # Add thousand separators
+          format_number_with_commas(num_str)
+        end
+      end
+
+      # plural, grammar, gender - just return first argument (proper handling would need language rules)
+      result.gsub!(/\{\{\s*plural\s*:\s*[^}|]*\s*\|\s*([^}|]*)[^}]*\}\}/i) { $1 }
+      result.gsub!(/\{\{\s*grammar\s*:\s*[^}|]*\s*\|\s*([^}|]*)[^}]*\}\}/i) { $1 }
+      result.gsub!(/\{\{\s*gender\s*:\s*[^}|]*\s*\|\s*([^}|]*)[^}]*\}\}/i) { $1 }
+
+      # int - internationalization, just return the message name
+      result.gsub!(/\{\{\s*int\s*:\s*([^}|]*)[^}]*\}\}/i) { $1.strip }
+
+      result
+    end
+
+    # Expand #titleparts parser function
+    # {{#titleparts:pagename|number of segments|first segment}}
+    def expand_titleparts(text)
+      text.gsub(/\{\{\s*#titleparts\s*:\s*([^}|]+)(?:\s*\|\s*([^}|]*))?(?:\s*\|\s*([^}]*))?\}\}/i) do
+        pagename = $1.strip
+        num_segments = $2&.strip&.to_i
+        first_segment = ($3&.strip&.to_i || 1)
+
+        parts = pagename.split("/")
+        first_segment = 1 if first_segment < 1
+        first_idx = first_segment - 1
+
+        if num_segments && num_segments > 0
+          parts[first_idx, num_segments]&.join("/") || ""
+        elsif num_segments && num_segments < 0
+          # Negative means "all but last N"
+          end_idx = parts.length + num_segments
+          end_idx > first_idx ? parts[first_idx...end_idx].join("/") : ""
+        else
+          parts[first_idx..]&.join("/") || ""
+        end
+      end
+    end
+
+    # URL encode a string (for PAGENAMEE variants)
+    def url_encode(str)
+      return "" if str.nil?
+
+      str.gsub(/[^a-zA-Z0-9\-._~]/) do |c|
+        c == " " ? "_" : "%" + c.unpack1("H*").upcase
+      end
+    end
+
+    # Anchor encode (for fragment identifiers)
+    def anchor_encode(str)
+      return "" if str.nil?
+
+      str.gsub(" ", "_").gsub(/[^\w\-.]/) do |c|
+        ".#{c.unpack1('H*').upcase}"
+      end
+    end
+
+    # Format number with thousand separators
+    def format_number_with_commas(num_str)
+      return num_str if num_str.nil? || num_str.empty?
+
+      # Split integer and decimal parts
+      parts = num_str.split(".")
+      integer_part = parts[0] || ""
+      decimal_part = parts[1]
+
+      # Add thousand separators to integer part
+      integer_part = integer_part.gsub(/(\d)(?=(\d{3})+(?!\d))/, '\\1,')
+
+      if decimal_part
+        "#{integer_part}.#{decimal_part}"
+      else
+        integer_part
+      end
+    end
+  end
+end