wp2txt 1.1.3 → 2.1.0

data/lib/wp2txt/formatter.rb

@@ -0,0 +1,352 @@
+# frozen_string_literal: true
+
+require_relative "utils"
+require_relative "regex"
+require_relative "section_extractor"
+
+module Wp2txt
+  # Article formatting utilities for WpApp
+  module Formatter
+    # Debug mode flag (inherited from including class if defined)
+    def formatter_debug_mode
+      defined?(DEBUG_MODE) ? DEBUG_MODE : false
+    end
+
+    # Format article based on configuration and output format
+    def format_article(article, config)
+      # Store original title for magic word expansion in content
+      original_title = article.title.dup
+      article.title = format_wiki(article.title, config)
+
+      # Add title to config for magic word expansion in content processing
+      config_with_title = config.merge(title: original_title)
+
+      # Handle metadata_only mode (title + sections + categories)
+      if config[:metadata_only]
+        return format_metadata_only(article, config_with_title)
+      end
+
+      # Handle summary_only as section extraction (for consistency)
+      if config[:summary_only]
+        summary_config = config_with_title.merge(
+          sections: [SectionExtractor::SUMMARY_KEY],
+          section_output: "combined"
+        )
+        return format_with_sections(article, summary_config)
+      end
+
+      # Handle section extraction mode (--sections option)
+      if config[:sections] && !config[:sections].empty?
+        return format_with_sections(article, config_with_title)
+      end
+
+      if config[:format] == :json
+        format_article_json(article, config_with_title)
+      else
+        format_article_text(article, config_with_title)
+      end
+    end
+
+    # Format article with specific section extraction
+    def format_with_sections(article, config)
+      extractor = SectionExtractor.new(
+        config[:sections],
+        min_length: config[:min_section_length] || 0,
+        skip_empty: config[:skip_empty] || false,
+        use_aliases: !config[:no_section_aliases],
+        alias_file: config[:alias_file],
+        track_matches: config[:show_matched_sections] || false
+      )
+
+      # Skip article if no matching sections and skip_empty is true
+      return nil if extractor.should_skip?(article)
+
+      sections = extractor.extract_sections(article, config)
+      matched_sections = extractor.matched_sections
+
+      # Apply format_wiki to section content
+      sections.transform_values! do |content|
+        next nil if content.nil?
+        cleanup(format_wiki(content, config))
+      end
+
+      output_mode = config[:section_output] || "structured"
+
+      if config[:format] == :json
+        if output_mode == "combined"
+          format_sections_combined_json(article, sections, config, matched_sections)
+        else
+          format_sections_structured_json(article, sections, config, matched_sections)
+        end
+      else
+        if output_mode == "combined"
+          format_sections_combined_text(article, sections, config)
+        else
+          format_sections_structured_text(article, sections, config)
+        end
+      end
+    end
+
+    # Format sections as structured JSON (each section as separate field)
+    def format_sections_structured_json(article, sections, config, matched_sections = {})
+      result = {
+        "title" => article.title,
+        "sections" => sections
+      }
+      result["categories"] = article.categories.flatten if config[:category]
+      # Include matched_sections if tracking is enabled and there are matches
+      if config[:show_matched_sections] && matched_sections && !matched_sections.empty?
+        result["matched_sections"] = matched_sections
+      end
+      result
+    end
+
+    # Format sections as combined JSON (all sections concatenated)
+    def format_sections_combined_json(article, sections, config, matched_sections = {})
+      included = sections.keys.select { |k| sections[k] && !sections[k].empty? }
+      text = included.map { |k| sections[k] }.join("\n\n")
+
+      result = {
+        "title" => article.title,
+        "text" => text,
+        "sections_included" => included
+      }
+      result["categories"] = article.categories.flatten if config[:category]
+      # Include matched_sections if tracking is enabled and there are matches
+      if config[:show_matched_sections] && matched_sections && !matched_sections.empty?
+        result["matched_sections"] = matched_sections
+      end
+      result
+    end
+
+    # Format sections as structured text
+    def format_sections_structured_text(article, sections, config)
+      output = +"TITLE: #{article.title}\n\n"
+
+      sections.each do |name, content|
+        if content.nil?
+          output << "SECTION [#{name}]: (not found)\n\n"
+        else
+          output << "SECTION [#{name}]:\n#{content}\n\n"
+        end
+      end
+
+      if config[:category] && !article.categories.empty?
+        output << "CATEGORIES: #{article.categories.flatten.join(', ')}\n"
+      end
+
+      output << "\n"
+      output
+    end
+
+    # Format sections as combined text
+    def format_sections_combined_text(article, sections, config)
+      included = sections.keys.select { |k| sections[k] && !sections[k].empty? }
+      text = included.map { |k| sections[k] }.join("\n\n")
+
+      output = +"TITLE: #{article.title}\n"
+      output << "SECTIONS: #{included.join(', ')}\n\n"
+      output << text
+      output << "\n\n"
+
+      if config[:category] && !article.categories.empty?
+        output << "CATEGORIES: #{article.categories.flatten.join(', ')}\n"
+      end
+
+      output << "\n"
+      output
+    end
+
+    # Format article with metadata only (title, section headings, categories)
+    # Used for analyzing section distribution across Wikipedia dumps
+    def format_metadata_only(article, config)
+      extractor = SectionExtractor.new
+      sections = extractor.extract_headings(article)
+
+      if config[:format] == :json
+        format_metadata_only_json(article, sections)
+      else
+        format_metadata_only_text(article, sections)
+      end
+    end
+
+    # Format metadata as JSON
+    def format_metadata_only_json(article, sections)
+      {
+        "title" => article.title,
+        "sections" => sections,
+        "categories" => article.categories.flatten
+      }
+    end
+
+    # Format metadata as TSV text
+    # Format: Title<TAB>Section1|Section2|...<TAB>Category1,Category2,...
+    def format_metadata_only_text(article, sections)
+      title = article.title
+      sections_str = sections.join("|")
+      categories_str = article.categories.flatten.join(",")
+
+      "#{title}\t#{sections_str}\t#{categories_str}\n"
+    end
+
+    # Format article as JSON hash
+    def format_article_json(article, config)
+      result = { "title" => article.title }
+
+      # Categories
+      if config[:category]
+        result["categories"] = article.categories.flatten
+      else
+        result["categories"] = nil
+      end
+
+      # Text content
+      if config[:category_only]
+        result["text"] = nil
+      else
+        text = build_text_content(article, config)
+        result["text"] = text.strip
+      end
+
+      # Redirect
+      redirect_target = extract_redirect(article)
+      result["redirect"] = redirect_target
+
+      result
+    end
+
+    # Extract redirect target from article if it's a redirect
+    def extract_redirect(article)
+      article.elements.each do |type, content|
+        if type == :mw_redirect
+          match = content.match(REDIRECT_REGEX)
+          return match[1] if match
+        end
+      end
+      nil
+    end
+
+    # Format article as text string
+    def format_article_text(article, config)
+      if config[:category_only]
+        format_category_only(article)
+      elsif config[:category] && !article.categories.empty?
+        format_with_categories(article, config)
+      else
+        format_full_article(article, config)
+      end
+    end
+
+    # Build text content from article elements
+    def build_text_content(article, config)
+      contents = +""
+      article.elements.each do |e|
+        line = process_element(e, config)
+        contents << line if line
+      end
+      # Apply cleanup to remove leftover markup, normalize whitespace, etc.
+      cleanup(contents)
+    end
+
+    # Format article with only category information (text format)
+    def format_category_only(article)
+      title = "#{article.title}\t"
+      contents = article.categories.join(", ")
+      contents << "\n"
+      title + contents
+    end
+
+    # Format article with categories (includes body text)
+    def format_with_categories(article, config)
+      title = "\n[[#{article.title}]]\n\n"
+      contents = build_text_content(article, config)
+
+      # Add categories at the end
+      contents << "\nCATEGORIES: "
+      contents << article.categories.join(", ")
+      contents << "\n\n"
+
+      config[:title] ? title + contents : contents
+    end
+
+    # Format full article content
+    def format_full_article(article, config)
+      title = "\n[[#{article.title}]]\n\n"
+      contents = build_text_content(article, config)
+
+      config[:title] ? title + contents : contents
+    end
+
+    # Process individual element of the article
+    def process_element(element, config)
+      type, content = element
+      debug_mode = formatter_debug_mode
+
+      case type
+      when :mw_heading
+        return nil if config[:summary_only]
+        return nil unless config[:heading]
+
+        content = format_wiki(content, config)
+        content += "+HEADING+" if debug_mode
+        content + "\n"
+      when :mw_paragraph
+        content = format_wiki(content, config)
+        content += "+PARAGRAPH+" if debug_mode
+        content + "\n"
+      when :mw_table, :mw_htable
+        return nil unless config[:table]
+
+        content += "+TABLE+" if debug_mode
+        content + "\n"
+      when :mw_pre
+        return nil unless config[:pre]
+
+        content += "+PRE+" if debug_mode
+        content + "\n"
+      when :mw_quote
+        content += "+QUOTE+" if debug_mode
+        content + "\n"
+      when :mw_unordered, :mw_ordered, :mw_definition
+        return nil unless config[:list]
+
+        content += "+LIST+" if debug_mode
+        content + "\n"
+      when :mw_ml_template
+        return nil unless config[:multiline]
+
+        content += "+MLTEMPLATE+" if debug_mode
+        content + "\n"
+      when :mw_link
+        content = format_wiki(content, config)
+        return nil if content.strip.empty?
+
+        content += "+LINK+" if debug_mode
+        content + "\n"
+      when :mw_ml_link
+        content = format_wiki(content, config)
+        return nil if content.strip.empty?
+
+        content += "+MLLINK+" if debug_mode
+        content + "\n"
+      when :mw_redirect
+        return nil unless config[:redirect]
+
+        content += "+REDIRECT+" if debug_mode
+        content + "\n\n"
+      when :mw_isolated_template
+        return nil unless config[:multiline]
+
+        content += "+ISOLATED_TEMPLATE+" if debug_mode
+        content + "\n"
+      when :mw_isolated_tag
+        nil
+      else
+        return nil unless debug_mode
+
+        content += "+OTHER+"
+        content + "\n"
+      end
+    end
+  end
+end

data/lib/wp2txt/global_data_cache.rb

@@ -0,0 +1,353 @@
+# frozen_string_literal: true
+
+require "sqlite3"
+require "json"
+require "fileutils"
+require "digest"
+
+module Wp2txt
+  # SQLite-based cache for global data files (templates, mediawiki aliases, entities)
+  # Dramatically speeds up startup by avoiding JSON parsing overhead
+  class GlobalDataCache
+    CACHE_VERSION = 1
+    DEFAULT_CACHE_DIR = File.expand_path("~/.wp2txt/cache")
+
+    # Data categories and their source paths
+    # Note: html_entities_combined has no direct source (derived from html_entities + wikipedia_entities)
+    DATA_SOURCES = {
+      mediawiki: "mediawiki_aliases.json",
+      template: "template_aliases.json",
+      html_entities: "html_entities.json",
+      wikipedia_entities: "wikipedia_entities.json",
+      language_metadata: "language_metadata.json",
+      language_tiers: "language_tiers.json"
+    }.freeze
+
+    # Categories that are derived (combined from multiple sources)
+    # These are validated by checking their source files
+    DERIVED_SOURCES = {
+      html_entities_combined: [:html_entities, :wikipedia_entities]
+    }.freeze
+
+    class << self
+      attr_accessor :cache_dir, :enabled
+
+      def configure(cache_dir: nil, enabled: true)
+        @cache_dir = cache_dir || DEFAULT_CACHE_DIR
+        @enabled = enabled
+      end
+
+      def cache_path
+        @cache_dir ||= DEFAULT_CACHE_DIR
+        File.join(@cache_dir, "global_data.sqlite3")
+      end
+
+      def data_dir
+        File.join(__dir__, "data")
+      end
+
+      # Check if cache is valid for all source files
+      def cache_valid?
+        return false unless @enabled
+        return false unless File.exist?(cache_path)
+
+        begin
+          db = open_db
+          DATA_SOURCES.each do |category, filename|
+            source_path = File.join(data_dir, filename)
+            next unless File.exist?(source_path)
+
+            meta = load_metadata(db, category)
+            return false unless meta
+
+            # Check version
+            return false if meta[:cache_version].to_i != CACHE_VERSION
+
+            # Check source file hasn't changed
+            source_stat = File.stat(source_path)
+            return false if meta[:source_mtime].to_i != source_stat.mtime.to_i
+            return false if meta[:source_size].to_i != source_stat.size
+          end
+          true
+        rescue SQLite3::Exception
+          false
+        ensure
+          db&.close
+        end
+      end
+
+      # Check if a specific category's cache is valid
+      def category_valid?(category)
+        return false unless @enabled
+        return false unless File.exist?(cache_path)
+
+        # For derived categories, check source categories
+        if DERIVED_SOURCES.key?(category)
+          return DERIVED_SOURCES[category].all? { |src| category_valid?(src) }
+        end
+
+        # For unknown categories (not in DATA_SOURCES), just check if it exists in cache
+        filename = DATA_SOURCES[category]
+        unless filename
+          begin
+            db = open_db
+            row = db.get_first_row("SELECT 1 FROM global_data WHERE category = ?", [category.to_s])
+            return !row.nil?
+          rescue SQLite3::Exception
+            return false
+          ensure
+            db&.close
+          end
+        end
+
+        # For known data sources, validate against source file
+        begin
+          db = open_db
+          source_path = File.join(data_dir, filename)
+          return true unless File.exist?(source_path)
+
+          meta = load_metadata(db, category)
+          return false unless meta
+          return false if meta[:cache_version].to_i != CACHE_VERSION
+
+          source_stat = File.stat(source_path)
+          return false if meta[:source_mtime].to_i != source_stat.mtime.to_i
+          return false if meta[:source_size].to_i != source_stat.size
+
+          true
+        rescue SQLite3::Exception
+          false
+        ensure
+          db&.close
+        end
+      end
+
+      # Load data from cache
+      # @param category [Symbol] Data category (:mediawiki, :template, etc.)
+      # @return [Hash, nil] Parsed data or nil if not cached or invalid
+      def load(category)
+        return nil unless @enabled
+        return nil unless File.exist?(cache_path)
+        return nil unless category_valid?(category)
+
+        begin
+          db = open_db
+          row = db.get_first_row(
+            "SELECT data FROM global_data WHERE category = ?",
+            [category.to_s]
+          )
+          return nil unless row
+
+          JSON.parse(row[0])
+        rescue SQLite3::Exception, JSON::ParserError
+          nil
+        ensure
+          db&.close
+        end
+      end
+
+      # Save data to cache
+      # @param category [Symbol] Data category
+      # @param data [Hash] Data to cache
+      def save(category, data)
+        return unless @enabled
+
+        FileUtils.mkdir_p(File.dirname(cache_path))
+
+        begin
+          db = open_db
+          create_schema(db)
+
+          db.execute(
+            "INSERT OR REPLACE INTO global_data (category, data, updated_at) VALUES (?, ?, ?)",
+            [category.to_s, JSON.generate(data), Time.now.to_i]
+          )
+
+          # For derived categories, save metadata from source files
+          if DERIVED_SOURCES.key?(category)
+            DERIVED_SOURCES[category].each do |src_category|
+              filename = DATA_SOURCES[src_category]
+              next unless filename
+
+              source_path = File.join(data_dir, filename)
+              next unless File.exist?(source_path)
+
+              source_stat = File.stat(source_path)
+              save_metadata(db, src_category,
+                source_path: source_path,
+                source_mtime: source_stat.mtime.to_i,
+                source_size: source_stat.size,
+                cache_version: CACHE_VERSION
+              )
+            end
+          else
+            # For regular categories, save metadata from the source file
+            filename = DATA_SOURCES[category]
+            if filename
+              source_path = File.join(data_dir, filename)
+              if File.exist?(source_path)
+                source_stat = File.stat(source_path)
+                save_metadata(db, category,
+                  source_path: source_path,
+                  source_mtime: source_stat.mtime.to_i,
+                  source_size: source_stat.size,
+                  cache_version: CACHE_VERSION
+                )
+              end
+            end
+          end
+        rescue SQLite3::Exception => e
+          warn "GlobalDataCache: Failed to save #{category}: #{e.message}"
+        ensure
+          db&.close
+        end
+      end
+
+      # Load all data categories at once (more efficient)
+      # @return [Hash] { category => data }
+      def load_all
+        return {} unless @enabled
+        return {} unless File.exist?(cache_path)
+
+        result = {}
+        begin
+          db = open_db
+          db.execute("SELECT category, data FROM global_data") do |row|
+            category = row[0].to_sym
+            result[category] = JSON.parse(row[1])
+          end
+          result
+        rescue SQLite3::Exception, JSON::ParserError
+          {}
+        ensure
+          db&.close
+        end
+      end
+
+      # Save all data categories at once
+      # @param data_hash [Hash] { category => data }
+      def save_all(data_hash)
+        return unless @enabled
+
+        FileUtils.mkdir_p(File.dirname(cache_path))
+
+        begin
+          db = open_db
+          create_schema(db)
+
+          db.execute("BEGIN TRANSACTION")
+
+          data_hash.each do |category, data|
+            db.execute(
+              "INSERT OR REPLACE INTO global_data (category, data, updated_at) VALUES (?, ?, ?)",
+              [category.to_s, JSON.generate(data), Time.now.to_i]
+            )
+
+            # Only save metadata if this is a known data source
+            filename = DATA_SOURCES[category]
+            if filename
+              source_path = File.join(data_dir, filename)
+              if File.exist?(source_path)
+                source_stat = File.stat(source_path)
+                save_metadata(db, category,
+                  source_path: source_path,
+                  source_mtime: source_stat.mtime.to_i,
+                  source_size: source_stat.size,
+                  cache_version: CACHE_VERSION
+                )
+              end
+            end
+          end
+
+          db.execute("COMMIT")
+        rescue SQLite3::Exception => e
+          db&.execute("ROLLBACK") rescue nil
+          warn "GlobalDataCache: Failed to save all: #{e.message}"
+        ensure
+          db&.close
+        end
+      end
+
+      # Clear cache
+      def clear!
+        FileUtils.rm_f(cache_path)
+      end
+
+      # Get cache statistics
+      def stats
+        return nil unless File.exist?(cache_path)
+
+        begin
+          db = open_db
+          categories = db.execute("SELECT category, LENGTH(data), updated_at FROM global_data")
+
+          {
+            cache_path: cache_path,
+            cache_size: File.size(cache_path),
+            categories: categories.map do |row|
+              {
+                category: row[0],
+                data_size: row[1],
+                updated_at: row[2] ? Time.at(row[2]) : nil
+              }
+            end
+          }
+        rescue SQLite3::Exception
+          nil
+        ensure
+          db&.close
+        end
+      end
+
+      private
+
+      def open_db
+        db = SQLite3::Database.new(cache_path)
+        db.execute("PRAGMA journal_mode = WAL")
+        db.execute("PRAGMA synchronous = NORMAL")
+        db
+      end
+
+      def create_schema(db)
+        db.execute(<<~SQL)
+          CREATE TABLE IF NOT EXISTS global_data (
+            category TEXT PRIMARY KEY,
+            data TEXT NOT NULL,
+            updated_at INTEGER
+          )
+        SQL
+
+        db.execute(<<~SQL)
+          CREATE TABLE IF NOT EXISTS metadata (
+            category TEXT,
+            key TEXT,
+            value TEXT,
+            PRIMARY KEY (category, key)
+          )
+        SQL
+      end
+
+      def save_metadata(db, category, hash)
+        hash.each do |key, value|
+          db.execute(
+            "INSERT OR REPLACE INTO metadata (category, key, value) VALUES (?, ?, ?)",
+            [category.to_s, key.to_s, value.to_s]
+          )
+        end
+      end
+
+      def load_metadata(db, category)
+        result = {}
+        db.execute("SELECT key, value FROM metadata WHERE category = ?", [category.to_s]) do |row|
+          result[row[0].to_sym] = row[1]
+        end
+        result.empty? ? nil : result
+      rescue SQLite3::Exception
+        nil
+      end
+    end
+
+    # Initialize with default settings
+    configure
+  end
+end