wp2txt 2.3.3 → 2.4.0

data/lib/wp2txt/corpus.rb

@@ -115,7 +115,9 @@ module Wp2txt
         fulltext_current: fts.built? && fts.valid_for?(@multistream_path),
         stats: stats,
         fulltext: fts.built? ? fts.stats : nil,
-        langlinks: @metadata.built? ? @metadata.langlinks_provenance : nil
+        langlinks: @metadata.built? ? @metadata.langlinks_provenance : nil,
+        page_properties: @metadata.built? ? @metadata.page_props_provenance : nil,
+        inlinks: @metadata.built? ? @metadata.links_provenance : nil
       }
     end
 
@@ -147,7 +149,10 @@ module Wp2txt
              else
                render_text(page)
              end
-      result = { id: page[:id], title: page[:title], format: format.to_s }
+      result = { id: page[:id], title: page[:title] }
+      properties = @metadata.built? ? @metadata.properties_for(page[:id]) : nil
+      result.merge!(properties) if properties
+      result[:format] = format.to_s
       if max_chars && body.length > max_chars
         result.merge(text: body[0, max_chars], truncated: true, total_chars: body.length)
       else
@@ -401,12 +406,16 @@ module Wp2txt
           titles.each_slice(EXTRACT_BATCH_SIZE) do |batch|
             raise Cancelled if cancel_check&.call
 
+            # Forked workers inherit this file's buffer and would write it again on exit
+            f.flush
             pages = reader.extract_articles_parallel(batch, num_processes: num_processes)
             batch.each do |t|
               page = pages[t]
               next unless page
 
               records = build_records(page, content, resolved_sections, chunk_size, chunk_overlap)
+              properties = @metadata.built? ? @metadata.properties_for(page[:id]) : nil
+              records.each { |record| record.merge!(properties) } if properties
               next if records.empty?
 
               articles_extracted += 1

data/lib/wp2txt/data/template_aliases.json

@@ -82,7 +82,7 @@
     "Cite web", "Citeweb", "Cit"
   ],
   "ruby_text_templates": [
-    "読み仮名", "よみがな", "ふりがな",
+    "読み仮名", "読み仮名 ruby不使用", "よみがな", "ふりがな",
     "ruby", "Ruby", "ruby-ja", "Ruby-ja",
     "ruby text", "ruby annotation",
     "rubi", "rubytext",

data/lib/wp2txt/extractor.rb

@@ -111,6 +111,7 @@ module Wp2txt
 
         if page
           article = Article.new(page[:text], page[:title], !config[:marker])
+          set_source_fields(article, page[:id], page[:revision_id], page[:text], config)
           result = format_article(article, config)
           writer.write(result)
           extracted_count += 1
@@ -253,11 +254,17 @@ module Wp2txt
         bz2_gem: opts[:bz2_gem]
       }
 
-      %i[title list heading table pre ref redirect multiline category category_only
+      %i[lead_terms title list heading table pre ref redirect multiline category category_only
          summary_only metadata_only marker extract_citations expand_templates].each do |opt|
         config[opt] = opts[opt]
       end
 
+      # Page properties, when imported into the index of the full cached dump
+      if format == :json && opts[:lang]
+        full_dump = Wp2txt::DumpManager.new(opts[:lang], cache_dir: opts[:cache_dir]).cached_multistream_path
+        config[:page_properties] = load_page_properties(full_dump, opts[:cache_dir])
+      end
+
       # Section extraction options
       %i[sections section_output min_section_length skip_empty
          alias_file no_section_aliases show_matched_sections].each do |opt|
@@ -480,6 +487,7 @@ module Wp2txt
 
         pages.each do |page|
           article = Article.new(page[:text], page[:title], !config[:marker])
+          set_source_fields(article, page[:id], page[:revision_id], page[:text], config)
           result = format_article(article, config)
           writer.write(result)
           extracted_count += 1
@@ -493,6 +501,7 @@ module Wp2txt
 
           if page
             article = Article.new(page[:text], page[:title], !config[:marker])
+            set_source_fields(article, page[:id], page[:revision_id], page[:text], config)
             result = format_article(article, config)
             writer.write(result)
             extracted_count += 1

data/lib/wp2txt/formatter.rb

@@ -14,6 +14,26 @@ module Wp2txt
 
     # Format article based on configuration and output format
     def format_article(article, config)
+      with_source_fields(format_article_body(article, config), article)
+    end
+
+    # JSON records carry the dump's page and revision IDs (and page properties,
+    # when imported) right after the title, so a record can be traced
+    # back to its source; lead terms, when requested, come last.
+    def with_source_fields(result, article)
+      return result unless result.is_a?(Hash)
+
+      ids = { "page_id" => article.page_id, "revision_id" => article.revision_id }.compact
+      ids.merge!(article.page_properties.transform_keys(&:to_s)) if article.page_properties
+      out = result.each_with_object({}) do |(key, value), acc|
+        acc[key] = value
+        acc.merge!(ids) if key == "title"
+      end
+      out["lead_terms"] = article.lead_terms if article.lead_terms
+      out
+    end
+
+    def format_article_body(article, config)
       # Store original title for magic word expansion in content
       original_title = article.title.dup
       article.title = format_wiki(article.title, config)

data/lib/wp2txt/index_commands.rb

@@ -4,6 +4,8 @@ require "json"
 require_relative "metadata_index"
 require_relative "fts_index"
 require_relative "langlinks_importer"
+require_relative "page_props_importer"
+require_relative "link_counter"
 require_relative "corpus"
 require_relative "multistream"
 require_relative "memory_monitor"
@@ -322,11 +324,97 @@ module Wp2txt
         end
       end
       CliUI::EXIT_SUCCESS
-    rescue ArgumentError => e
+    rescue ArgumentError, Wp2txt::Error => e
       print_error(e.message)
       CliUI::EXIT_ERROR
     end
 
+    # Import each article's page properties from the page_props dump of the
+    # same date as the metadata index
+    def run_import_page_props(opts)
+      db_path, dump_date, manager = built_index_for(opts)
+      return CliUI::EXIT_ERROR unless db_path
+
+      source = opts[:page_props_file] || begin
+        print_header("Downloading page_props for '#{opts[:lang]}' (#{dump_date})")
+        manager.download_page_props(date: dump_date)
+      end
+      print_mode_banner("Import Page Properties", { "Source" => File.basename(source), "Metadata DB" => db_path })
+
+      time_start = Time.now
+      result = PagePropsImporter.new(db_path).import!(source, force: opts[:update_cache])
+      if result[:status] == :already_imported
+        print_success("Page properties already imported (at #{result[:imported_at]}, #{result[:row_count]} pages).")
+        print_info_message("Use -U/--update-cache to re-import.")
+      else
+        print_success("Page properties imported: #{result[:row_count]} pages in #{format_duration(Time.now - time_start)}")
+        provenance = result[:provenance]
+        print_info("Properties", "#{provenance[:qid_count]} QIDs, #{provenance[:disambiguation_count]} disambiguation flags, " \
+                                 "#{provenance[:sort_key_count]} sort keys; #{provenance[:skipped_invalid_sort_keys]} invalid sort keys skipped")
+        print_info("SHA-256", result[:provenance][:source_sha256].to_s)
+      end
+      CliUI::EXIT_SUCCESS
+    rescue ArgumentError, Wp2txt::Error => e
+      print_error(e.message)
+      CliUI::EXIT_ERROR
+    end
+
+    # Count incoming links to every article and store them in the metadata index
+    def run_count_links(opts)
+      db_path, _dump_date, manager = built_index_for(opts)
+      return CliUI::EXIT_ERROR unless db_path
+
+      multistream = manager.cached_multistream_path
+      stream_offsets, = load_stream_offsets(manager.cached_index_path, opts)
+      num_processes = opts[:num_procs] || MemoryMonitor.optimal_processes
+      print_mode_banner("Count Incoming Links", {
+        "Dump" => File.basename(multistream), "Streams" => stream_offsets.size.to_s,
+        "Processes" => num_processes.to_s
+      })
+
+      time_start = Time.now
+      last_report = Time.now
+      result = LinkCounter.new(multistream, stream_offsets, db_path: db_path,
+                                                           num_processes: num_processes).count! do |done, total|
+        now = Time.now
+        if !quiet? && (now - last_report >= DEFAULT_PROGRESS_INTERVAL || done == total)
+          last_report = now
+          puts pastel.dim(format("  [%s] %d/%d batches", now.strftime("%H:%M:%S"), done, total))
+        end
+      end
+      print_success("Incoming links counted for #{result[:articles]} articles " \
+                    "(#{result[:with_inlinks]} linked at least once) in #{format_duration(Time.now - time_start)}")
+      CliUI::EXIT_SUCCESS
+    rescue ArgumentError, Wp2txt::Error => e
+      print_error(e.message)
+      CliUI::EXIT_ERROR
+    end
+
+    # The metadata index of the cached dump for --lang, or nil with an error printed
+    # @return [Array(String, String, DumpManager)] [db_path, dump_date, manager]
+    def built_index_for(opts)
+      manager = DumpManager.new(opts[:lang], cache_dir: opts[:cache_dir],
+                                             dump_expiry_days: CLI.config.dump_expiry_days)
+      multistream = manager.cached_multistream_path
+      unless File.exist?(multistream)
+        print_error("No cached dump found for '#{opts[:lang]}'.")
+        print_info_message("Download and index it with: wp2txt --build-index -L #{opts[:lang]}")
+        return nil
+      end
+
+      db_path = MetadataIndex.path_for(multistream, cache_dir: opts[:cache_dir])
+      meta = MetadataIndex.new(db_path)
+      unless meta.built?
+        meta.close
+        print_error("Metadata index not found for this dump.")
+        print_info_message("Build it first with: wp2txt --build-index -L #{opts[:lang]}")
+        return nil
+      end
+      dump_date = meta.stats[:dump_name][/\d{8}\z/]
+      meta.close
+      [db_path, dump_date, manager]
+    end
+
     # Query the metadata index and print matching article titles
     def run_find_articles(opts)
       multistream_path, = resolve_dump_paths(opts, download: false)

data/lib/wp2txt/langlinks_importer.rb

@@ -4,6 +4,7 @@ require "sqlite3"
 require "set"
 require "time"
 require "zlib"
+require_relative "sql_dump_reader"
 require_relative "metadata_index"
 require_relative "version"
 
@@ -28,14 +29,6 @@ module Wp2txt
     SANITY_SAMPLE_SIZE = 1000
     SANITY_WARN_THRESHOLD = 0.9
 
-    INSERT_PREFIX = /\A\s*INSERT\s+INTO\s+`langlinks`\s+VALUES\s+/i
-
-    # MySQL backslash escapes inside mysqldump string literals
-    UNESCAPES = {
-      "0" => "\0", "'" => "'", '"' => '"', "b" => "\b", "n" => "\n",
-      "r" => "\r", "t" => "\t", "Z" => "\x1A", "\\" => "\\"
-    }.freeze
-
     def initialize(db_path, cache_dir: nil)
       @db_path = db_path
       @cache_dir = cache_dir
@@ -99,7 +92,9 @@ module Wp2txt
         batch.clear
       end
 
+      rows_seen = 0
       skipped_invalid = each_source_row(source_path) do |ll_from, ll_lang, ll_title|
+        rows_seen += 1
         next if lang_filter && !lang_filter.include?(ll_lang)
 
         batch << [ll_from, ll_lang, MetadataIndex.normalize_title(ll_title)]
@@ -107,6 +102,13 @@ module Wp2txt
       end
       flush.call unless batch.empty?
 
+      # Reporting success after reading nothing would leave every language
+      # query silently empty; an unreadable dump must fail loudly instead.
+      if rows_seen.zero? && skipped_invalid.zero?
+        raise Wp2txt::Error, "no langlinks rows found in #{File.basename(source_path)}; " \
+                             "the file may be empty or in an unrecognized format"
+      end
+
       # Indexes are created after the load, not before (insert speed)
       db.execute("CREATE INDEX idx_langlinks_from ON langlinks(ll_from, ll_lang)")
       db.execute("CREATE INDEX idx_langlinks_lang_title ON langlinks(ll_lang, ll_title)")
@@ -224,32 +226,18 @@ module Wp2txt
     # tagged UTF-8 and validated — a garbled title could never join
     # pages.title anyway, so such rows are skipped (and counted), not scrubbed.
     def each_source_row(source_path)
-      io = if source_path.end_with?(".gz")
-             # GzipReader ignores set_encoding; the encoding must be given
-             # at open time (lines must come out as BINARY — see below)
-             Zlib::GzipReader.open(source_path, encoding: Encoding::BINARY.to_s)
-           else
-             File.open(source_path, "rb")
-           end
-
       skipped = 0
-      begin
-        io.each_line do |line|
-          next unless INSERT_PREFIX.match?(line)
-
-          line.scan(TUPLE_REGEX) do |ll_from, ll_lang, ll_title|
-            lang = unescape_mysql(ll_lang).force_encoding(Encoding::UTF_8)
-            title = unescape_mysql(ll_title).force_encoding(Encoding::UTF_8)
-            unless lang.valid_encoding? && title.valid_encoding?
-              skipped += 1
-              next
-            end
-
-            yield ll_from.to_i, lang, title
+      SqlDumpReader.each_insert_line(source_path, "langlinks") do |line|
+        line.scan(TUPLE_REGEX) do |ll_from, ll_lang, ll_title|
+          lang = SqlDumpReader.unescape(ll_lang).force_encoding(Encoding::UTF_8)
+          title = SqlDumpReader.unescape(ll_title).force_encoding(Encoding::UTF_8)
+          unless lang.valid_encoding? && title.valid_encoding?
+            skipped += 1
+            next
           end
+
+          yield ll_from.to_i, lang, title
         end
-      ensure
-        io.close
       end
       skipped
     end
@@ -262,12 +250,8 @@ module Wp2txt
     # (equivalent to the old parser skipping malformed tuples)
     TUPLE_REGEX = /\((\d+),'((?:[^'\\]|\\.)*)','((?:[^'\\]|\\.)*)'\)/
 
-    UNESCAPE_REGEX = /\\(.)/m
 
     # Resolve MySQL backslash escapes in a captured string literal
     # (same mapping as the old hand-rolled parser)
-    def unescape_mysql(str)
-      str.gsub(UNESCAPE_REGEX) { UNESCAPES[::Regexp.last_match(1)] || ::Regexp.last_match(1) }
-    end
   end
 end

data/lib/wp2txt/lead_terms.rb

@@ -0,0 +1,228 @@
+# frozen_string_literal: true
+
+require_relative "wikitext_regions"
+
+module Wp2txt
+  # Finds the terms an article introduces in its lead: bold spans in the first
+  # paragraph that has one, the parenthesized notes written right after each,
+  # and reading (ruby) templates. Nothing is judged — which note is a reading,
+  # a native spelling, or a date is left to the caller.
+  #
+  # Positions are character offsets [start, end) into the article's wikitext
+  # as stored in the dump, XML entities decoded, before any other processing
+  # (comments included), so a caller holding the same text can cut out and
+  # hash exactly the span a term came from.
+  module LeadTerms
+    MAX_TERMS = 5
+    HEADING = /^={2,6}[^=\n].*?={2,6}[ \t]*$/
+    OPENERS = { "(" => ")", "（" => "）" }.freeze
+    SEPARATORS = ["、", ",", "，", ";", "；"].freeze
+    PARAGRAPH_BREAK = /\n[ \t\r]*\n/
+    # Regions whose bold text is not the article's own lead prose
+    SKIP_OPEN = { "{{" => "}}", "{|" => "|}", "<!--" => "-->" }.freeze
+    REF_OPEN = /\A<ref(?:\s[^>]*)?>/i
+    REF_SELF_CLOSING = /\A<ref(?:\s[^>]*)?\/>/i
+    FILE_LINK = /\A\[\[\s*(?:file|image|ファイル|画像|media)\s*:/i
+
+    module_function
+
+    # @param wikitext [String] decoded article wikitext
+    # @param render [#call] turns a wikitext fragment into clean text
+    # @return [Array<Hash>] terms in order of appearance, at most MAX_TERMS
+    def extract(wikitext, render:)
+      return [] if wikitext.nil? || wikitext.empty?
+
+      visible = WikitextRegions.mask_lead(wikitext)
+      bolds, rubies, lead_end = scan(visible, visible.length, source: wikitext)
+      original_render = render
+      render = ->(fragment) { original_render.call(fragment.gsub(WikitextRegions::COMMENT, "")) }
+      terms = []
+
+      if (first = bolds.first)
+        paragraph = paragraph_bounds(visible, first[0], lead_end)
+        bolds.select { |s, e| s >= paragraph[0] && e <= paragraph[1] }.first(MAX_TERMS).each do |s, e|
+          terms << bold_term(wikitext, s, e, paragraph[1], render)
+        end
+      end
+      rubies.each do |s, e, parts|
+        terms << { "text" => render.call(parts[1].to_s).strip, "reading" => render.call(parts[2].to_s).strip,
+                   "source" => parts[0].strip, "span" => { "template" => [s, e] } }
+      end
+
+      terms.sort_by { |t| t["span"].values.first.first }.first(MAX_TERMS)
+           .each_with_index.map { |t, i| { "index" => i }.merge(t) }
+    end
+
+    # Top-level bold spans [start, end) including the quote marks, and ruby
+    # templates [start, end, [name, text, reading]] found in the lead
+    def scan(text, limit, source: text)
+      bolds = []
+      rubies = []
+      i = 0
+      open_bold = nil
+      while i < limit
+        if (i.zero? || text[i - 1] == "\n") && text[i] == "=" &&
+           HEADING.match?(text[i...(text.index("\n", i) || limit)])
+          limit = i
+          break
+        elsif text[i] == "\n"
+          open_bold = nil # bold does not continue across lines
+          i += 1
+        elsif (close = SKIP_OPEN[text[i, 4] == "<!--" ? "<!--" : text[i, 2]])
+          opener = text[i, 4] == "<!--" ? "<!--" : text[i, 2]
+          stop = matching_end(text, i, opener, close)
+          if opener == "{{" && (ruby = ruby_template(source[(i + 2)...(stop - 2)]))
+            rubies << [i, stop, ruby]
+          end
+          i = stop
+        elsif text[i, 2] == "[[" && FILE_LINK.match?(text[i, 40])
+          i = matching_end(text, i, "[[", "]]")
+        elsif text[i] == "<" && (m = REF_SELF_CLOSING.match(text[i, 200]))
+          i += m[0].length
+        elsif text[i] == "<" && (m = REF_OPEN.match(text[i, 200]))
+          close_at = text.index(%r{</ref\s*>}i, i + m[0].length)
+          i = close_at ? text.index(">", close_at) + 1 : limit
+        elsif text[i, 3] == "'''"
+          run = text[i..][/\A'+/].length
+          if open_bold
+            bolds << [open_bold, i + run]
+            open_bold = nil
+          else
+            open_bold = i
+          end
+          i += run
+        else
+          i += 1
+        end
+      end
+      [bolds, rubies, limit]
+    end
+
+    # End index (exclusive) of the construct opened at start, honouring nesting
+    def matching_end(text, start, opener, closer)
+      depth = 0
+      i = start
+      while i < text.length
+        if text[i, opener.length] == opener
+          depth += 1
+          i += opener.length
+        elsif text[i, closer.length] == closer
+          depth -= 1
+          i += closer.length
+          return i if depth.zero?
+        else
+          i += 1
+        end
+      end
+      text.length
+    end
+
+    def ruby_template(content)
+      parts = split_top_level(content, ["|"])
+      name = parts.first.to_s
+      return nil unless ruby_name?(name)
+
+      [name, parts[1], parts[2]]
+    end
+
+    # Same name rule as the cleaner: "_" and " " alike, case-insensitive
+    def ruby_name?(name)
+      @ruby_names ||= Wp2txt::RUBY_TEXT_TEMPLATES.to_set { |t| t.tr("_", " ").strip.downcase }
+      @ruby_names.include?(name.to_s.tr("_", " ").strip.downcase)
+    end
+
+    def paragraph_bounds(text, pos, limit)
+      start = 0
+      text[0...pos].to_enum(:scan, PARAGRAPH_BREAK).each { start = Regexp.last_match.end(0) }
+      stop = text.index(PARAGRAPH_BREAK, pos) || limit
+      [start, [stop, limit].min]
+    end
+
+    def bold_term(text, s, e, limit, render)
+      inner = text[s...e].sub(/\A'+/, "").sub(/'+\z/, "")
+      term = { "text" => render.call(plain_ruby(inner)).strip, "notes" => [], "notes_text" => nil,
+               "source" => "bold", "span" => { "bold" => [s, e] } }
+      j = e
+      j += 1 while j < limit && [" ", "\t", "　", "\r", "\n"].include?(text[j])
+      return term if j >= limit
+      closer = OPENERS[text[j]]
+      return term unless closer
+
+      pe = paren_end(text, j, text[j], closer, limit)
+      return term unless pe
+
+      raw = text[(j + 1)...(pe - 1)]
+      term["notes_text"] = render.call(raw).strip
+      term["notes"] = split_top_level(raw, SEPARATORS).map { |part| render.call(part).strip }.reject(&:empty?)
+      term["span"]["paren"] = [j, pe]
+      term
+    end
+
+    # Index after the bracket closing the one at start, or nil if unclosed
+    def paren_end(text, start, opener, closer, limit)
+      depth = 0
+      i = start
+      while i < limit
+        two = text[i, 2]
+        if text[i] == "<" && (stop = WikitextRegions.end_at(text, i, lead: true))
+          i = stop
+          next
+        elsif ["{{", "[["].include?(two)
+          i = matching_end(text, i, two, two == "{{" ? "}}" : "]]")
+          next
+        end
+        if [opener, "(", "（"].include?(text[i])
+          depth += 1
+        elsif [closer, ")", "）"].include?(text[i])
+          depth -= 1
+          return i + 1 if depth.zero?
+        elsif text[i] == "\n" && /\A\n[ \t\r]*\n/.match?(text[i..])
+          return nil
+        end
+        i += 1
+      end
+      nil
+    end
+
+    # Split at separators that sit outside brackets, templates, and links
+    def split_top_level(text, separators)
+      parts = [+""]
+      depth = 0
+      i = 0
+      while i < text.length
+        two = text[i, 2]
+        if text[i] == "<" && (stop = WikitextRegions.end_at(text, i, lead: true))
+          parts.last << text[i...stop]
+          i = stop
+        elsif ["{{", "[["].include?(two)
+          depth += 1
+          parts.last << two
+          i += 2
+        elsif ["}}", "]]"].include?(two)
+          depth -= 1
+          parts.last << two
+          i += 2
+        else
+          ch = text[i]
+          depth += 1 if ["(", "（"].include?(ch)
+          depth -= 1 if [")", "）"].include?(ch)
+          if depth <= 0 && separators.include?(ch)
+            parts << +""
+          else
+            parts.last << ch
+          end
+          i += 1
+        end
+      end
+      parts
+    end
+
+    # In a bold headword, a ruby template stands for its base text; the
+    # reading is reported separately as its own term
+    def plain_ruby(fragment)
+      fragment.gsub(/\{\{([^{}]*)\}\}/) do |whole|
+        (ruby = ruby_template(Regexp.last_match(1))) ? ruby[1].to_s : whole
+      end
+    end
+  end
+end