wp2txt 2.3.4 → 2.4.0

data/lib/wp2txt/formatter.rb

@@ -14,21 +14,23 @@ module Wp2txt
 
     # Format article based on configuration and output format
     def format_article(article, config)
-      with_page_ids(format_article_body(article, config), article)
+      with_source_fields(format_article_body(article, config), article)
     end
 
-    # JSON records carry the dump's page and revision IDs right after the title,
-    # when the reader supplied them, so a record can be traced back to its source.
-    def with_page_ids(result, article)
-      return result unless result.is_a?(Hash) && (article.page_id || article.revision_id)
-
-      result.each_with_object({}) do |(key, value), out|
-        out[key] = value
-        next unless key == "title"
-
-        out["page_id"] = article.page_id
-        out["revision_id"] = article.revision_id
+    # JSON records carry the dump's page and revision IDs (and page properties,
+    # when imported) right after the title, so a record can be traced
+    # back to its source; lead terms, when requested, come last.
+    def with_source_fields(result, article)
+      return result unless result.is_a?(Hash)
+
+      ids = { "page_id" => article.page_id, "revision_id" => article.revision_id }.compact
+      ids.merge!(article.page_properties.transform_keys(&:to_s)) if article.page_properties
+      out = result.each_with_object({}) do |(key, value), acc|
+        acc[key] = value
+        acc.merge!(ids) if key == "title"
       end
+      out["lead_terms"] = article.lead_terms if article.lead_terms
+      out
     end
 
     def format_article_body(article, config)

data/lib/wp2txt/index_commands.rb

@@ -4,6 +4,8 @@ require "json"
 require_relative "metadata_index"
 require_relative "fts_index"
 require_relative "langlinks_importer"
+require_relative "page_props_importer"
+require_relative "link_counter"
 require_relative "corpus"
 require_relative "multistream"
 require_relative "memory_monitor"
@@ -322,11 +324,97 @@ module Wp2txt
         end
       end
       CliUI::EXIT_SUCCESS
-    rescue ArgumentError => e
+    rescue ArgumentError, Wp2txt::Error => e
       print_error(e.message)
       CliUI::EXIT_ERROR
     end
 
+    # Import each article's page properties from the page_props dump of the
+    # same date as the metadata index
+    def run_import_page_props(opts)
+      db_path, dump_date, manager = built_index_for(opts)
+      return CliUI::EXIT_ERROR unless db_path
+
+      source = opts[:page_props_file] || begin
+        print_header("Downloading page_props for '#{opts[:lang]}' (#{dump_date})")
+        manager.download_page_props(date: dump_date)
+      end
+      print_mode_banner("Import Page Properties", { "Source" => File.basename(source), "Metadata DB" => db_path })
+
+      time_start = Time.now
+      result = PagePropsImporter.new(db_path).import!(source, force: opts[:update_cache])
+      if result[:status] == :already_imported
+        print_success("Page properties already imported (at #{result[:imported_at]}, #{result[:row_count]} pages).")
+        print_info_message("Use -U/--update-cache to re-import.")
+      else
+        print_success("Page properties imported: #{result[:row_count]} pages in #{format_duration(Time.now - time_start)}")
+        provenance = result[:provenance]
+        print_info("Properties", "#{provenance[:qid_count]} QIDs, #{provenance[:disambiguation_count]} disambiguation flags, " \
+                                 "#{provenance[:sort_key_count]} sort keys; #{provenance[:skipped_invalid_sort_keys]} invalid sort keys skipped")
+        print_info("SHA-256", result[:provenance][:source_sha256].to_s)
+      end
+      CliUI::EXIT_SUCCESS
+    rescue ArgumentError, Wp2txt::Error => e
+      print_error(e.message)
+      CliUI::EXIT_ERROR
+    end
+
+    # Count incoming links to every article and store them in the metadata index
+    def run_count_links(opts)
+      db_path, _dump_date, manager = built_index_for(opts)
+      return CliUI::EXIT_ERROR unless db_path
+
+      multistream = manager.cached_multistream_path
+      stream_offsets, = load_stream_offsets(manager.cached_index_path, opts)
+      num_processes = opts[:num_procs] || MemoryMonitor.optimal_processes
+      print_mode_banner("Count Incoming Links", {
+        "Dump" => File.basename(multistream), "Streams" => stream_offsets.size.to_s,
+        "Processes" => num_processes.to_s
+      })
+
+      time_start = Time.now
+      last_report = Time.now
+      result = LinkCounter.new(multistream, stream_offsets, db_path: db_path,
+                                                           num_processes: num_processes).count! do |done, total|
+        now = Time.now
+        if !quiet? && (now - last_report >= DEFAULT_PROGRESS_INTERVAL || done == total)
+          last_report = now
+          puts pastel.dim(format("  [%s] %d/%d batches", now.strftime("%H:%M:%S"), done, total))
+        end
+      end
+      print_success("Incoming links counted for #{result[:articles]} articles " \
+                    "(#{result[:with_inlinks]} linked at least once) in #{format_duration(Time.now - time_start)}")
+      CliUI::EXIT_SUCCESS
+    rescue ArgumentError, Wp2txt::Error => e
+      print_error(e.message)
+      CliUI::EXIT_ERROR
+    end
+
+    # The metadata index of the cached dump for --lang, or nil with an error printed
+    # @return [Array(String, String, DumpManager)] [db_path, dump_date, manager]
+    def built_index_for(opts)
+      manager = DumpManager.new(opts[:lang], cache_dir: opts[:cache_dir],
+                                             dump_expiry_days: CLI.config.dump_expiry_days)
+      multistream = manager.cached_multistream_path
+      unless File.exist?(multistream)
+        print_error("No cached dump found for '#{opts[:lang]}'.")
+        print_info_message("Download and index it with: wp2txt --build-index -L #{opts[:lang]}")
+        return nil
+      end
+
+      db_path = MetadataIndex.path_for(multistream, cache_dir: opts[:cache_dir])
+      meta = MetadataIndex.new(db_path)
+      unless meta.built?
+        meta.close
+        print_error("Metadata index not found for this dump.")
+        print_info_message("Build it first with: wp2txt --build-index -L #{opts[:lang]}")
+        return nil
+      end
+      dump_date = meta.stats[:dump_name][/\d{8}\z/]
+      meta.close
+      [db_path, dump_date, manager]
+    end
+
     # Query the metadata index and print matching article titles
     def run_find_articles(opts)
       multistream_path, = resolve_dump_paths(opts, download: false)

data/lib/wp2txt/langlinks_importer.rb

@@ -4,6 +4,7 @@ require "sqlite3"
 require "set"
 require "time"
 require "zlib"
+require_relative "sql_dump_reader"
 require_relative "metadata_index"
 require_relative "version"
 
@@ -28,14 +29,6 @@ module Wp2txt
     SANITY_SAMPLE_SIZE = 1000
     SANITY_WARN_THRESHOLD = 0.9
 
-    INSERT_PREFIX = /\A\s*INSERT\s+INTO\s+`langlinks`\s+VALUES\s+/i
-
-    # MySQL backslash escapes inside mysqldump string literals
-    UNESCAPES = {
-      "0" => "\0", "'" => "'", '"' => '"', "b" => "\b", "n" => "\n",
-      "r" => "\r", "t" => "\t", "Z" => "\x1A", "\\" => "\\"
-    }.freeze
-
     def initialize(db_path, cache_dir: nil)
       @db_path = db_path
       @cache_dir = cache_dir
@@ -99,7 +92,9 @@ module Wp2txt
         batch.clear
       end
 
+      rows_seen = 0
       skipped_invalid = each_source_row(source_path) do |ll_from, ll_lang, ll_title|
+        rows_seen += 1
         next if lang_filter && !lang_filter.include?(ll_lang)
 
         batch << [ll_from, ll_lang, MetadataIndex.normalize_title(ll_title)]
@@ -107,6 +102,13 @@ module Wp2txt
       end
       flush.call unless batch.empty?
 
+      # Reporting success after reading nothing would leave every language
+      # query silently empty; an unreadable dump must fail loudly instead.
+      if rows_seen.zero? && skipped_invalid.zero?
+        raise Wp2txt::Error, "no langlinks rows found in #{File.basename(source_path)}; " \
+                             "the file may be empty or in an unrecognized format"
+      end
+
       # Indexes are created after the load, not before (insert speed)
       db.execute("CREATE INDEX idx_langlinks_from ON langlinks(ll_from, ll_lang)")
       db.execute("CREATE INDEX idx_langlinks_lang_title ON langlinks(ll_lang, ll_title)")
@@ -224,32 +226,18 @@ module Wp2txt
     # tagged UTF-8 and validated — a garbled title could never join
     # pages.title anyway, so such rows are skipped (and counted), not scrubbed.
     def each_source_row(source_path)
-      io = if source_path.end_with?(".gz")
-             # GzipReader ignores set_encoding; the encoding must be given
-             # at open time (lines must come out as BINARY — see below)
-             Zlib::GzipReader.open(source_path, encoding: Encoding::BINARY.to_s)
-           else
-             File.open(source_path, "rb")
-           end
-
       skipped = 0
-      begin
-        io.each_line do |line|
-          next unless INSERT_PREFIX.match?(line)
-
-          line.scan(TUPLE_REGEX) do |ll_from, ll_lang, ll_title|
-            lang = unescape_mysql(ll_lang).force_encoding(Encoding::UTF_8)
-            title = unescape_mysql(ll_title).force_encoding(Encoding::UTF_8)
-            unless lang.valid_encoding? && title.valid_encoding?
-              skipped += 1
-              next
-            end
-
-            yield ll_from.to_i, lang, title
+      SqlDumpReader.each_insert_line(source_path, "langlinks") do |line|
+        line.scan(TUPLE_REGEX) do |ll_from, ll_lang, ll_title|
+          lang = SqlDumpReader.unescape(ll_lang).force_encoding(Encoding::UTF_8)
+          title = SqlDumpReader.unescape(ll_title).force_encoding(Encoding::UTF_8)
+          unless lang.valid_encoding? && title.valid_encoding?
+            skipped += 1
+            next
           end
+
+          yield ll_from.to_i, lang, title
         end
-      ensure
-        io.close
       end
       skipped
     end
@@ -262,12 +250,8 @@ module Wp2txt
     # (equivalent to the old parser skipping malformed tuples)
     TUPLE_REGEX = /\((\d+),'((?:[^'\\]|\\.)*)','((?:[^'\\]|\\.)*)'\)/
 
-    UNESCAPE_REGEX = /\\(.)/m
 
     # Resolve MySQL backslash escapes in a captured string literal
     # (same mapping as the old hand-rolled parser)
-    def unescape_mysql(str)
-      str.gsub(UNESCAPE_REGEX) { UNESCAPES[::Regexp.last_match(1)] || ::Regexp.last_match(1) }
-    end
   end
 end

data/lib/wp2txt/lead_terms.rb

@@ -0,0 +1,228 @@
+# frozen_string_literal: true
+
+require_relative "wikitext_regions"
+
+module Wp2txt
+  # Finds the terms an article introduces in its lead: bold spans in the first
+  # paragraph that has one, the parenthesized notes written right after each,
+  # and reading (ruby) templates. Nothing is judged — which note is a reading,
+  # a native spelling, or a date is left to the caller.
+  #
+  # Positions are character offsets [start, end) into the article's wikitext
+  # as stored in the dump, XML entities decoded, before any other processing
+  # (comments included), so a caller holding the same text can cut out and
+  # hash exactly the span a term came from.
+  module LeadTerms
+    MAX_TERMS = 5
+    HEADING = /^={2,6}[^=\n].*?={2,6}[ \t]*$/
+    OPENERS = { "(" => ")", "（" => "）" }.freeze
+    SEPARATORS = ["、", ",", "，", ";", "；"].freeze
+    PARAGRAPH_BREAK = /\n[ \t\r]*\n/
+    # Regions whose bold text is not the article's own lead prose
+    SKIP_OPEN = { "{{" => "}}", "{|" => "|}", "<!--" => "-->" }.freeze
+    REF_OPEN = /\A<ref(?:\s[^>]*)?>/i
+    REF_SELF_CLOSING = /\A<ref(?:\s[^>]*)?\/>/i
+    FILE_LINK = /\A\[\[\s*(?:file|image|ファイル|画像|media)\s*:/i
+
+    module_function
+
+    # @param wikitext [String] decoded article wikitext
+    # @param render [#call] turns a wikitext fragment into clean text
+    # @return [Array<Hash>] terms in order of appearance, at most MAX_TERMS
+    def extract(wikitext, render:)
+      return [] if wikitext.nil? || wikitext.empty?
+
+      visible = WikitextRegions.mask_lead(wikitext)
+      bolds, rubies, lead_end = scan(visible, visible.length, source: wikitext)
+      original_render = render
+      render = ->(fragment) { original_render.call(fragment.gsub(WikitextRegions::COMMENT, "")) }
+      terms = []
+
+      if (first = bolds.first)
+        paragraph = paragraph_bounds(visible, first[0], lead_end)
+        bolds.select { |s, e| s >= paragraph[0] && e <= paragraph[1] }.first(MAX_TERMS).each do |s, e|
+          terms << bold_term(wikitext, s, e, paragraph[1], render)
+        end
+      end
+      rubies.each do |s, e, parts|
+        terms << { "text" => render.call(parts[1].to_s).strip, "reading" => render.call(parts[2].to_s).strip,
+                   "source" => parts[0].strip, "span" => { "template" => [s, e] } }
+      end
+
+      terms.sort_by { |t| t["span"].values.first.first }.first(MAX_TERMS)
+           .each_with_index.map { |t, i| { "index" => i }.merge(t) }
+    end
+
+    # Top-level bold spans [start, end) including the quote marks, and ruby
+    # templates [start, end, [name, text, reading]] found in the lead
+    def scan(text, limit, source: text)
+      bolds = []
+      rubies = []
+      i = 0
+      open_bold = nil
+      while i < limit
+        if (i.zero? || text[i - 1] == "\n") && text[i] == "=" &&
+           HEADING.match?(text[i...(text.index("\n", i) || limit)])
+          limit = i
+          break
+        elsif text[i] == "\n"
+          open_bold = nil # bold does not continue across lines
+          i += 1
+        elsif (close = SKIP_OPEN[text[i, 4] == "<!--" ? "<!--" : text[i, 2]])
+          opener = text[i, 4] == "<!--" ? "<!--" : text[i, 2]
+          stop = matching_end(text, i, opener, close)
+          if opener == "{{" && (ruby = ruby_template(source[(i + 2)...(stop - 2)]))
+            rubies << [i, stop, ruby]
+          end
+          i = stop
+        elsif text[i, 2] == "[[" && FILE_LINK.match?(text[i, 40])
+          i = matching_end(text, i, "[[", "]]")
+        elsif text[i] == "<" && (m = REF_SELF_CLOSING.match(text[i, 200]))
+          i += m[0].length
+        elsif text[i] == "<" && (m = REF_OPEN.match(text[i, 200]))
+          close_at = text.index(%r{</ref\s*>}i, i + m[0].length)
+          i = close_at ? text.index(">", close_at) + 1 : limit
+        elsif text[i, 3] == "'''"
+          run = text[i..][/\A'+/].length
+          if open_bold
+            bolds << [open_bold, i + run]
+            open_bold = nil
+          else
+            open_bold = i
+          end
+          i += run
+        else
+          i += 1
+        end
+      end
+      [bolds, rubies, limit]
+    end
+
+    # End index (exclusive) of the construct opened at start, honouring nesting
+    def matching_end(text, start, opener, closer)
+      depth = 0
+      i = start
+      while i < text.length
+        if text[i, opener.length] == opener
+          depth += 1
+          i += opener.length
+        elsif text[i, closer.length] == closer
+          depth -= 1
+          i += closer.length
+          return i if depth.zero?
+        else
+          i += 1
+        end
+      end
+      text.length
+    end
+
+    def ruby_template(content)
+      parts = split_top_level(content, ["|"])
+      name = parts.first.to_s
+      return nil unless ruby_name?(name)
+
+      [name, parts[1], parts[2]]
+    end
+
+    # Same name rule as the cleaner: "_" and " " alike, case-insensitive
+    def ruby_name?(name)
+      @ruby_names ||= Wp2txt::RUBY_TEXT_TEMPLATES.to_set { |t| t.tr("_", " ").strip.downcase }
+      @ruby_names.include?(name.to_s.tr("_", " ").strip.downcase)
+    end
+
+    def paragraph_bounds(text, pos, limit)
+      start = 0
+      text[0...pos].to_enum(:scan, PARAGRAPH_BREAK).each { start = Regexp.last_match.end(0) }
+      stop = text.index(PARAGRAPH_BREAK, pos) || limit
+      [start, [stop, limit].min]
+    end
+
+    def bold_term(text, s, e, limit, render)
+      inner = text[s...e].sub(/\A'+/, "").sub(/'+\z/, "")
+      term = { "text" => render.call(plain_ruby(inner)).strip, "notes" => [], "notes_text" => nil,
+               "source" => "bold", "span" => { "bold" => [s, e] } }
+      j = e
+      j += 1 while j < limit && [" ", "\t", "　", "\r", "\n"].include?(text[j])
+      return term if j >= limit
+      closer = OPENERS[text[j]]
+      return term unless closer
+
+      pe = paren_end(text, j, text[j], closer, limit)
+      return term unless pe
+
+      raw = text[(j + 1)...(pe - 1)]
+      term["notes_text"] = render.call(raw).strip
+      term["notes"] = split_top_level(raw, SEPARATORS).map { |part| render.call(part).strip }.reject(&:empty?)
+      term["span"]["paren"] = [j, pe]
+      term
+    end
+
+    # Index after the bracket closing the one at start, or nil if unclosed
+    def paren_end(text, start, opener, closer, limit)
+      depth = 0
+      i = start
+      while i < limit
+        two = text[i, 2]
+        if text[i] == "<" && (stop = WikitextRegions.end_at(text, i, lead: true))
+          i = stop
+          next
+        elsif ["{{", "[["].include?(two)
+          i = matching_end(text, i, two, two == "{{" ? "}}" : "]]")
+          next
+        end
+        if [opener, "(", "（"].include?(text[i])
+          depth += 1
+        elsif [closer, ")", "）"].include?(text[i])
+          depth -= 1
+          return i + 1 if depth.zero?
+        elsif text[i] == "\n" && /\A\n[ \t\r]*\n/.match?(text[i..])
+          return nil
+        end
+        i += 1
+      end
+      nil
+    end
+
+    # Split at separators that sit outside brackets, templates, and links
+    def split_top_level(text, separators)
+      parts = [+""]
+      depth = 0
+      i = 0
+      while i < text.length
+        two = text[i, 2]
+        if text[i] == "<" && (stop = WikitextRegions.end_at(text, i, lead: true))
+          parts.last << text[i...stop]
+          i = stop
+        elsif ["{{", "[["].include?(two)
+          depth += 1
+          parts.last << two
+          i += 2
+        elsif ["}}", "]]"].include?(two)
+          depth -= 1
+          parts.last << two
+          i += 2
+        else
+          ch = text[i]
+          depth += 1 if ["(", "（"].include?(ch)
+          depth -= 1 if [")", "）"].include?(ch)
+          if depth <= 0 && separators.include?(ch)
+            parts << +""
+          else
+            parts.last << ch
+          end
+          i += 1
+        end
+      end
+      parts
+    end
+
+    # In a bold headword, a ruby template stands for its base text; the
+    # reading is reported separately as its own term
+    def plain_ruby(fragment)
+      fragment.gsub(/\{\{([^{}]*)\}\}/) do |whole|
+        (ruby = ruby_template(Regexp.last_match(1))) ? ruby[1].to_s : whole
+      end
+    end
+  end
+end

data/lib/wp2txt/link_counter.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+require "parallel"
+require "sqlite3"
+require "time"
+require_relative "metadata_index"
+require_relative "version"
+require_relative "text_processing"
+require_relative "wikitext_regions"
+
+module Wp2txt
+  # Counts, for every article, how many other articles link to it, and stores
+  # the result in the metadata index as page_inlinks(page_id, inlinks,
+  # via_redirects).
+  #
+  # Rules (recorded as RULE_VERSION so counts from different rules are never
+  # compared unknowingly):
+  # - sources are articles (namespace 0) that are not redirects
+  # - each source counts once per target, however many times it links there
+  # - a link to a redirect counts for the redirect's target (one hop);
+  #   via_redirects is how many sources reached the target only that way
+  # - only links written in the article's own wikitext count; links that
+  #   templates add when rendered (navigation boxes) are not in the dump text
+  # - commented-out links do not count
+  class LinkCounter
+    include Wp2txt
+
+    RULE_VERSION = "2"
+    STREAMS_PER_BATCH = 50
+    LINK_REGEX = /\[\[([^\[\]|<>{}\n]+)(?=\||\]\])/
+    COMMENT_REGEX = /<!--.*?-->/m
+
+    def initialize(multistream_path, stream_offsets, db_path:, num_processes: 4)
+      @multistream_path = multistream_path
+      @stream_offsets = stream_offsets
+      @db_path = db_path
+      @num_processes = num_processes
+    end
+
+    # @return [Hash] { articles:, with_inlinks:, rule_version: }
+    def count!(&progress)
+      titles, redirects = load_titles
+      # Inherited by the forked workers (copy-on-write); they only read it
+      @redirects = redirects
+
+      direct = Hash.new(0)
+      via = Hash.new(0)
+      pairs = @stream_offsets.zip(@stream_offsets[1..].to_a + [nil])
+      batches = pairs.each_slice(STREAMS_PER_BATCH).to_a
+      done = 0
+
+      Parallel.each(
+        batches,
+        in_processes: @num_processes,
+        finish: lambda { |_item, _idx, result|
+          result[:direct].each { |t, n| direct[t] += n }
+          result[:via].each { |t, n| via[t] += n }
+          done += 1
+          progress&.call(done, batches.size)
+        }
+      ) do |batch|
+        scan_batch(batch)
+      end
+
+      write(titles, direct, via)
+    end
+
+    private
+
+    # Articles (title => page_id) and redirects (title => target title)
+    def load_titles
+      db = SQLite3::Database.new(@db_path, readonly: true)
+      @case_rule = db.get_first_value("SELECT value FROM metadata WHERE key = 'case_rule'") || "first-letter"
+      titles = {}
+      redirects = {}
+      db.execute("SELECT page_id, title, redirect_to FROM pages WHERE namespace = 0") do |id, title, target|
+        if target
+          redirects[title] = normalize_target(target)
+        else
+          titles[title] = id
+        end
+      end
+      [titles, redirects]
+    ensure
+      db&.close
+    end
+
+    # Runs in a worker: returns per-target counts for this batch of streams
+    def scan_batch(offset_pairs)
+      direct = Hash.new(0)
+      via = Hash.new(0)
+      File.open(@multistream_path, "rb") do |f|
+        offset_pairs.each do |offset, next_offset|
+          f.seek(offset)
+          data = next_offset ? f.read(next_offset - offset) : f.read
+          xml = MetadataIndexBuilder.decompress_bz2(data)
+          xml.scan(MetadataIndexBuilder::PAGE_BLOCK_REGEX) do
+            count_page(::Regexp.last_match(1), direct, via)
+          end
+        end
+      end
+      { direct: direct, via: via }
+    end
+
+    def count_page(block, direct, via)
+      return unless Wp2txt.namespace_id(block[MetadataIndexBuilder::NS_REGEX, 1]).zero?
+
+      text = MetadataIndexBuilder.unescape_xml(block[MetadataIndexBuilder::TEXT_REGEX, 1] || "")
+      return if Wp2txt::REDIRECT_REGEX.match?(text)
+
+      reached = {} # target => true if reached directly at least once
+      WikitextRegions.remove_literal(text).scan(LINK_REGEX) do |(raw)|
+        name = normalize_target(raw)
+        next if name.empty?
+
+        if (target = @redirects[name])
+          reached[target] ||= false
+        else
+          reached[name] = true
+        end
+      end
+      reached.each do |target, directly|
+        direct[target] += 1
+        via[target] += 1 unless directly
+      end
+    end
+
+    def normalize_target(raw)
+      # Decode each reference in the original input, never references created
+      # by decoding another one (special_chr has two decoding stages).
+      decoded = raw.gsub(/&(?:#[xX][0-9a-fA-F]+|#\d+|[a-zA-Z][a-zA-Z0-9]*);/) { |entity| special_chr(entity) }
+      title = decoded.split("#", 2).first.to_s.strip.sub(/\A:/, "")
+      MetadataIndex.normalize_title(title, case_rule: @case_rule || "first-letter")
+    end
+
+    def write(titles, direct, via)
+      db = SQLite3::Database.new(@db_path)
+      db.busy_timeout = 5000
+      db.execute("DROP TABLE IF EXISTS page_inlinks")
+      db.execute("DELETE FROM metadata WHERE key LIKE 'links\\_%' ESCAPE '\\'")
+      db.execute(<<~SQL)
+        CREATE TABLE page_inlinks (
+          page_id       INTEGER PRIMARY KEY,
+          inlinks       INTEGER NOT NULL,
+          via_redirects INTEGER NOT NULL
+        )
+      SQL
+      with_inlinks = 0
+      db.transaction do
+        stmt = db.prepare("INSERT INTO page_inlinks (page_id, inlinks, via_redirects) VALUES (?, ?, ?)")
+        titles.each do |title, page_id|
+          n = direct[title]
+          with_inlinks += 1 if n.positive?
+          stmt.execute([page_id, n, via[title]])
+        end
+        stmt.close
+      end
+      db.execute("CREATE INDEX idx_page_inlinks_count ON page_inlinks(inlinks)")
+      {
+        links_counted_at: Time.now.utc.iso8601,
+        links_rule_version: RULE_VERSION,
+        links_wp2txt_version: Wp2txt::VERSION,
+        links_article_count: titles.size,
+        links_with_inlinks: with_inlinks
+      }.each { |k, v| db.execute("INSERT OR REPLACE INTO metadata (key, value) VALUES (?, ?)", [k.to_s, v.to_s]) }
+      { articles: titles.size, with_inlinks: with_inlinks, rule_version: RULE_VERSION }
+    ensure
+      db&.close
+    end
+  end
+end