wp2txt 2.1.2 → 2.3.0

data/lib/wp2txt/index_commands.rb

@@ -0,0 +1,427 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "metadata_index"
+require_relative "fts_index"
+require_relative "langlinks_importer"
+require_relative "corpus"
+require_relative "multistream"
+require_relative "memory_monitor"
+
+module Wp2txt
+  # CLI command handlers for --build-index and --find-articles.
+  # Mixed into WpApp (bin/wp2txt); relies on CliUI helpers for output.
+  module IndexCommands
+    # Build (or refresh) the local metadata index for a dump
+    def run_build_index(opts)
+      multistream_path, index_path = resolve_dump_paths(opts, download: true)
+      return CliUI::EXIT_ERROR unless multistream_path
+
+      db_path = MetadataIndex.path_for(multistream_path, cache_dir: opts[:cache_dir])
+      meta = MetadataIndex.new(db_path)
+
+      if meta.valid_for?(multistream_path) && !opts[:update_cache]
+        print_success("Metadata index is up to date: #{db_path}")
+        print_index_stats(meta.stats)
+        meta.close
+        return CliUI::EXIT_SUCCESS unless opts[:fulltext]
+
+        stream_offsets, = load_stream_offsets(index_path, opts)
+        num_processes = opts[:num_procs] || MemoryMonitor.optimal_processes
+        return build_fulltext_index(opts, multistream_path, stream_offsets, db_path, num_processes)
+      end
+      meta.close
+
+      print_mode_banner("Build Metadata Index", {
+        "Dump" => File.basename(multistream_path),
+        "Output" => db_path
+      })
+
+      time_start = Time.now
+      puts pastel.cyan("Loading multistream index...") unless quiet?
+      stream_offsets, entry_count = load_stream_offsets(index_path, opts)
+      if stream_offsets.empty?
+        print_error("Multistream index is empty or unreadable: #{index_path}")
+        return CliUI::EXIT_ERROR
+      end
+
+      num_processes = opts[:num_procs] || MemoryMonitor.optimal_processes
+      puts pastel.cyan("Scanning #{entry_count} pages in #{stream_offsets.size} streams (#{num_processes} processes)...") unless quiet?
+
+      ok = run_phase_isolated do
+        builder = MetadataIndexBuilder.new(
+          multistream_path, stream_offsets,
+          db_path: db_path, num_processes: num_processes
+        )
+
+        last_report = Time.now
+        built = builder.build do |done, total|
+          now = Time.now
+          if !quiet? && (now - last_report >= DEFAULT_PROGRESS_INTERVAL || done == total)
+            last_report = now
+            percent = (done.to_f / total * 100).round(1)
+            elapsed = now - time_start
+            eta = done.positive? ? (total - done) * (elapsed / done) : 0
+            puts pastel.dim(format("  [%d/%d] %.1f%% | ETA: %s", done, total, percent, format_duration(eta)))
+          end
+        end
+
+        puts unless quiet?
+        print_success("Metadata index built in #{format_duration(Time.now - time_start)}")
+        print_index_stats(built.stats)
+        built.close
+        true
+      end
+      return CliUI::EXIT_ERROR unless ok
+
+      return build_fulltext_index(opts, multistream_path, stream_offsets, db_path, num_processes) if opts[:fulltext]
+
+      CliUI::EXIT_SUCCESS
+    end
+
+    # Run a build phase in a forked child process. Each phase pumps its whole
+    # dataset through the parent's heap (Marshal.load of every worker batch),
+    # leaving a multi-GB high-water mark; a later phase forking workers from
+    # that bloated parent duplicates it per worker via copy-on-write breakage
+    # and OOMs the machine (observed on enwiki: 18GB per FTS worker). A child
+    # process confines each phase's high-water mark to that phase.
+    # @return [Boolean] whether the phase succeeded
+    def run_phase_isolated(&block)
+      return block.call unless Process.respond_to?(:fork)
+
+      pid = Process.fork do
+        status = 1
+        begin
+          status = block.call ? 0 : 1
+        rescue StandardError => e
+          warn "#{e.class}: #{e.message}"
+        end
+        $stdout.flush
+        $stderr.flush
+        exit!(status) # skip at_exit handlers; they belong to the parent
+      end
+      _, status = Process.waitpid2(pid)
+      status.success?
+    end
+
+    # Load stream offsets without holding the full title index in memory.
+    # The builders fork worker processes; a parent heap holding millions of
+    # index entries (~14GB for enwiki) gets duplicated through copy-on-write
+    # breakage in every child and OOMs the machine. Offsets are a small
+    # integer array, so prefer reading just them from the SQLite cache.
+    # @return [Array(Array<Integer>, Integer)] [stream_offsets, entry_count]
+    def load_stream_offsets(index_path, opts)
+      cache = IndexCache.new(index_path, cache_dir: opts[:cache_dir])
+      if cache.valid?
+        [cache.stream_offsets, cache.stats[:entry_count]]
+      else
+        # First run: parse the index (this also writes the SQLite cache),
+        # then discard the entry hashes before any fork
+        ms_index = MultistreamIndex.new(index_path, cache_dir: opts[:cache_dir], show_progress: !quiet?)
+        offsets = ms_index.stream_offsets
+        count = ms_index.size
+        ms_index = nil
+        GC.start
+        GC.compact if GC.respond_to?(:compact)
+        [offsets, count]
+      end
+    end
+
+    # Build the FTS5 full-text index (Tier 2) after the metadata index
+    def build_fulltext_index(opts, multistream_path, stream_offsets, meta_db_path, num_processes)
+      fts_db_path = FtsIndex.path_for(multistream_path, cache_dir: opts[:cache_dir])
+      tokenizer = opts[:fts_tokenizer] || FtsIndex.default_tokenizer(multistream_path)
+
+      fts = FtsIndex.new(fts_db_path, meta_db_path)
+      if fts.valid_for?(multistream_path) && !opts[:update_cache]
+        print_success("Full-text index is up to date: #{fts_db_path}")
+        fts.close
+        return CliUI::EXIT_SUCCESS
+      end
+      fts.close
+
+      puts pastel.cyan("Building full-text index (tokenizer: #{tokenizer})...") unless quiet?
+      time_start = Time.now
+      ok = run_phase_isolated do
+        builder = FtsIndexBuilder.new(
+          multistream_path, stream_offsets,
+          db_path: fts_db_path, meta_db_path: meta_db_path,
+          tokenizer: tokenizer, num_processes: num_processes,
+          optimize: !opts[:skip_fts_optimize]
+        )
+
+        last_report = Time.now
+        built = builder.build do |done, total|
+          now = Time.now
+          if !quiet? && (now - last_report >= DEFAULT_PROGRESS_INTERVAL || done == total)
+            last_report = now
+            percent = (done.to_f / total * 100).round(1)
+            elapsed = now - time_start
+            eta = done.positive? ? (total - done) * (elapsed / done) : 0
+            puts pastel.dim(format("  [%d/%d] %.1f%% | ETA: %s", done, total, percent, format_duration(eta)))
+          end
+        end
+
+        puts unless quiet?
+        print_success("Full-text index built in #{format_duration(Time.now - time_start)}")
+        stats = built.stats
+        print_info("Tokenizer", stats[:tokenizer].to_s)
+        print_info("Sections", stats[:section_count].to_s)
+        print_info("Size", format_size(stats[:db_size]))
+        unless stats[:optimized]
+          print_info_message("Index is unoptimized (built with --skip-fts-optimize). Run 'wp2txt --fts-optimize' later for best query speed.")
+        end
+        built.close
+        true
+      end
+      ok ? CliUI::EXIT_SUCCESS : CliUI::EXIT_ERROR
+    end
+
+    # Standalone optimize of an existing full-text index (--fts-optimize)
+    def run_fts_optimize(opts)
+      multistream_path, = resolve_dump_paths(opts, download: false)
+      return CliUI::EXIT_ERROR unless multistream_path
+
+      fts = FtsIndex.new(
+        FtsIndex.path_for(multistream_path, cache_dir: opts[:cache_dir]),
+        MetadataIndex.path_for(multistream_path, cache_dir: opts[:cache_dir])
+      )
+      unless fts.built?
+        print_error("Full-text index not found for this dump.")
+        print_info_message("Build it first with: wp2txt --build-index --fulltext #{opts[:lang] ? "-L #{opts[:lang]}" : "-i #{opts[:input]}"}")
+        return CliUI::EXIT_ERROR
+      end
+
+      if fts.optimized?
+        print_success("Full-text index is already optimized.")
+        fts.close
+        return CliUI::EXIT_SUCCESS
+      end
+
+      print_info_message("Optimizing full-text index (single-threaded; can take 30-60+ minutes on large indexes)...")
+      time_start = Time.now
+      fts.optimize!
+      print_success("Optimize complete in #{format_duration(Time.now - time_start)}")
+      print_info("Size", format_size(File.size(fts.db_path)))
+      fts.close
+      CliUI::EXIT_SUCCESS
+    end
+
+    # Full-text search from the CLI (--search)
+    def run_search(opts)
+      multistream_path, = resolve_dump_paths(opts, download: false)
+      return CliUI::EXIT_ERROR unless multistream_path
+
+      corpus = Corpus.new(
+        multistream_path: multistream_path,
+        index_path: resolve_dump_paths(opts, download: false)[1],
+        cache_dir: opts[:cache_dir]
+      )
+
+      unless corpus.fts.built?
+        print_error("Full-text index not found for this dump.")
+        print_info_message("Build it first with: wp2txt --build-index --fulltext #{opts[:lang] ? "-L #{opts[:lang]}" : "-i #{opts[:input]}"}")
+        return CliUI::EXIT_ERROR
+      end
+
+      result = corpus.search_text(
+        opts[:search],
+        sections: opts[:has_section] ? [opts[:has_section]] : nil,
+        category: opts[:in_category],
+        depth: opts[:in_category] ? opts[:depth] : 0,
+        limit: opts[:limit].positive? ? opts[:limit] : 20,
+        count: "exact"
+      )
+
+      if opts[:format].to_s.downcase == "json"
+        puts JSON.generate(result)
+      else
+        result[:hits].each do |hit|
+          puts "#{hit[:section_path]}: #{hit[:snippet]}"
+        end
+        $stderr.puts pastel.dim("# #{result[:returned]} of #{result[:total]} matches (dump: #{result[:dump]})")
+      end
+      corpus.close
+      CliUI::EXIT_SUCCESS
+    end
+
+    # Import the official langlinks dump (interlanguage links) into the
+    # metadata index. Version pinning: the langlinks file must carry the same
+    # dump date as the built index (enforced by LanglinksImporter, no override)
+    def run_import_langlinks(opts)
+      manager = DumpManager.new(
+        opts[:lang],
+        cache_dir: opts[:cache_dir],
+        dump_expiry_days: CLI.config.dump_expiry_days
+      )
+      multistream = manager.cached_multistream_path
+      unless File.exist?(multistream)
+        print_error("No cached dump found for '#{opts[:lang]}'.")
+        print_info_message("Download and index it with: wp2txt --build-index -L #{opts[:lang]}")
+        return CliUI::EXIT_ERROR
+      end
+
+      db_path = MetadataIndex.path_for(multistream, cache_dir: opts[:cache_dir])
+      meta = MetadataIndex.new(db_path)
+      unless meta.built?
+        meta.close
+        print_error("Metadata index not found for this dump.")
+        print_info_message("Build it first with: wp2txt --build-index -L #{opts[:lang]}")
+        return CliUI::EXIT_ERROR
+      end
+
+      dump_name = meta.stats[:dump_name]
+      dump_date = dump_name[/\d{8}\z/]
+      meta.close
+
+      source = opts[:langlinks_file] || begin
+        print_header("Downloading langlinks for '#{opts[:lang]}' (#{dump_date})")
+        manager.download_langlinks(date: dump_date)
+      end
+
+      langs = opts[:langlinks_langs]&.split(",")&.map(&:strip)&.reject(&:empty?)
+      langs = nil if langs&.empty?
+
+      print_mode_banner("Import Langlinks", {
+        "Source" => File.basename(source),
+        "Metadata DB" => db_path,
+        "Languages" => langs ? langs.join(",") : "all"
+      })
+
+      importer = LanglinksImporter.new(db_path, cache_dir: opts[:cache_dir])
+      time_start = Time.now
+      last_report = Time.now
+      result = importer.import!(source, langs: langs, force: opts[:update_cache]) do |rows|
+        now = Time.now
+        if !quiet? && now - last_report >= DEFAULT_PROGRESS_INTERVAL
+          last_report = now
+          puts pastel.dim(format("  [%s] %d rows imported", now.strftime("%H:%M:%S"), rows))
+        end
+      end
+
+      if result[:status] == :already_imported
+        print_success("Langlinks already imported (at #{result[:imported_at]}, #{result[:row_count]} rows).")
+        print_info_message("Use -U/--update-cache to re-import.")
+      else
+        print_success("Langlinks imported: #{result[:row_count]} rows in #{format_duration(Time.now - time_start)}")
+        prov = result[:provenance]
+        print_info("Source", prov[:source].to_s)
+        print_info("Languages", prov[:lang_filter].to_s)
+        if result[:skipped_invalid].to_i.positive?
+          print_warning("Skipped #{result[:skipped_invalid]} rows containing invalid UTF-8 bytes (recorded as langlinks_skipped_invalid)")
+        end
+        (result[:sanity] || []).each do |check|
+          msg = format("join check ll_lang=%s: %d/%d titles found in %s (%.1f%%)",
+                       check[:lang], check[:matched], check[:sampled],
+                       check[:against], check[:match_rate] * 100)
+          if check[:warning]
+            print_warning("#{msg} — below 90%; title normalization may mismatch")
+          else
+            print_info_message(msg)
+          end
+        end
+      end
+      CliUI::EXIT_SUCCESS
+    rescue ArgumentError => e
+      print_error(e.message)
+      CliUI::EXIT_ERROR
+    end
+
+    # Query the metadata index and print matching article titles
+    def run_find_articles(opts)
+      multistream_path, = resolve_dump_paths(opts, download: false)
+      return CliUI::EXIT_ERROR unless multistream_path
+
+      db_path = MetadataIndex.path_for(multistream_path, cache_dir: opts[:cache_dir])
+      meta = MetadataIndex.new(db_path)
+
+      unless meta.built?
+        print_error("Metadata index not found for this dump.")
+        print_info_message("Build it first with: wp2txt --build-index #{opts[:lang] ? "-L #{opts[:lang]}" : "-i #{opts[:input]}"}")
+        return CliUI::EXIT_ERROR
+      end
+
+      unless meta.valid_for?(multistream_path)
+        print_warning("Metadata index was built from a different version of this dump. Consider re-running --build-index.")
+      end
+
+      filters = {
+        category: opts[:in_category],
+        depth: opts[:in_category] ? opts[:depth] : 0,
+        has_section: opts[:has_section],
+        use_aliases: !opts[:no_section_aliases],
+        alias_file: opts[:alias_file],
+        title_match: opts[:title_match]
+      }
+
+      total = meta.count_articles(**filters)
+      titles = meta.find_articles(**filters, limit: opts[:limit])
+      dump_name = meta.stats[:dump_name]
+      meta.close
+
+      if opts[:format].to_s.downcase == "json"
+        puts JSON.generate({ dump: dump_name, total: total, returned: titles.size, titles: titles })
+      else
+        titles.each { |t| puts t }
+        $stderr.puts pastel.dim("# #{titles.size} of #{total} matching articles (dump: #{dump_name})")
+      end
+      CliUI::EXIT_SUCCESS
+    end
+
+    private
+
+    def print_index_stats(stats)
+      return unless stats && !quiet?
+
+      print_info("Dump", stats[:dump_name].to_s)
+      print_info("Pages", stats[:page_count].to_s)
+      print_info("Articles", stats[:article_count].to_s)
+      print_info("Categories", stats[:category_count].to_s)
+      print_info("Sections", stats[:section_count].to_s)
+      print_info("Size", format_size(stats[:db_size]))
+    end
+
+    # Resolve [multistream_path, index_path] from --lang (cached dump) or --input.
+    # Returns [nil, nil] after printing an error when files cannot be located.
+    def resolve_dump_paths(opts, download: false)
+      if opts[:lang]
+        manager = DumpManager.new(
+          opts[:lang],
+          cache_dir: opts[:cache_dir],
+          dump_expiry_days: CLI.config.dump_expiry_days
+        )
+        multistream = manager.cached_multistream_path
+        index = manager.cached_index_path
+        unless File.exist?(multistream) && File.exist?(index)
+          unless download
+            print_error("No cached dump found for '#{opts[:lang]}'.")
+            print_info_message("Download and index it with: wp2txt --build-index -L #{opts[:lang]}")
+            return [nil, nil]
+          end
+          print_header("Downloading dump files for '#{opts[:lang]}'")
+          manager.download_index
+          manager.download_multistream
+        end
+        [multistream, index]
+      else
+        multistream = opts[:input]
+        index = locate_index_file(multistream)
+        unless index
+          print_error("Could not find the multistream index file for #{multistream}")
+          print_info_message("Expected e.g. #{File.basename(multistream).sub(/multistream\.xml\.bz2\z/, 'multistream-index.txt.bz2')} next to the dump.")
+          return [nil, nil]
+        end
+        [multistream, index]
+      end
+    end
+
+    def locate_index_file(multistream_path)
+      candidates = [
+        multistream_path.sub(/multistream\.xml\.bz2\z/, "multistream-index.txt.bz2"),
+        multistream_path.sub(/\.xml\.bz2\z/, "-index.txt.bz2"),
+        multistream_path.sub(/\.xml\.bz2\z/, "-index.txt")
+      ].uniq
+      candidates.find { |c| c != multistream_path && File.exist?(c) }
+    end
+  end
+end

data/lib/wp2txt/langlinks_importer.rb

@@ -0,0 +1,273 @@
+# frozen_string_literal: true
+
+require "sqlite3"
+require "set"
+require "time"
+require "zlib"
+require_relative "metadata_index"
+require_relative "version"
+
+module Wp2txt
+  # Imports the official langlinks dump ({lang}wiki-{date}-langlinks.sql.gz,
+  # MySQL dump format) into the Tier 1 metadata DB as a `langlinks` table
+  # (ll_from = source page_id, ll_lang = target language, ll_title = title in
+  # the target edition, normalized like pages.title).
+  #
+  # Version pinning is the reason this feature exists: the langlinks file's
+  # dump name (e.g. jawiki-20260701) MUST equal the metadata DB's dump_name;
+  # a mismatch is rejected with no override.
+  class LanglinksImporter
+    # Rows inserted per transaction (index creation is deferred until after
+    # the load, so inserts stay fast)
+    BATCH_SIZE = 10_000
+
+    # Post-import sanity check (design doc §1.6): per target language, join a
+    # random sample of ll_title values against the target language's local
+    # meta DB (when installed) and report the match rate; a low rate signals
+    # a title-normalization mismatch
+    SANITY_SAMPLE_SIZE = 1000
+    SANITY_WARN_THRESHOLD = 0.9
+
+    INSERT_PREFIX = /\A\s*INSERT\s+INTO\s+`langlinks`\s+VALUES\s+/i
+
+    # MySQL backslash escapes inside mysqldump string literals
+    UNESCAPES = {
+      "0" => "\0", "'" => "'", '"' => '"', "b" => "\b", "n" => "\n",
+      "r" => "\r", "t" => "\t", "Z" => "\x1A", "\\" => "\\"
+    }.freeze
+
+    def initialize(db_path, cache_dir: nil)
+      @db_path = db_path
+      @cache_dir = cache_dir
+    end
+
+    # "jawiki-20260701-langlinks.sql.gz" => "jawiki-20260701" (same extraction
+    # rule as the dump_name recorded in the metadata DB)
+    def self.dump_name_of(path)
+      File.basename(path)[/\A[a-z0-9_\-]+?-\d{8}/]
+    end
+
+    # @param source_path [String] langlinks .sql or .sql.gz file
+    # @param langs [Array<String>, nil] target languages to import (nil = all)
+    # @param force [Boolean] drop and re-import an existing langlinks table
+    # @param progress [Proc, nil] called with the running row count per batch
+    # @return [Hash] { status: :imported | :already_imported, ... }
+    def import!(source_path, langs: nil, force: false, progress: nil)
+      raise ArgumentError, "langlinks file not found: #{source_path}" unless File.exist?(source_path)
+
+      db = open_db
+      dump_name = metadata_value(db, "dump_name")
+      raise ArgumentError, "metadata index is not built: #{@db_path}" unless dump_name
+
+      source_dump = self.class.dump_name_of(source_path)
+      unless source_dump && source_dump == dump_name
+        raise ArgumentError,
+              "dump version mismatch: the metadata index is #{dump_name} but the langlinks file is " \
+              "#{source_dump || File.basename(source_path)} (versions must match; there is no override)"
+      end
+
+      if !force && (existing = imported_at(db))
+        return { status: :already_imported, imported_at: existing,
+                 row_count: db.get_first_value("SELECT COUNT(*) FROM langlinks").to_i }
+      end
+
+      lang_filter = langs && Set.new(langs)
+
+      db.execute("DROP TABLE IF EXISTS langlinks")
+      # Clear stale provenance immediately: if the load below fails midway,
+      # the DB must be left as "not imported" (partial table only), so the
+      # next non-force run re-imports instead of reporting a stale success
+      db.execute("DELETE FROM metadata WHERE key LIKE 'langlinks\\_%' ESCAPE '\\'")
+      db.execute(<<~SQL)
+        CREATE TABLE langlinks (
+          ll_from  INTEGER NOT NULL,
+          ll_lang  TEXT    NOT NULL,
+          ll_title TEXT    NOT NULL
+        )
+      SQL
+
+      row_count = 0
+      batch = []
+      flush = lambda do
+        db.transaction do
+          stmt = db.prepare("INSERT INTO langlinks (ll_from, ll_lang, ll_title) VALUES (?, ?, ?)")
+          batch.each { |row| stmt.execute(row) }
+          stmt.close
+        end
+        row_count += batch.size
+        progress&.call(row_count)
+        batch.clear
+      end
+
+      skipped_invalid = each_source_row(source_path) do |ll_from, ll_lang, ll_title|
+        next if lang_filter && !lang_filter.include?(ll_lang)
+
+        batch << [ll_from, ll_lang, MetadataIndex.normalize_title(ll_title)]
+        flush.call if batch.size >= BATCH_SIZE
+      end
+      flush.call unless batch.empty?
+
+      # Indexes are created after the load, not before (insert speed)
+      db.execute("CREATE INDEX idx_langlinks_from ON langlinks(ll_from, ll_lang)")
+      db.execute("CREATE INDEX idx_langlinks_lang_title ON langlinks(ll_lang, ll_title)")
+
+      stamp_provenance(db, source_path, langs, row_count, skipped_invalid)
+
+      { status: :imported, row_count: row_count, skipped_invalid: skipped_invalid,
+        provenance: read_provenance(db),
+        sanity: sanity_check(db, dump_name) }
+    ensure
+      db&.close
+    end
+
+    private
+
+    def open_db
+      db = SQLite3::Database.new(@db_path)
+      db.busy_timeout = 5000
+      db
+    end
+
+    def metadata_value(db, key)
+      db.get_first_value("SELECT value FROM metadata WHERE key = ?", [key])
+    rescue SQLite3::Exception
+      nil
+    end
+
+    # Non-nil only when a previous completed import is still in place
+    def imported_at(db)
+      table = db.get_first_value(
+        "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'langlinks'"
+      )
+      table && metadata_value(db, "langlinks_imported_at")
+    end
+
+    def stamp_provenance(db, source_path, langs, row_count, skipped_invalid)
+      values = {
+        langlinks_source: File.basename(source_path),
+        langlinks_source_size: File.size(source_path),
+        langlinks_imported_at: Time.now.utc.iso8601,
+        langlinks_wp2txt_version: Wp2txt::VERSION,
+        langlinks_lang_filter: langs.nil? ? "all" : langs.join(","),
+        langlinks_row_count: row_count,
+        langlinks_skipped_invalid: skipped_invalid
+      }
+      stmt = db.prepare("INSERT OR REPLACE INTO metadata (key, value) VALUES (?, ?)")
+      values.each { |k, v| stmt.execute([k.to_s, v.to_s]) }
+      stmt.close
+    end
+
+    def read_provenance(db)
+      {
+        source: metadata_value(db, "langlinks_source"),
+        source_size: metadata_value(db, "langlinks_source_size").to_i,
+        imported_at: metadata_value(db, "langlinks_imported_at"),
+        imported_with: metadata_value(db, "langlinks_wp2txt_version"),
+        lang_filter: metadata_value(db, "langlinks_lang_filter"),
+        row_count: metadata_value(db, "langlinks_row_count").to_i,
+        skipped_invalid: metadata_value(db, "langlinks_skipped_invalid").to_i
+      }
+    end
+
+    # Post-import sanity check (§1.6): for each target language whose meta DB
+    # is installed locally, draw a random sample and report how many ll_title
+    # values exist in that DB's pages.title
+    def sanity_check(db, dump_name)
+      reports = []
+      main_date = dump_name[/\d{8}\z/]
+      db.execute("SELECT DISTINCT ll_lang FROM langlinks ORDER BY ll_lang").flatten.each do |lang|
+        candidates = MetadataIndex.cached_candidates(lang, cache_dir: @cache_dir)
+        next if candidates.empty? # target edition not installed locally: skip
+
+        pick = candidates.find { |c| c[:dump_name].to_s.end_with?(main_date.to_s) } || candidates.first
+        sample = db.execute(
+          "SELECT ll_title FROM langlinks WHERE ll_lang = ? ORDER BY RANDOM() LIMIT ?",
+          [lang, SANITY_SAMPLE_SIZE]
+        ).flatten
+        next if sample.empty?
+
+        other = SQLite3::Database.new(pick[:db_path], readonly: true)
+        begin
+          stmt = other.prepare("SELECT 1 FROM pages WHERE title = ? LIMIT 1")
+          matched = sample.count { |title| stmt.execute(title).any? }
+          stmt.close
+        ensure
+          other.close
+        end
+
+        rate = matched.to_f / sample.size
+        reports << { lang: lang, sampled: sample.size, matched: matched,
+                     match_rate: rate.round(3), against: pick[:dump_name],
+                     warning: rate < SANITY_WARN_THRESHOLD }
+      end
+      reports
+    end
+
+    # ------------------------------------------------------------------
+    # Streaming MySQL dump parser
+    # ------------------------------------------------------------------
+
+    # Yield [ll_from, ll_lang, ll_title] for every VALID tuple of every
+    # INSERT INTO `langlinks` statement, streaming (the dump is never
+    # loaded into memory whole). Handles .sql.gz and plain .sql.
+    # @return [Integer] number of tuples skipped for invalid UTF-8
+    #
+    # Tuple extraction is regex-based: a hand-rolled line[i] character-index
+    # loop is O(n²) on multibyte (UTF-8 code-range) lines, and real extended
+    # INSERT lines are MB-scale with multilingual titles — the regex engine
+    # scans at C speed and is O(n) regardless of encoding.
+    #
+    # Lines are read as BINARY: ll_title is VARBINARY in MySQL and real dumps
+    # contain historically corrupted bytes, so regex matching on UTF-8-tagged
+    # strings can raise "invalid byte sequence". The patterns are ASCII-only,
+    # so they run on byte strings without encoding checks; captures are then
+    # tagged UTF-8 and validated — a garbled title could never join
+    # pages.title anyway, so such rows are skipped (and counted), not scrubbed.
+    def each_source_row(source_path)
+      io = if source_path.end_with?(".gz")
+             # GzipReader ignores set_encoding; the encoding must be given
+             # at open time (lines must come out as BINARY — see below)
+             Zlib::GzipReader.open(source_path, encoding: Encoding::BINARY.to_s)
+           else
+             File.open(source_path, "rb")
+           end
+
+      skipped = 0
+      begin
+        io.each_line do |line|
+          next unless INSERT_PREFIX.match?(line)
+
+          line.scan(TUPLE_REGEX) do |ll_from, ll_lang, ll_title|
+            lang = unescape_mysql(ll_lang).force_encoding(Encoding::UTF_8)
+            title = unescape_mysql(ll_title).force_encoding(Encoding::UTF_8)
+            unless lang.valid_encoding? && title.valid_encoding?
+              skipped += 1
+              next
+            end
+
+            yield ll_from.to_i, lang, title
+          end
+        end
+      ensure
+        io.close
+      end
+      skipped
+    end
+
+    # One extended-INSERT tuple: (123,'lang','Title'). The string classes
+    # [^'\\]|\\. match any run of non-quote/non-backslash characters and
+    # backslash escape pairs, so escaped quotes (\') and backslashes (\\) —
+    # and commas/parens inside titles — do not terminate the capture.
+    # Tuples that do not match this shape are simply not extracted
+    # (equivalent to the old parser skipping malformed tuples)
+    TUPLE_REGEX = /\((\d+),'((?:[^'\\]|\\.)*)','((?:[^'\\]|\\.)*)'\)/
+
+    UNESCAPE_REGEX = /\\(.)/m
+
+    # Resolve MySQL backslash escapes in a captured string literal
+    # (same mapping as the old hand-rolled parser)
+    def unescape_mysql(str)
+      str.gsub(UNESCAPE_REGEX) { UNESCAPES[::Regexp.last_match(1)] || ::Regexp.last_match(1) }
+    end
+  end
+end