wp2txt 2.3.3 → 2.4.0

data/lib/wp2txt/link_counter.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+require "parallel"
+require "sqlite3"
+require "time"
+require_relative "metadata_index"
+require_relative "version"
+require_relative "text_processing"
+require_relative "wikitext_regions"
+
+module Wp2txt
+  # Counts, for every article, how many other articles link to it, and stores
+  # the result in the metadata index as page_inlinks(page_id, inlinks,
+  # via_redirects).
+  #
+  # Rules (recorded as RULE_VERSION so counts from different rules are never
+  # compared unknowingly):
+  # - sources are articles (namespace 0) that are not redirects
+  # - each source counts once per target, however many times it links there
+  # - a link to a redirect counts for the redirect's target (one hop);
+  #   via_redirects is how many sources reached the target only that way
+  # - only links written in the article's own wikitext count; links that
+  #   templates add when rendered (navigation boxes) are not in the dump text
+  # - commented-out links do not count
+  class LinkCounter
+    include Wp2txt
+
+    RULE_VERSION = "2"
+    STREAMS_PER_BATCH = 50
+    LINK_REGEX = /\[\[([^\[\]|<>{}\n]+)(?=\||\]\])/
+    COMMENT_REGEX = /<!--.*?-->/m
+
+    def initialize(multistream_path, stream_offsets, db_path:, num_processes: 4)
+      @multistream_path = multistream_path
+      @stream_offsets = stream_offsets
+      @db_path = db_path
+      @num_processes = num_processes
+    end
+
+    # @return [Hash] { articles:, with_inlinks:, rule_version: }
+    def count!(&progress)
+      titles, redirects = load_titles
+      # Inherited by the forked workers (copy-on-write); they only read it
+      @redirects = redirects
+
+      direct = Hash.new(0)
+      via = Hash.new(0)
+      pairs = @stream_offsets.zip(@stream_offsets[1..].to_a + [nil])
+      batches = pairs.each_slice(STREAMS_PER_BATCH).to_a
+      done = 0
+
+      Parallel.each(
+        batches,
+        in_processes: @num_processes,
+        finish: lambda { |_item, _idx, result|
+          result[:direct].each { |t, n| direct[t] += n }
+          result[:via].each { |t, n| via[t] += n }
+          done += 1
+          progress&.call(done, batches.size)
+        }
+      ) do |batch|
+        scan_batch(batch)
+      end
+
+      write(titles, direct, via)
+    end
+
+    private
+
+    # Articles (title => page_id) and redirects (title => target title)
+    def load_titles
+      db = SQLite3::Database.new(@db_path, readonly: true)
+      @case_rule = db.get_first_value("SELECT value FROM metadata WHERE key = 'case_rule'") || "first-letter"
+      titles = {}
+      redirects = {}
+      db.execute("SELECT page_id, title, redirect_to FROM pages WHERE namespace = 0") do |id, title, target|
+        if target
+          redirects[title] = normalize_target(target)
+        else
+          titles[title] = id
+        end
+      end
+      [titles, redirects]
+    ensure
+      db&.close
+    end
+
+    # Runs in a worker: returns per-target counts for this batch of streams
+    def scan_batch(offset_pairs)
+      direct = Hash.new(0)
+      via = Hash.new(0)
+      File.open(@multistream_path, "rb") do |f|
+        offset_pairs.each do |offset, next_offset|
+          f.seek(offset)
+          data = next_offset ? f.read(next_offset - offset) : f.read
+          xml = MetadataIndexBuilder.decompress_bz2(data)
+          xml.scan(MetadataIndexBuilder::PAGE_BLOCK_REGEX) do
+            count_page(::Regexp.last_match(1), direct, via)
+          end
+        end
+      end
+      { direct: direct, via: via }
+    end
+
+    def count_page(block, direct, via)
+      return unless Wp2txt.namespace_id(block[MetadataIndexBuilder::NS_REGEX, 1]).zero?
+
+      text = MetadataIndexBuilder.unescape_xml(block[MetadataIndexBuilder::TEXT_REGEX, 1] || "")
+      return if Wp2txt::REDIRECT_REGEX.match?(text)
+
+      reached = {} # target => true if reached directly at least once
+      WikitextRegions.remove_literal(text).scan(LINK_REGEX) do |(raw)|
+        name = normalize_target(raw)
+        next if name.empty?
+
+        if (target = @redirects[name])
+          reached[target] ||= false
+        else
+          reached[name] = true
+        end
+      end
+      reached.each do |target, directly|
+        direct[target] += 1
+        via[target] += 1 unless directly
+      end
+    end
+
+    def normalize_target(raw)
+      # Decode each reference in the original input, never references created
+      # by decoding another one (special_chr has two decoding stages).
+      decoded = raw.gsub(/&(?:#[xX][0-9a-fA-F]+|#\d+|[a-zA-Z][a-zA-Z0-9]*);/) { |entity| special_chr(entity) }
+      title = decoded.split("#", 2).first.to_s.strip.sub(/\A:/, "")
+      MetadataIndex.normalize_title(title, case_rule: @case_rule || "first-letter")
+    end
+
+    def write(titles, direct, via)
+      db = SQLite3::Database.new(@db_path)
+      db.busy_timeout = 5000
+      db.execute("DROP TABLE IF EXISTS page_inlinks")
+      db.execute("DELETE FROM metadata WHERE key LIKE 'links\\_%' ESCAPE '\\'")
+      db.execute(<<~SQL)
+        CREATE TABLE page_inlinks (
+          page_id       INTEGER PRIMARY KEY,
+          inlinks       INTEGER NOT NULL,
+          via_redirects INTEGER NOT NULL
+        )
+      SQL
+      with_inlinks = 0
+      db.transaction do
+        stmt = db.prepare("INSERT INTO page_inlinks (page_id, inlinks, via_redirects) VALUES (?, ?, ?)")
+        titles.each do |title, page_id|
+          n = direct[title]
+          with_inlinks += 1 if n.positive?
+          stmt.execute([page_id, n, via[title]])
+        end
+        stmt.close
+      end
+      db.execute("CREATE INDEX idx_page_inlinks_count ON page_inlinks(inlinks)")
+      {
+        links_counted_at: Time.now.utc.iso8601,
+        links_rule_version: RULE_VERSION,
+        links_wp2txt_version: Wp2txt::VERSION,
+        links_article_count: titles.size,
+        links_with_inlinks: with_inlinks
+      }.each { |k, v| db.execute("INSERT OR REPLACE INTO metadata (key, value) VALUES (?, ?)", [k.to_s, v.to_s]) }
+      { articles: titles.size, with_inlinks: with_inlinks, rule_version: RULE_VERSION }
+    ensure
+      db&.close
+    end
+  end
+end

data/lib/wp2txt/metadata_index.rb

@@ -39,9 +39,10 @@ module Wp2txt
 
     # Normalize a page title the way MediaWiki treats titles:
     # underscores to spaces, trimmed, first letter capitalized
-    def self.normalize_title(name)
+    def self.normalize_title(name, case_rule: "first-letter")
       n = name.to_s.tr("_", " ").strip.squeeze(" ")
       return n if n.empty?
+      return n if case_rule == "case-sensitive"
 
       n[0].upcase + n[1..].to_s
     end
@@ -161,6 +162,69 @@ module Wp2txt
         skipped_invalid: meta[:langlinks_skipped_invalid].to_i }
     end
 
+    # Provenance and counts of imported page properties, or nil
+    def page_props_provenance
+      return nil unless File.exist?(@db_path)
+
+      meta = read_metadata
+      return nil unless meta && meta[:page_props_imported_at] && page_properties_imported?
+
+      { source: meta[:page_props_source],
+        source_size: meta[:page_props_source_size].to_i,
+        source_sha256: meta[:page_props_source_sha256],
+        imported_at: meta[:page_props_imported_at],
+        imported_with: meta[:page_props_wp2txt_version],
+        page_count: meta[:page_props_page_count].to_i,
+        qid_count: meta[:page_props_qid_count].to_i,
+        disambiguation_count: meta[:page_props_disambiguation_count].to_i,
+        sort_key_count: meta[:page_props_sort_key_count].to_i,
+        skipped_invalid_sort_keys: meta[:page_props_skipped_invalid_sort_keys].to_i }
+    end
+
+    # How incoming links were counted (wp2txt --count-links), or nil
+    def links_provenance
+      return nil unless File.exist?(@db_path)
+
+      meta = read_metadata
+      return nil unless meta && meta[:links_counted_at]
+
+      { counted_at: meta[:links_counted_at],
+        rule_version: meta[:links_rule_version],
+        counted_with: meta[:links_wp2txt_version],
+        article_count: meta[:links_article_count].to_i,
+        with_inlinks: meta[:links_with_inlinks].to_i }
+    end
+
+    # An incomplete import must not be reported as an absent property.
+    def self.page_properties_imported?(db)
+      !!db.get_first_value(<<~SQL)
+        SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'page_properties'
+        AND EXISTS (SELECT 1 FROM metadata WHERE key = 'page_props_imported_at')
+      SQL
+    end
+
+    def page_properties_imported?
+      File.exist?(@db_path) && self.class.page_properties_imported?(open_db)
+    rescue SQLite3::Exception
+      false
+    end
+
+    def self.property_values(row = nil)
+      qid, sort_key, disambiguation = row
+      { qid: qid, sort_key: sort_key, disambiguation: disambiguation == 1 }
+    end
+
+    # nil means not imported; null/false values mean imported but absent.
+    def properties_for(page_id)
+      return nil unless page_properties_imported?
+
+      self.class.property_values(open_db.get_first_row(
+        "SELECT qid, sort_key, disambiguation FROM page_properties WHERE page_id = ?", [page_id]
+      ))
+    rescue SQLite3::Exception
+      nil
+    end
+
     def close
       @db&.close
       @db = nil
@@ -230,7 +294,7 @@ module Wp2txt
       end
     end
 
-    def finalize_build!(source_path)
+    def finalize_build!(source_path, case_rule: "first-letter")
       db = open_db
       db.execute("CREATE INDEX IF NOT EXISTS idx_pages_title ON pages(title)")
       db.execute("CREATE INDEX IF NOT EXISTS idx_pc_category ON page_categories(category)")
@@ -248,6 +312,7 @@ module Wp2txt
         source_size: stat.size,
         source_mtime: stat.mtime.to_i,
         dump_name: dump_name,
+        case_rule: case_rule,
         built_at: Time.now.utc.iso8601
       )
       db.execute("ANALYZE")
@@ -629,6 +694,7 @@ module Wp2txt
     # Build the index. Yields (batches_done, batches_total) after each batch.
     # @return [MetadataIndex] the built index
     def build(&progress)
+      case_rule = case_rule_from_header
       index = MetadataIndex.new(@db_path)
       index.prepare_build!
       # Close before Parallel forks workers so children do not inherit a
@@ -652,10 +718,20 @@ module Wp2txt
         self.class.scan_batch(@multistream_path, batch)
       end
 
-      index.finalize_build!(@multistream_path)
+      index.finalize_build!(@multistream_path, case_rule: case_rule)
       index
     end
 
+    # The siteinfo may occupy its own stream before the first page offset.
+    def case_rule_from_header
+      return "first-letter" if @stream_offsets.empty?
+
+      header_end = @stream_offsets.first.positive? ? @stream_offsets.first : @stream_offsets[1]
+      xml = self.class.decompress_bz2(File.binread(@multistream_path, header_end))
+      siteinfo = xml[%r{<siteinfo\b[^>]*>(.*?)</siteinfo>}m, 1].to_s
+      siteinfo[%r{<case>\s*(first-letter|case-sensitive)\s*</case>}, 1] || "first-letter"
+    end
+
     # Scan a batch of [offset, next_offset] stream pairs.
     # Runs inside worker processes: must not touch SQLite.
     def self.scan_batch(multistream_path, offset_pairs)

data/lib/wp2txt/multistream.rb

@@ -95,6 +95,11 @@ module Wp2txt
       @early_terminated == true
     end
 
+    # Where the stream holding the last found article ends. Scanning stops as
+    # soon as every target is found, so without this the reader cannot tell how
+    # far that stream extends and would read the dump to its end.
+    attr_reader :stream_end_offset
+
     def find_by_title(title)
       @entries_by_title[title]
     end
@@ -179,6 +184,14 @@ module Wp2txt
       end
     end
 
+    def next_stream_offset_after(io, offset)
+      io.each_line do |line|
+        next_offset = line.split(":", 2).first.to_i
+        return next_offset if next_offset > offset
+      end
+      nil
+    end
+
     def parse_index_stream(io)
       count = 0
       io.each_line do |line|
@@ -205,6 +218,7 @@ module Wp2txt
           @found_targets << title if @target_titles.include?(title)
           if @found_targets.size == @target_titles.size
             @early_terminated = true
+            @stream_end_offset = next_stream_offset_after(io, offset)
             print "\r  Found all #{@target_titles.size} target articles" if @show_progress
             puts if @show_progress
             break
@@ -223,6 +237,10 @@ module Wp2txt
 
   # Reads articles from multistream bz2 files
   class MultistreamReader
+    # A single multistream bz2 stream holds about 100 pages; anything this large
+    # past the last known offset is several streams, not one.
+    MAX_TAIL_STREAM_BYTES = 64 * 1024 * 1024
+
     attr_reader :multistream_path, :index
 
     # Initialize reader with multistream file and index
@@ -357,7 +375,14 @@ module Wp2txt
         if next_offset
           compressed_data = f.read(next_offset - offset)
         else
-          # Last stream - read to end
+          # Last stream of the dump: the rest of the file is that one stream.
+          # A large remainder means the end was never recorded, and reading it
+          # in one call fails outright on some platforms (EINVAL on macOS).
+          remaining = File.size(@multistream_path) - offset
+          if remaining > MAX_TAIL_STREAM_BYTES
+            raise Wp2txt::Error, "cannot locate the end of the stream at offset #{offset} " \
+                                 "(#{remaining} bytes to end of file); the index may be incomplete"
+          end
           compressed_data = f.read
         end
 
@@ -372,7 +397,9 @@ module Wp2txt
       # here would silently read gigabytes to EOF, so fail fast instead
       raise "Stream offset #{current_offset} not found in index (#{offsets.size} streams known)" unless idx
 
-      offsets[idx + 1]
+      return offsets[idx + 1] if idx + 1 < offsets.size
+
+      @index.respond_to?(:stream_end_offset) ? @index.stream_end_offset : nil
     end
 
     def decompress_bz2(data)
@@ -394,6 +421,7 @@ module Wp2txt
           return {
             title: page_title,
             id: page_node.at_xpath("id")&.text&.to_i,
+            revision_id: page_node.at_xpath("revision/id")&.text&.to_i,
             text: page_node.at_xpath(".//text")&.text || ""
           }
         end
@@ -409,6 +437,7 @@ module Wp2txt
         page = {
           title: page_node.at_xpath("title")&.text,
           id: page_node.at_xpath("id")&.text&.to_i,
+          revision_id: page_node.at_xpath("revision/id")&.text&.to_i,
           text: page_node.at_xpath(".//text")&.text || ""
         }
         yield page if page[:title]
@@ -837,6 +866,27 @@ module Wp2txt
       File.join(@cache_dir, "#{@lang}wiki-#{date}-langlinks.sql.gz")
     end
 
+    def page_props_url(date)
+      wiki = "#{@lang}wiki"
+      "#{DUMP_BASE_URL}/#{wiki}/#{date}/#{wiki}-#{date}-page_props.sql.gz"
+    end
+
+    def cached_page_props_path(date)
+      File.join(@cache_dir, "#{@lang}wiki-#{date}-page_props.sql.gz")
+    end
+
+    # Download the page_props dump for an explicit dump date (must match the index)
+    def download_page_props(date:, force: false)
+      path = cached_page_props_path(date)
+      return path if File.exist?(path) && !force
+
+      url = page_props_url(date)
+      puts "Downloading page_props: #{url}"
+      $stdout.flush
+      download_file(url, path)
+      path
+    end
+
     # Download the langlinks dump for an explicit dump date. The date MUST
     # come from the built metadata index (not latest_dump_date) so the
     # imported links stay pinned to the indexed dump version.

data/lib/wp2txt/output_writer.rb

@@ -100,6 +100,14 @@ module Wp2txt
       raise Wp2txt::FileIOError, "Write failed: #{e.message}"
     end
 
+    # Push buffered output to the file. Call before forking: a child process
+    # inherits the buffer and writes it out again when it exits.
+    def flush
+      @mutex.synchronize do
+        @current_file.flush if @current_file && !@current_file.closed?
+      end
+    end
+
     # Close current file and finalize
     def close
       @mutex.synchronize do

data/lib/wp2txt/page_props_importer.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+require "sqlite3"
+require "time"
+require "digest"
+require_relative "sql_dump_reader"
+require_relative "version"
+
+module Wp2txt
+  # Imports Wikidata IDs, disambiguation flags, and default sort keys
+  # from the official page_props dump into the metadata index, as
+  # page_properties. Like langlinks, the dump must carry the same
+  # date as the index: IDs are only meaningful for the pages they came with.
+  class PagePropsImporter
+    BATCH_SIZE = 50_000
+
+    # (pp_page,'pp_propname','pp_value',pp_sortkey) — string classes accept
+    # escaped quotes and backslashes; sortkey is NULL or a number
+    TUPLE_REGEX = /\((\d+),'((?:[^'\\]|\\.)*)','((?:[^'\\]|\\.)*)',(?:NULL|[-+.\deE]+)\)/
+    QID_REGEX = /\AQ\d+\z/
+
+    def initialize(db_path)
+      @db_path = db_path
+    end
+
+    # "jawiki-20260901-page_props.sql.gz" => "jawiki-20260901"
+    def self.dump_name_of(path)
+      File.basename(path)[/\A[a-z0-9_\-]+?-\d{8}/]
+    end
+
+    # @return [Hash] { status: :imported | :already_imported, row_count:, provenance: }
+    def import!(source_path, force: false)
+      raise ArgumentError, "page_props file not found: #{source_path}" unless File.exist?(source_path)
+
+      db = SQLite3::Database.new(@db_path)
+      db.busy_timeout = 5000
+      dump_name = metadata_value(db, "dump_name")
+      raise ArgumentError, "metadata index is not built: #{@db_path}" unless dump_name
+
+      source_dump = self.class.dump_name_of(source_path)
+      unless source_dump && source_dump == dump_name
+        raise ArgumentError,
+              "dump version mismatch: the metadata index is #{dump_name} but the page_props file is " \
+              "#{source_dump || File.basename(source_path)} (versions must match; there is no override)"
+      end
+
+      if !force && (existing = imported_at(db))
+        return { status: :already_imported, imported_at: existing,
+                 row_count: db.get_first_value("SELECT COUNT(*) FROM page_properties").to_i, provenance: read_provenance(db) }
+      end
+
+      db.execute("DROP TABLE IF EXISTS page_qids") # replace the unreleased QID-only schema
+      db.execute("DROP TABLE IF EXISTS page_properties")
+      # A failed load must leave the index looking "not imported"
+      db.execute("DELETE FROM metadata WHERE key LIKE 'page\\_props\\_%' ESCAPE '\\'")
+      db.execute(<<~SQL)
+        CREATE TABLE page_properties (
+          page_id INTEGER PRIMARY KEY,
+          qid TEXT,
+          disambiguation INTEGER NOT NULL DEFAULT 0,
+          sort_key TEXT
+        )
+      SQL
+
+      rows_seen = 0
+      skipped_invalid = 0
+      batch = []
+      flush = lambda do
+        db.transaction do
+          stmt = db.prepare(<<~SQL)
+            INSERT INTO page_properties (page_id, qid, disambiguation, sort_key) VALUES (?, ?, ?, ?)
+            ON CONFLICT(page_id) DO UPDATE SET
+              qid = COALESCE(excluded.qid, page_properties.qid),
+              disambiguation = MAX(excluded.disambiguation, page_properties.disambiguation),
+              sort_key = COALESCE(excluded.sort_key, page_properties.sort_key)
+          SQL
+          batch.each { |row| stmt.execute(row) }
+          stmt.close
+        end
+        batch.clear
+      end
+
+      SqlDumpReader.each_insert_line(source_path, "page_props") do |line|
+        line.scan(TUPLE_REGEX) do |page, name, value|
+          rows_seen += 1
+          row = [page.to_i, nil, 0, nil]
+          case name
+          when "wikibase_item"
+            qid = SqlDumpReader.unescape(value)
+            next unless QID_REGEX.match?(qid)
+
+            row[1] = qid.force_encoding(Encoding::UTF_8)
+          when "disambiguation"
+            row[2] = 1
+          when "defaultsort"
+            key = SqlDumpReader.unescape(value).force_encoding(Encoding::UTF_8)
+            unless key.valid_encoding?
+              skipped_invalid += 1
+              next
+            end
+            row[3] = key
+          else
+            next
+          end
+          batch << row
+          flush.call if batch.size >= BATCH_SIZE
+        end
+      end
+      flush.call unless batch.empty?
+
+      if rows_seen.zero?
+        raise Wp2txt::Error, "no page_props rows found in #{File.basename(source_path)}; " \
+                             "the file may be empty or in an unrecognized format"
+      end
+
+      counts = db.get_first_row("SELECT COUNT(*), COUNT(qid), SUM(disambiguation), COUNT(sort_key) FROM page_properties")
+      row_count = counts[0]
+      stamp_provenance(db, source_path, counts, skipped_invalid)
+      { status: :imported, row_count: row_count, provenance: read_provenance(db) }
+    ensure
+      db&.close
+    end
+
+    private
+
+    def metadata_value(db, key)
+      db.get_first_value("SELECT value FROM metadata WHERE key = ?", [key])
+    rescue SQLite3::Exception
+      nil
+    end
+
+    def imported_at(db)
+      table = db.get_first_value("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'page_properties'")
+      table && metadata_value(db, "page_props_imported_at")
+    end
+
+    def stamp_provenance(db, source_path, counts, skipped_invalid)
+      values = {
+        page_props_source: File.basename(source_path),
+        page_props_source_size: File.size(source_path),
+        page_props_source_sha256: Digest::SHA256.file(source_path).hexdigest,
+        page_props_imported_at: Time.now.utc.iso8601,
+        page_props_wp2txt_version: Wp2txt::VERSION,
+        page_props_page_count: counts[0],
+        page_props_qid_count: counts[1],
+        page_props_disambiguation_count: counts[2].to_i,
+        page_props_sort_key_count: counts[3],
+        page_props_skipped_invalid_sort_keys: skipped_invalid
+      }
+      stmt = db.prepare("INSERT OR REPLACE INTO metadata (key, value) VALUES (?, ?)")
+      values.each { |k, v| stmt.execute([k.to_s, v.to_s]) }
+      stmt.close
+    end
+
+    def read_provenance(db)
+      {
+        source: metadata_value(db, "page_props_source"),
+        source_size: metadata_value(db, "page_props_source_size").to_i,
+        source_sha256: metadata_value(db, "page_props_source_sha256"),
+        imported_at: metadata_value(db, "page_props_imported_at"),
+        imported_with: metadata_value(db, "page_props_wp2txt_version"),
+        page_count: metadata_value(db, "page_props_page_count").to_i,
+        qid_count: metadata_value(db, "page_props_qid_count").to_i,
+        disambiguation_count: metadata_value(db, "page_props_disambiguation_count").to_i,
+        sort_key_count: metadata_value(db, "page_props_sort_key_count").to_i,
+        skipped_invalid_sort_keys: metadata_value(db, "page_props_skipped_invalid_sort_keys").to_i
+      }
+    end
+  end
+end

data/lib/wp2txt/sql_dump_reader.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require "zlib"
+
+module Wp2txt
+  # Streams the rows of one table's INSERT statements out of an official
+  # MySQL table dump (.sql or .sql.gz), without loading the file whole.
+  #
+  # Dumps write each INSERT either on one line or with the header on its own
+  # line and one tuple per following line (current dumps do the latter), so
+  # every line of a statement is read up to its terminating semicolon.
+  # Statements for other tables are skipped.
+  #
+  # Lines are read as BINARY: string columns are VARBINARY and real dumps
+  # contain historically corrupted bytes. Callers match tuples with ASCII-only
+  # patterns and validate the captures themselves.
+  module SqlDumpReader
+    STATEMENT_START = /\A\s*INSERT\s+INTO\s/i
+
+    module_function
+
+    # @yield [String] each line (BINARY) that belongs to an INSERT statement for table
+    def each_insert_line(source_path, table)
+      header = /\A\s*INSERT\s+INTO\s+`#{Regexp.escape(table)}`\s+VALUES\b/i
+      io = if source_path.end_with?(".gz")
+             # GzipReader ignores set_encoding; the encoding must be given at open time
+             Zlib::GzipReader.open(source_path, encoding: Encoding::BINARY.to_s)
+           else
+             File.open(source_path, "rb")
+           end
+      inside = false
+      begin
+        io.each_line do |line|
+          inside = header.match?(line) if STATEMENT_START.match?(line)
+          next unless inside
+
+          inside = false if line.rstrip.end_with?(";")
+          yield line
+        end
+      ensure
+        io.close
+      end
+    end
+
+    UNESCAPE_REGEX = /\\(.)/m
+    # MySQL backslash escapes inside mysqldump string literals
+    UNESCAPES = {
+      "0" => "\0", "'" => "'", '"' => '"', "b" => "\b", "n" => "\n",
+      "r" => "\r", "t" => "\t", "Z" => "\x1A", "\\" => "\\"
+    }.freeze
+
+    # Undo MySQL string escaping (\' \\ \n ...) in a captured value
+    def unescape(value)
+      value.gsub(UNESCAPE_REGEX) { UNESCAPES.fetch(Regexp.last_match(1), Regexp.last_match(1)) }
+    end
+  end
+end