wp2txt 2.3.4 → 2.4.0

data/spec/lead_terms_edge_cases_spec.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'wp2txt'
+require 'wp2txt/lead_terms'
+require 'wp2txt/link_counter'
+require 'wp2txt/metadata_index'
+require 'open3'
+require 'tmpdir'
+require_relative 'support/multistream_fixture'
+
+RSpec.describe 'lead terms and link counting on unusual wikitext' do
+  include MultistreamFixture
+
+  let(:cleaner) { Object.new.extend(Wp2txt) }
+  let(:render) { ->(s) { cleaner.format_wiki(s, expand_templates: true, markers: [:all]) } }
+  def terms(text)
+    Wp2txt::LeadTerms.extract(text, render: render)
+  end
+
+  it '#1 preserves link display and reading with expansion enabled or disabled' do
+    {
+      '{{読み仮名|[[東京|東京都]]|とうきょう}}' => '東京都（とうきょう）',
+      '{{仮リンク|[[東京|東京都]]|en|Tokyo}}' => '東京都',
+      '{{読み仮名|{{lang|ja|[[東京|東京都]]}}|とうきょう}}' => '東京都（とうきょう）',
+      '{{読み仮名|東京|}}' => '東京'
+    }.each do |input, expected|
+      [true, false].each do |expand|
+        expect(cleaner.format_wiki(input, expand_templates: expand)).to eq(expected)
+      end
+    end
+  end
+
+  def build_case_dump(dir, rule)
+    alias_title = rule == 'first-letter' ? 'Alias' : 'alias'
+    pages = [[1, 'apple', ''], [2, 'Apple', ''], [3, alias_title, '#REDIRECT [[apple]]'],
+             [4, 'Direct', '[[apple]]'], [5, 'Via', '[[alias]]'], [6, '東京', '']]
+    xml = "<mediawiki><siteinfo><case>#{rule}</case></siteinfo>" + pages.map do |id, title, text|
+      page_xml(id: id, ns: 0, title: title, text: text)
+    end.join + '</mediawiki>'
+    dump = File.join(dir, 'testwiki-20260101-multistream.xml.bz2')
+    File.binwrite(dump, bzip2(xml))
+    db = File.join(dir, 'meta.sqlite3')
+    Wp2txt::MetadataIndexBuilder.new(dump, [0], db_path: db, num_processes: 0).build.close
+    [dump, db]
+  end
+
+  it '#2 stores siteinfo case and preserves direct and redirected case-sensitive targets' do
+    Dir.mktmpdir do |dir|
+      dump, path = build_case_dump(dir, 'case-sensitive')
+      db = SQLite3::Database.new(path)
+      expect(db.get_first_value("SELECT value FROM metadata WHERE key = 'case_rule'")).to eq('case-sensitive')
+      db.close
+      Wp2txt::LinkCounter.new(dump, [0], db_path: path, num_processes: 0).count!
+      db = SQLite3::Database.new(path)
+      expect(db.execute('SELECT page_id,inlinks,via_redirects FROM page_inlinks WHERE page_id IN (1,2) ORDER BY page_id'))
+        .to eq([[1, 2, 1], [2, 0, 0]])
+      db.close
+      expect(Wp2txt::MetadataIndex.normalize_title('apple')).to eq('Apple')
+    end
+  end
+
+  it '#2 uses first-letter for explicit siteinfo and legacy metadata without the rule' do
+    Dir.mktmpdir do |dir|
+      dump, path = build_case_dump(dir, 'first-letter')
+      [false, true].each do |legacy|
+        db = SQLite3::Database.new(path)
+        if legacy
+          db.execute("DELETE FROM metadata WHERE key = 'case_rule'")
+        else
+          expect(db.get_first_value("SELECT value FROM metadata WHERE key = 'case_rule'")).to eq('first-letter')
+        end
+        db.close
+        Wp2txt::LinkCounter.new(dump, [0], db_path: path, num_processes: 0).count!
+        db = SQLite3::Database.new(path)
+        expect(db.get_first_value('SELECT inlinks FROM page_inlinks WHERE page_id=1')).to eq(0)
+        expect(db.execute('SELECT inlinks,via_redirects FROM page_inlinks WHERE page_id=2')).to eq([[2, 1]])
+        db.close
+      end
+    end
+  end
+
+  it '#2 reads a separate siteinfo stream before the first indexed page' do
+    Dir.mktmpdir do |dir|
+      header = bzip2('<mediawiki><siteinfo><case>case-sensitive</case></siteinfo>')
+      dump = File.join(dir, 'dump.xml.bz2')
+      File.binwrite(dump, header + bzip2(page_xml(id: 1, ns: 0, title: 'apple', text: '') + '</mediawiki>'))
+      path = File.join(dir, 'meta.sqlite3')
+      Wp2txt::MetadataIndexBuilder.new(dump, [header.bytesize], db_path: path, num_processes: 0).build.close
+      db = SQLite3::Database.new(path)
+      expect(db.get_first_value("SELECT value FROM metadata WHERE key = 'case_rule'")).to eq('case-sensitive')
+      db.close
+    end
+  end
+
+  %w[nowiki pre math syntaxhighlight source gallery score timeline chem ce graph mapframe templatedata].each do |tag|
+    it "#3 skips #{tag} contents and self-closing forms while preserving source offsets" do
+      text = "<#{tag.upcase} data-x='>'>'''偽'''（にせ）{{ruby|偽|にせ}}</#{tag.upcase}>\n\n" \
+             "<#{tag} />'''東京'''（とうきょう）。"
+      result = terms(text)
+      expect(result.map { |t| t['text'] }).to eq(['東京'])
+      s, e = result.first['span']['bold']
+      expect(text[s...e]).to eq("'''東京'''")
+      expect(terms("<#{tag}>'''偽'''" )).to eq([])
+    end
+  end
+
+  it '#4 ignores delimiters in comments and excluded tags inside parentheses' do
+    text = "'''東京'''（とうきょう<!-- ) , -->、<nowiki>),</nowiki>Tokyo）は都市。"
+    result = terms(text).first
+    s, e = result['span']['paren']
+    expect(text[s...e]).to eq('（とうきょう<!-- ) , -->、<nowiki>),</nowiki>Tokyo）')
+    expect(result['notes'].size).to eq(2)
+    expect(result['notes'].first).to eq('とうきょう')
+    expect(result['notes_text']).not_to include('<!--')
+  end
+
+  it 'retains the original text of ruby arguments containing excluded regions' do
+    expect(terms('{{ruby|<nowiki>東京</nowiki>|とうきょう}}').first.slice('text', 'reading'))
+      .to eq('text' => '東京', 'reading' => 'とうきょう')
+  end
+
+  it '#5 detects only headings outside skipped regions including trailing comments' do
+    ["<!--\n== 偽 ==\n-->", "<nowiki>\n== 偽 ==\n</nowiki>", "{{box|\n== 偽 ==\n}}", "{|\n== 偽 ==\n|}"].each do |prefix|
+      expect(terms("#{prefix}\n'''東京'''（とうきょう）").map { |t| t['text'] }).to eq(['東京'])
+    end
+    expect(terms("導入文。\n\n== 歴史 == <!-- comment -->\n'''後の語'''（あと）。")).to eq([])
+  end
+
+  it '#6 allows single newlines before and inside notes but stops at a blank line' do
+    expect(terms("'''東京'''\n（とうきょう）は都市。").first['notes']).to eq(['とうきょう'])
+    text = "'''東京'''（とうきょう、\nTokyo）は都市。"
+    term = terms(text).first
+    expect(term['notes']).to eq(['とうきょう', 'Tokyo'])
+    s, e = term['span']['paren']
+    expect(text[s...e]).to eq("（とうきょう、\nTokyo）")
+    ["'''東京'''\n\n（とうきょう）", "'''東京'''（とうきょう、\n \nTokyo）"].each do |input|
+      expect(terms(input).first['notes']).to eq([])
+    end
+  end
+
+  it '#7 stops at empty lines containing whitespace' do
+    ["\n \n", "\n\t\n", "\r\n \r\n"].each do |gap|
+      expect(terms("'''東京'''（とうきょう）。#{gap}'''別段落'''（べつだんらく）。").map { |t| t['text'] }).to eq(['東京'])
+    end
+  end
+
+  def counts(text)
+    counter = Wp2txt::LinkCounter.new('', [], db_path: '')
+    counter.instance_variable_set(:@redirects, {})
+    direct, via = Hash.new(0), Hash.new(0)
+    counter.send(:count_page, page_xml(id: 1, ns: 0, title: 'Source', text: text), direct, via)
+    direct
+  end
+
+  it '#8 decodes title entities before sections and normalizes whitespace before colon' do
+    expect(counts('[[M&amp;A]] [[Caf&#233;]] [[ :東京#節|表示]] [[Caf&#xE9;#section]] [[#節]] [[]]'))
+      .to eq('M&A' => 1, 'Café' => 1, '東京' => 1)
+    expect(counts('[[東京&#35;節]] [[東京|表示]]')).to eq('東京' => 1)
+  end
+
+  it '#9 excludes every non-wikitext tag and comments from incoming links' do
+    %w[nowiki pre math syntaxhighlight source score chem ce graph mapframe templatedata].each do |tag|
+      expect(counts("<#{tag}>[[東京]]</#{tag}>[[大阪]]<#{tag}/><!-- [[京都]] -->")).to eq('大阪' => 1)
+    end
+  end
+
+  it 'handles empty text and binary invalid bytes in the shared region helper' do
+    expect(terms('')).to eq([])
+    expect(counts('')).to eq({})
+    expect(Wp2txt::WikitextRegions.remove_literal("<nowiki>\xFF</nowiki>\xFE".b)).to eq("\xFE".b)
+  end
+
+  def cli(*options)
+    Dir.mktmpdir do |dir|
+      input = File.join(dir, 'x.xml.bz2')
+      File.binwrite(input, bzip2("<mediawiki>#{page_xml(id: 1, ns: 0, title: '東京', text: "'''東京'''。")}</mediawiki>"))
+      Open3.capture3(RbConfig.ruby, '-I', File.expand_path('../lib', __dir__),
+                     File.expand_path('../bin/wp2txt', __dir__), '-i', input, '-o', dir, *options)
+    end
+  end
+
+  it '#10 rejects lead terms with ractor' do
+    _, err, status = cli('--format', 'json', '--lead-terms', '--ractor', '--no-turbo')
+    expect(status.success?).to be(false)
+    expect(err).to include('--lead-terms cannot be combined with --ractor')
+  end
+
+  it '#10 warns about missing source identifiers in ractor JSON without rejecting it' do
+    _, err, status = cli('--format', 'json', '--ractor', '--no-turbo', '-n', '1')
+    expect(status.success?).to be(true), err
+    expect(err).to include('does not include page IDs or revision IDs')
+  end
+end

data/spec/lead_terms_links_qids_spec.rb

@@ -0,0 +1,204 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+require "json"
+require "open3"
+require "tmpdir"
+require "zlib"
+require_relative "support/multistream_fixture"
+require "wp2txt"
+require "wp2txt/metadata_index"
+require "wp2txt/multistream"
+require "wp2txt/sql_dump_reader"
+require "wp2txt/page_props_importer"
+require "wp2txt/link_counter"
+require "wp2txt/lead_terms"
+
+RSpec.describe "lead terms, incoming links, and Wikidata IDs" do
+  include MultistreamFixture
+
+  around do |example|
+    Dir.mktmpdir("wp2txt-ltq-") do |dir|
+      @dir = dir
+      example.run
+    end
+  end
+
+  def xml_page(id:, title:, text:, ns: 0)
+    esc = ->(s) { s.gsub("&", "&amp;").gsub("<", "&lt;").gsub(">", "&gt;") }
+    "<page>\n<title>#{esc.(title)}</title>\n<ns>#{ns}</ns>\n<id>#{id}</id>\n<revision>\n<id>#{id * 100}</id>\n" \
+      "<text bytes=\"#{text.bytesize}\">#{esc.(text)}</text>\n</revision>\n</page>\n"
+  end
+
+  PAGES = [
+    [1, "東京", "'''東京'''（とうきょう、Tokyo）は首都。\n\n== 歴史 ==\n'''後'''（あと）", 0],
+    [2, "東京都区部", "#REDIRECT [[東京]]", 0],
+    [3, "記事A", "[[東京]]と[[東京]]、[[東京都区部]]。[[M&A|買収]]。<!-- [[隠れ]] -->[[#節]][[]]", 0],
+    [4, "記事B", "[[東京#歴史|東京の歴史]]", 0],
+    [5, "記事C", "[[東京都区部]]のみ。", 0],
+    [6, "M&A", "'''{{読み仮名|合併|がっぺい}}'''と買収。", 0],
+    [7, "隠れ", "", 0],
+    [8, "Category:X", "[[東京]]", 14]
+  ].freeze
+
+  # One-stream dump with its index and a built metadata index
+  def build_dump
+    dump = File.join(@dir, "testwiki-20260101-pages-articles-multistream.xml.bz2")
+    File.binwrite(dump, bzip2(PAGES.map { |id, t, x, ns| xml_page(id: id, title: t, text: x, ns: ns) }.join))
+    File.write(File.join(@dir, "testwiki-20260101-pages-articles-multistream-index.txt"),
+               PAGES.map { |id, t, _, _| "0:#{id}:#{t}" }.join("\n") + "\n")
+    db = Wp2txt::MetadataIndex.path_for(dump, cache_dir: @dir)
+    Wp2txt::MetadataIndexBuilder.new(dump, [0], db_path: db, num_processes: 0).build.close
+    [dump, db]
+  end
+
+  def page_props_file(content, name: "testwiki-20260101-page_props.sql.gz")
+    path = File.join(@dir, name)
+    Zlib::GzipWriter.open(path) { |gz| gz.write(content) }
+    path
+  end
+
+  PAGE_PROPS_SQL = <<~SQL
+    INSERT INTO `page_props` VALUES
+    (1,'wikibase_item','Q1490',NULL),
+    (1,'page_image_free','T\\'kyo.jpg',NULL),
+    (3,'wikibase_item','Q9',1.5),
+    (6,'wikibase_item','Q1\\'bad',NULL),
+    (6,'defaultsort','\xFF\xFE',NULL);
+    INSERT INTO `page_restrictions` VALUES
+    (1,'wikibase_item','Q777',NULL);
+  SQL
+
+  describe Wp2txt::SqlDumpReader do
+    it "reads statements written on one line and one tuple per line, skipping other tables" do
+      lines = []
+      path = page_props_file("INSERT INTO `t` VALUES (1),(2);\nINSERT INTO `u` VALUES\n(9);\nINSERT INTO `t` VALUES\n(3),\n(4);\n")
+      described_class.each_insert_line(path, "t") { |l| lines << l.strip }
+      expect(lines).to eq(["INSERT INTO `t` VALUES (1),(2);", "INSERT INTO `t` VALUES", "(3),", "(4);"])
+    end
+  end
+
+  describe Wp2txt::PagePropsImporter do
+    it "imports only well-formed wikibase_item values, and records where they came from" do
+      _dump, db = build_dump
+      path = page_props_file(PAGE_PROPS_SQL.b)
+      result = described_class.new(db).import!(path)
+      expect(result[:row_count]).to eq(2)
+      rows = SQLite3::Database.new(db, readonly: true).execute("SELECT page_id, qid FROM page_properties ORDER BY page_id")
+      expect(rows).to eq([[1, "Q1490"], [3, "Q9"]])
+      expect(result[:provenance][:source_sha256]).to eq(Digest::SHA256.file(path).hexdigest)
+      expect(described_class.new(db).import!(path)[:status]).to eq(:already_imported)
+    end
+
+    it "refuses a dump of another date" do
+      _dump, db = build_dump
+      path = page_props_file(PAGE_PROPS_SQL.b, name: "testwiki-20250101-page_props.sql.gz")
+      expect { described_class.new(db).import!(path) }.to raise_error(ArgumentError, /version mismatch/)
+    end
+
+    it "fails when nothing could be read" do
+      _dump, db = build_dump
+      path = page_props_file("-- empty\n")
+      expect { described_class.new(db).import!(path) }.to raise_error(Wp2txt::Error, /no page_props rows/)
+    end
+  end
+
+  describe Wp2txt::LinkCounter do
+    it "counts each linking article once, through redirects, ignoring comments and non-articles" do
+      dump, db = build_dump
+      described_class.new(dump, [0], db_path: db, num_processes: 0).count!
+      counts = SQLite3::Database.new(db, readonly: true)
+                                .execute("SELECT p.title, i.inlinks, i.via_redirects FROM page_inlinks i " \
+                                         "JOIN pages p USING (page_id)").to_h { |t, n, v| [t, [n, v]] }
+      expect(counts["東京"]).to eq([3, 1]) # 記事A and 記事B directly, 記事C only via the redirect
+      expect(counts["M&A"]).to eq([1, 0])
+      expect(counts["隠れ"]).to eq([0, 0]) # linked only from a comment
+      expect(counts["記事A"]).to eq([0, 0])
+      expect(counts).not_to have_key("東京都区部") # redirects get no row of their own
+    end
+  end
+
+  describe Wp2txt::LeadTerms do
+    let(:render) { ->(fragment) { Object.new.extend(Wp2txt).format_wiki(fragment, { expand_templates: true, markers: [:all] }) } }
+
+    def terms(text)
+      Wp2txt::LeadTerms.extract(text, render: render)
+    end
+
+    it "takes bold terms of the first paragraph that has one, skipping templates, files, refs, and comments" do
+      text = "{{Otheruses|'''x'''}}\n[[ファイル:A.jpg|thumb|'''偽''']]<!-- '''隠''' --><ref>'''注'''</ref>\n" \
+             "'''日本語'''（にほんご、にっぽんご{{Refnest|注}}）は言語。'''和語'''とも。\n\n次の段落の'''別'''。"
+      expect(terms(text).map { |t| [t["text"], t["notes"]] }).to eq([["日本語", %w[にほんご にっぽんご]], ["和語", []]])
+    end
+
+    it "reports spans that cut the bold and the parentheses out of the original text" do
+      text = "前置き。'''東京'''　（とうきょう）は首都。"
+      term = terms(text).first
+      expect(text[Range.new(*term["span"]["bold"], true)]).to eq("'''東京'''")
+      expect(text[Range.new(*term["span"]["paren"], true)]).to eq("（とうきょう）")
+      expect(term["notes_text"]).to eq("とうきょう")
+    end
+
+    it "does not split inside nested brackets, templates, or links" do
+      text = "'''山野太郎'''（やまの たろう、[[2001年]]（平成13年、辛巳）[[4月1日]] - 、{{lang|en|a, b}}）は架空の人物。"
+      expect(terms(text).first["notes"]).to eq(["やまの たろう", "2001年（平成13年、辛巳）4月1日 -", "a, b"])
+    end
+
+    it "reports reading templates as pairs and keeps their reading out of the bold text" do
+      result = terms("'''{{読み仮名|言語|げんご}}'''は記号体系。")
+      expect(result.map { |t| t.slice("text", "reading", "source") })
+        .to eq([{ "text" => "言語", "source" => "bold" },
+                { "text" => "言語", "reading" => "げんご", "source" => "読み仮名" }])
+    end
+
+    it "stops at the first heading, ignores an unclosed bracket, and caps the count" do
+      expect(terms("'''甲'''（こう\n\n== 節 ==\n'''乙'''").map { |t| [t["text"], t["notes"]] }).to eq([["甲", []]])
+      many = (1..8).map { |i| "'''語#{i}'''" }.join("、")
+      expect(terms(many).map { |t| t["index"] }).to eq([0, 1, 2, 3, 4])
+    end
+
+    it "returns nothing for empty text" do
+      expect(terms("")).to eq([])
+    end
+  end
+
+  describe "command line" do
+    let(:cli) { File.expand_path("../bin/wp2txt", __dir__) }
+    let(:lib) { File.expand_path("../lib", __dir__) }
+
+    def records(*args)
+      out = File.join(@dir, "out#{args.hash.abs}")
+      Dir.mkdir(out)
+      _stdout, stderr, status = Open3.capture3(RbConfig.ruby, "-I", lib, cli, *args, "-o", out)
+      expect(status.success?).to be(true), stderr
+      Dir[File.join(out, "*")].flat_map { |f| File.readlines(f) }.map { |l| JSON.parse(l) }.to_h { |r| [r["title"], r] }
+    end
+
+    it "adds the Wikidata ID and lead terms to JSON, identically on both extraction paths" do
+      dump, db = build_dump
+      Wp2txt::PagePropsImporter.new(db).import!(page_props_file(PAGE_PROPS_SQL.b))
+      common = ["-i", dump, "--cache-dir", @dir, "--format", "json", "--summary-only", "--lead-terms"]
+      turbo = records(*common)
+      streamed = records(*common, "--no-turbo")
+
+      expect(turbo["東京"].keys.first(4)).to eq(%w[title page_id revision_id qid])
+      expect(turbo["東京"]["qid"]).to eq("Q1490")
+      expect(turbo["記事B"].slice("qid", "sort_key", "disambiguation")).to eq("qid" => nil, "sort_key" => nil, "disambiguation" => false)
+      expect(turbo["東京"]["lead_terms"].first.slice("text", "notes"))
+        .to eq("text" => "東京", "notes" => %w[とうきょう Tokyo])
+      # (the default path skips articles with empty text; compare what both emit)
+      common_titles = turbo.keys & streamed.keys
+      expect(common_titles.size).to eq(turbo.size)
+      expect(streamed.slice(*common_titles).transform_values { |r| r["lead_terms"] })
+        .to eq(turbo.transform_values { |r| r["lead_terms"] })
+    end
+
+    it "requires JSON output for --lead-terms" do
+      input = File.join(@dir, "x.xml.bz2")
+      File.write(input, "")
+      _stdout, stderr, status = Open3.capture3(RbConfig.ruby, "-I", lib, cli, "-i", input, "--lead-terms")
+      expect(status.success?).to be(false)
+      expect(stderr).to include("--lead-terms requires --format json")
+    end
+  end
+end

data/spec/p1_correctness_spec.rb

@@ -356,17 +356,20 @@ RSpec.describe "P1 correctness contracts" do
       Rake.application = previous
     end
 
-    [[false, "Cannot connect to daemon"], [false, "image missing"], [true, "LEAK:/wp2txt/tmp\n"]].each do |success, output|
-      it "fails verification for #{output.strip}" do
-        allow(Open3).to receive(:capture2e).and_return([output, double(success?: success)])
-        expect { suppress_stderr { Rake::Task[:verify_image].invoke("test-image") } }.to raise_error(SystemExit)
-      end
+    it "builds from a clean clone and hands the image to the CI gate with that clone as context" do
+      commands = []
+      allow(TOPLEVEL_BINDING.receiver).to receive(:sh) { |*args| commands << args }
+      Rake::Task[:check_image].invoke
+      clone, build, gate = commands
+      expect(clone.first(4)).to eq(%w[git clone --quiet --no-local])
+      dir = clone.last
+      expect(build).to eq(["docker", "build", "-t", "wp2txt-verify:local", dir])
+      expect(gate).to eq(["ruby", "scripts/verify_image.rb", "wp2txt-verify:local", "--context", dir])
     end
 
-    it "passes only after a successful clean container inspection" do
-      expect(Open3).to receive(:capture2e).with("docker", "run", "--rm", "test-image", "sh", "-c", kind_of(String))
-        .and_return(["", double(success?: true)])
-      expect { Rake::Task[:verify_image].invoke("test-image") }.to output(/OK:/).to_stdout
+    it "no longer carries a hand-written list of forbidden paths" do
+      expect(Rake::Task.task_defined?(:verify_image)).to be(false)
+      expect(defined?(IMAGE_FORBIDDEN_PATHS)).to be_nil
     end
   end
 end

data/spec/page_properties_spec.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+require "wp2txt/page_props_importer"
+require "wp2txt/metadata_index"
+require "wp2txt/corpus"
+require "wp2txt/link_counter"
+require "tmpdir"
+require "open3"
+require "json"
+require "zlib"
+require_relative "support/multistream_fixture"
+
+RSpec.describe "page properties" do
+  include MultistreamFixture
+  TITLES = %w[東京 QID パイプ Sort Invalid Other Empty Absent].freeze
+  FIELDS = %w[qid sort_key disambiguation].freeze
+
+  around do |example|
+    Dir.mktmpdir do |dir|
+      @dir = dir
+      @dump = File.join(dir, "testwiki-20260101-pages-articles-multistream.xml.bz2")
+      xml = TITLES.each_with_index.map { |t, i| page_xml(id: i + 1, ns: 0, title: t, text: "'''#{t}'''です。") }.join
+      File.binwrite(@dump, bzip2("<mediawiki>\n#{xml}</mediawiki>"))
+      File.write(@dump.sub(".xml.bz2", "-index.txt"), TITLES.each_with_index.map { |t, i| "0:#{i + 1}:#{t}\n" }.join)
+      @db_path = Wp2txt::MetadataIndex.path_for(@dump, cache_dir: dir)
+      Wp2txt::MetadataIndexBuilder.new(@dump, [0], db_path: @db_path, num_processes: 0).build.close
+      @source = File.join(dir, "testwiki-20260101-page_props.sql.gz")
+      @sql = <<~SQL
+        INSERT INTO `page_props` VALUES
+        (1,'defaultsort','とうきよう',NULL),
+        (1,'disambiguation','ignored',NULL),
+        (1,'wikibase_item','Q1490',NULL),
+        (2,'wikibase_item','Q2',NULL),
+        (3,'disambiguation','',NULL),
+        (4,'defaultsort','O\\'Brien あ',NULL),
+        (5,'defaultsort','\xFF\xFE',NULL),
+        (6,'other','ignored',NULL),
+        (7,'defaultsort','',NULL),
+        (8,'wikibase_item','bad',NULL);
+        INSERT INTO `other_table` VALUES (8,'wikibase_item','Q999',NULL);
+      SQL
+      Zlib::GzipWriter.open(@source) { |gz| gz.write(@sql.b) }
+      example.run
+    end
+  end
+
+  def import(**options)
+    Wp2txt::PagePropsImporter.new(@db_path).import!(@source, **options)
+  end
+
+  def with_db
+    db = SQLite3::Database.new(@db_path)
+    yield db
+  ensure
+    db&.close
+  end
+
+  it "merges all three properties across batches and records counts and invalid sort keys" do
+    stub_const("Wp2txt::PagePropsImporter::BATCH_SIZE", 1)
+    result = import
+    expect(result[:row_count]).to eq(5)
+    with_db do |db|
+      expect(db.execute("SELECT page_id,qid,disambiguation,sort_key FROM page_properties ORDER BY page_id"))
+        .to eq([[1, "Q1490", 1, "とうきよう"], [2, "Q2", 0, nil], [3, nil, 1, nil],
+                [4, nil, 0, "O'Brien あ"], [7, nil, 0, ""]])
+    end
+    expect(result[:provenance]).to include(page_count: 5, qid_count: 2, disambiguation_count: 2,
+                                         sort_key_count: 3, skipped_invalid_sort_keys: 1,
+                                         source_sha256: Digest::SHA256.file(@source).hexdigest)
+    expect(import).to include(status: :already_imported, row_count: 5, provenance: result[:provenance])
+  end
+
+  it "replaces the unreleased QID table without requiring force" do
+    with_db do |db|
+      db.execute("CREATE TABLE page_qids (page_id INTEGER PRIMARY KEY, qid TEXT)")
+      db.execute("INSERT INTO metadata VALUES ('page_props_imported_at','old')")
+    end
+    expect(import[:status]).to eq(:imported)
+    with_db { |db| expect(db.get_first_value("SELECT 1 FROM sqlite_master WHERE name='page_qids'")).to be_nil }
+  end
+
+  def records(*flags)
+    out = Dir.mktmpdir("out-", @dir)
+    _, stderr, status = Open3.capture3(RbConfig.ruby, "-I", File.expand_path("../lib", __dir__),
+                                      File.expand_path("../bin/wp2txt", __dir__), "-i", @dump,
+                                      "--cache-dir", @dir, "--format", "json", "-n", "2", "-o", out, *flags)
+    expect(status.success?).to be(true), stderr
+    Dir[File.join(out, "*.jsonl")].flat_map { |f| File.readlines(f).map { |s| JSON.parse(s) } }.to_h { |r| [r["title"], r] }
+  end
+
+  it "always emits the three fields after import on both CLI paths, including null and false" do
+    import
+    turbo, stream = records, records("--no-turbo")
+    expect(turbo.size).to eq(8)
+    expect(turbo).to eq(stream)
+    turbo.each_value { |r| expect(r.keys.first(6)).to eq(%w[title page_id revision_id qid sort_key disambiguation]) }
+    expect(turbo["東京"].slice(*FIELDS)).to eq("qid" => "Q1490", "sort_key" => "とうきよう", "disambiguation" => true)
+    expect(turbo["パイプ"].slice(*FIELDS)).to eq("qid" => nil, "sort_key" => nil, "disambiguation" => true)
+    expect(turbo["Absent"].slice(*FIELDS)).to eq("qid" => nil, "sort_key" => nil, "disambiguation" => false)
+    expect(turbo["Empty"]["sort_key"]).to eq("")
+  end
+
+  it "omits all three fields before import and after a failed force import" do
+    [false, true].each do |fail_import|
+      if fail_import
+        import
+        Zlib::GzipWriter.open(@source) { |gz| gz.write("-- empty\n") }
+        expect { import(force: true) }.to raise_error(Wp2txt::Error, /no page_props rows/)
+      end
+      [[], ["--no-turbo"]].each do |flags|
+        records(*flags).each_value { |r| expect(r.keys & FIELDS).to eq([]) }
+      end
+    end
+  end
+
+  it "attaches properties to Ractor JSON in the parent without omitting nulls" do
+    import
+    result = records("--no-turbo", "--ractor")
+    expect(result["東京"].slice(*FIELDS)).to eq("qid" => "Q1490", "sort_key" => "とうきよう", "disambiguation" => true)
+    expect(result["Absent"].slice(*FIELDS)).to eq("qid" => nil, "sort_key" => nil, "disambiguation" => false)
+  end
+
+  it "distinguishes an imported dump with no relevant properties from an unimported dump" do
+    Zlib::GzipWriter.open(@source) { |gz| gz.write("INSERT INTO `page_props` VALUES (1,'other','',NULL);\n") }
+    expect(import[:row_count]).to eq(0)
+    expect(records["東京"].slice(*FIELDS)).to eq("qid" => nil, "sort_key" => nil, "disambiguation" => false)
+  end
+
+  it "applies the same contract to MCP's Corpus methods and provenance" do
+    corpus = Wp2txt::Corpus.for_input(@dump, cache_dir: @dir)
+    expect(corpus.get_article("東京").keys & FIELDS.map(&:to_sym)).to eq([])
+    unimported = File.join(@dir, "unimported.jsonl")
+    corpus.extract_corpus(output_path: unimported, titles: ["東京", "Absent"], content: "full", num_processes: 0)
+    File.readlines(unimported).each { |line| expect(JSON.parse(line).keys & FIELDS).to eq([]) }
+    corpus.close
+    import
+    corpus = Wp2txt::Corpus.for_input(@dump, cache_dir: @dir)
+    expect(corpus.get_article("東京")).to include(qid: "Q1490", sort_key: "とうきよう", disambiguation: true)
+    expect(corpus.get_article("Absent")).to include(qid: nil, sort_key: nil, disambiguation: false)
+    output = File.join(@dir, "corpus.jsonl")
+    corpus.extract_corpus(output_path: output, titles: ["東京", "パイプ", "Absent"], content: "full", num_processes: 0)
+    rows = File.readlines(output).map { |s| JSON.parse(s) }
+    expect(rows.size).to eq(3)
+    rows.each { |r| expect(r.keys & FIELDS).to match_array(FIELDS) }
+    expect(rows.last.slice(*FIELDS)).to eq("qid" => nil, "sort_key" => nil, "disambiguation" => false)
+    expect(corpus.dump_info[:page_properties]).to include(qid_count: 2, disambiguation_count: 2, sort_key_count: 3,
+                                                       skipped_invalid_sort_keys: 1)
+  ensure
+    corpus&.close
+  end
+
+  it "decodes title references once, including escaped ampersands and numeric references" do
+    counter = Wp2txt::LinkCounter.new("", [], db_path: "")
+    counter.instance_variable_set(:@redirects, {})
+    direct, via = Hash.new(0), Hash.new(0)
+    text = "[[A&amp;amp;B]] [[Caf&#233;]] [[東京&#35;節]] [[X&amp;lt;Y]]"
+    counter.send(:count_page, page_xml(id: 1, ns: 0, title: "Source", text: text), direct, via)
+    expect(direct).to eq("A&amp;B" => 1, "Café" => 1, "東京" => 1, "X&lt;Y" => 1)
+  end
+end

data/spec/region_semantics_spec.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+require "wp2txt/lead_terms"
+require "wp2txt/link_counter"
+require_relative "support/multistream_fixture"
+
+RSpec.describe "literal regions and lead prose" do
+  include MultistreamFixture
+
+  def terms(text)
+    render = ->(s) { Object.new.extend(Wp2txt).format_wiki(s, expand_templates: true, markers: [:all]) }
+    Wp2txt::LeadTerms.extract(text, render: render)
+  end
+
+  def counts(text)
+    counter = Wp2txt::LinkCounter.new("", [], db_path: "")
+    counter.instance_variable_set(:@redirects, {})
+    direct, via = Hash.new(0), Hash.new(0)
+    counter.send(:count_page, page_xml(id: 1, ns: 0, title: "Source", text: text), direct, via)
+    direct
+  end
+
+  %w[gallery timeline].each do |tag|
+    it "counts links inside #{tag}, but excludes its bold text from the lead" do
+      text = "<#{tag}>[[東京]] '''偽'''（にせ）</#{tag}>\n\n'''大阪'''（おおさか）。"
+      expect(counts(text)).to eq("東京" => 1)
+      expect(terms(text).map { |term| term["text"] }).to eq(["大阪"])
+      expect(counts("<#{tag}>[[東京]]<nowiki>[[京都]]</nowiki>[[大阪]]</#{tag}>"))
+        .to eq("東京" => 1, "大阪" => 1)
+    end
+  end
+
+  it "counts links and extracts bold terms inside code as ordinary wikitext" do
+    text = "<code>'''東京'''（とうきょう） [[大阪]]</code>"
+    expect(counts(text)).to eq("大阪" => 1)
+    term = terms(text).first
+    expect(term).not_to be_nil
+    expect(term.slice("text", "notes")).to eq("text" => "東京", "notes" => ["とうきょう"])
+    s, e = term["span"]["bold"]
+    expect(text[s...e]).to eq("'''東京'''")
+    s, e = term["span"]["paren"]
+    expect(text[s...e]).to eq("（とうきょう）")
+  end
+
+  %w[nowiki pre math chem ce score syntaxhighlight source graph mapframe templatedata].each do |tag|
+    it "excludes literal #{tag} contents from both consumers and protects its pipes" do
+      region = "<#{tag}>[[東京]] '''偽'''（にせ）A|B</#{tag}>"
+      expect(counts(region + "[[大阪]]")).to eq("大阪" => 1)
+      expect(terms(region + "\n\n'''大阪'''（おおさか）。").map { |term| term["text"] }).to eq(["大阪"])
+      expect(Wp2txt::WikitextRegions.split_pipes("ruby|#{region}|よみ")).to eq(["ruby", region, "よみ"])
+    end
+  end
+
+  %w[gallery timeline code].each do |tag|
+    it "splits template pipes inside #{tag} while still protecting nested literal regions" do
+      expect(Wp2txt::WikitextRegions.split_pipes("ruby|<#{tag}>A|B</#{tag}>|よみ"))
+        .to eq(["ruby", "<#{tag}>A", "B</#{tag}>", "よみ"])
+      expect(Wp2txt::WikitextRegions.split_pipes("ruby|<#{tag}>A<nowiki>|</nowiki>B|C</#{tag}>|よみ"))
+        .to eq(["ruby", "<#{tag}>A<nowiki>|</nowiki>B", "C</#{tag}>", "よみ"])
+    end
+  end
+
+  it "continues to exclude comments and keeps rule version 2" do
+    expect(counts("<!-- [[東京]] -->[[大阪]]")).to eq("大阪" => 1)
+    expect(terms("<!-- '''偽''' -->'''大阪'''（おおさか）").first["text"]).to eq("大阪")
+    expect(Wp2txt::WikitextRegions.split_pipes("ruby|<!-- a|b -->東京|よみ"))
+      .to eq(["ruby", "<!-- a|b -->東京", "よみ"])
+    expect(Wp2txt::LinkCounter::RULE_VERSION).to eq("2")
+  end
+end