wp2txt 2.3.3 → 2.4.0

data/spec/lead_terms_links_qids_spec.rb

@@ -0,0 +1,204 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+require "json"
+require "open3"
+require "tmpdir"
+require "zlib"
+require_relative "support/multistream_fixture"
+require "wp2txt"
+require "wp2txt/metadata_index"
+require "wp2txt/multistream"
+require "wp2txt/sql_dump_reader"
+require "wp2txt/page_props_importer"
+require "wp2txt/link_counter"
+require "wp2txt/lead_terms"
+
+RSpec.describe "lead terms, incoming links, and Wikidata IDs" do
+  include MultistreamFixture
+
+  around do |example|
+    Dir.mktmpdir("wp2txt-ltq-") do |dir|
+      @dir = dir
+      example.run
+    end
+  end
+
+  def xml_page(id:, title:, text:, ns: 0)
+    esc = ->(s) { s.gsub("&", "&amp;").gsub("<", "&lt;").gsub(">", "&gt;") }
+    "<page>\n<title>#{esc.(title)}</title>\n<ns>#{ns}</ns>\n<id>#{id}</id>\n<revision>\n<id>#{id * 100}</id>\n" \
+      "<text bytes=\"#{text.bytesize}\">#{esc.(text)}</text>\n</revision>\n</page>\n"
+  end
+
+  PAGES = [
+    [1, "東京", "'''東京'''（とうきょう、Tokyo）は首都。\n\n== 歴史 ==\n'''後'''（あと）", 0],
+    [2, "東京都区部", "#REDIRECT [[東京]]", 0],
+    [3, "記事A", "[[東京]]と[[東京]]、[[東京都区部]]。[[M&A|買収]]。<!-- [[隠れ]] -->[[#節]][[]]", 0],
+    [4, "記事B", "[[東京#歴史|東京の歴史]]", 0],
+    [5, "記事C", "[[東京都区部]]のみ。", 0],
+    [6, "M&A", "'''{{読み仮名|合併|がっぺい}}'''と買収。", 0],
+    [7, "隠れ", "", 0],
+    [8, "Category:X", "[[東京]]", 14]
+  ].freeze
+
+  # One-stream dump with its index and a built metadata index
+  def build_dump
+    dump = File.join(@dir, "testwiki-20260101-pages-articles-multistream.xml.bz2")
+    File.binwrite(dump, bzip2(PAGES.map { |id, t, x, ns| xml_page(id: id, title: t, text: x, ns: ns) }.join))
+    File.write(File.join(@dir, "testwiki-20260101-pages-articles-multistream-index.txt"),
+               PAGES.map { |id, t, _, _| "0:#{id}:#{t}" }.join("\n") + "\n")
+    db = Wp2txt::MetadataIndex.path_for(dump, cache_dir: @dir)
+    Wp2txt::MetadataIndexBuilder.new(dump, [0], db_path: db, num_processes: 0).build.close
+    [dump, db]
+  end
+
+  def page_props_file(content, name: "testwiki-20260101-page_props.sql.gz")
+    path = File.join(@dir, name)
+    Zlib::GzipWriter.open(path) { |gz| gz.write(content) }
+    path
+  end
+
+  PAGE_PROPS_SQL = <<~SQL
+    INSERT INTO `page_props` VALUES
+    (1,'wikibase_item','Q1490',NULL),
+    (1,'page_image_free','T\\'kyo.jpg',NULL),
+    (3,'wikibase_item','Q9',1.5),
+    (6,'wikibase_item','Q1\\'bad',NULL),
+    (6,'defaultsort','\xFF\xFE',NULL);
+    INSERT INTO `page_restrictions` VALUES
+    (1,'wikibase_item','Q777',NULL);
+  SQL
+
+  describe Wp2txt::SqlDumpReader do
+    it "reads statements written on one line and one tuple per line, skipping other tables" do
+      lines = []
+      path = page_props_file("INSERT INTO `t` VALUES (1),(2);\nINSERT INTO `u` VALUES\n(9);\nINSERT INTO `t` VALUES\n(3),\n(4);\n")
+      described_class.each_insert_line(path, "t") { |l| lines << l.strip }
+      expect(lines).to eq(["INSERT INTO `t` VALUES (1),(2);", "INSERT INTO `t` VALUES", "(3),", "(4);"])
+    end
+  end
+
+  describe Wp2txt::PagePropsImporter do
+    it "imports only well-formed wikibase_item values, and records where they came from" do
+      _dump, db = build_dump
+      path = page_props_file(PAGE_PROPS_SQL.b)
+      result = described_class.new(db).import!(path)
+      expect(result[:row_count]).to eq(2)
+      rows = SQLite3::Database.new(db, readonly: true).execute("SELECT page_id, qid FROM page_properties ORDER BY page_id")
+      expect(rows).to eq([[1, "Q1490"], [3, "Q9"]])
+      expect(result[:provenance][:source_sha256]).to eq(Digest::SHA256.file(path).hexdigest)
+      expect(described_class.new(db).import!(path)[:status]).to eq(:already_imported)
+    end
+
+    it "refuses a dump of another date" do
+      _dump, db = build_dump
+      path = page_props_file(PAGE_PROPS_SQL.b, name: "testwiki-20250101-page_props.sql.gz")
+      expect { described_class.new(db).import!(path) }.to raise_error(ArgumentError, /version mismatch/)
+    end
+
+    it "fails when nothing could be read" do
+      _dump, db = build_dump
+      path = page_props_file("-- empty\n")
+      expect { described_class.new(db).import!(path) }.to raise_error(Wp2txt::Error, /no page_props rows/)
+    end
+  end
+
+  describe Wp2txt::LinkCounter do
+    it "counts each linking article once, through redirects, ignoring comments and non-articles" do
+      dump, db = build_dump
+      described_class.new(dump, [0], db_path: db, num_processes: 0).count!
+      counts = SQLite3::Database.new(db, readonly: true)
+                                .execute("SELECT p.title, i.inlinks, i.via_redirects FROM page_inlinks i " \
+                                         "JOIN pages p USING (page_id)").to_h { |t, n, v| [t, [n, v]] }
+      expect(counts["東京"]).to eq([3, 1]) # 記事A and 記事B directly, 記事C only via the redirect
+      expect(counts["M&A"]).to eq([1, 0])
+      expect(counts["隠れ"]).to eq([0, 0]) # linked only from a comment
+      expect(counts["記事A"]).to eq([0, 0])
+      expect(counts).not_to have_key("東京都区部") # redirects get no row of their own
+    end
+  end
+
+  describe Wp2txt::LeadTerms do
+    let(:render) { ->(fragment) { Object.new.extend(Wp2txt).format_wiki(fragment, { expand_templates: true, markers: [:all] }) } }
+
+    def terms(text)
+      Wp2txt::LeadTerms.extract(text, render: render)
+    end
+
+    it "takes bold terms of the first paragraph that has one, skipping templates, files, refs, and comments" do
+      text = "{{Otheruses|'''x'''}}\n[[ファイル:A.jpg|thumb|'''偽''']]<!-- '''隠''' --><ref>'''注'''</ref>\n" \
+             "'''日本語'''（にほんご、にっぽんご{{Refnest|注}}）は言語。'''和語'''とも。\n\n次の段落の'''別'''。"
+      expect(terms(text).map { |t| [t["text"], t["notes"]] }).to eq([["日本語", %w[にほんご にっぽんご]], ["和語", []]])
+    end
+
+    it "reports spans that cut the bold and the parentheses out of the original text" do
+      text = "前置き。'''東京'''　（とうきょう）は首都。"
+      term = terms(text).first
+      expect(text[Range.new(*term["span"]["bold"], true)]).to eq("'''東京'''")
+      expect(text[Range.new(*term["span"]["paren"], true)]).to eq("（とうきょう）")
+      expect(term["notes_text"]).to eq("とうきょう")
+    end
+
+    it "does not split inside nested brackets, templates, or links" do
+      text = "'''山野太郎'''（やまの たろう、[[2001年]]（平成13年、辛巳）[[4月1日]] - 、{{lang|en|a, b}}）は架空の人物。"
+      expect(terms(text).first["notes"]).to eq(["やまの たろう", "2001年（平成13年、辛巳）4月1日 -", "a, b"])
+    end
+
+    it "reports reading templates as pairs and keeps their reading out of the bold text" do
+      result = terms("'''{{読み仮名|言語|げんご}}'''は記号体系。")
+      expect(result.map { |t| t.slice("text", "reading", "source") })
+        .to eq([{ "text" => "言語", "source" => "bold" },
+                { "text" => "言語", "reading" => "げんご", "source" => "読み仮名" }])
+    end
+
+    it "stops at the first heading, ignores an unclosed bracket, and caps the count" do
+      expect(terms("'''甲'''（こう\n\n== 節 ==\n'''乙'''").map { |t| [t["text"], t["notes"]] }).to eq([["甲", []]])
+      many = (1..8).map { |i| "'''語#{i}'''" }.join("、")
+      expect(terms(many).map { |t| t["index"] }).to eq([0, 1, 2, 3, 4])
+    end
+
+    it "returns nothing for empty text" do
+      expect(terms("")).to eq([])
+    end
+  end
+
+  describe "command line" do
+    let(:cli) { File.expand_path("../bin/wp2txt", __dir__) }
+    let(:lib) { File.expand_path("../lib", __dir__) }
+
+    def records(*args)
+      out = File.join(@dir, "out#{args.hash.abs}")
+      Dir.mkdir(out)
+      _stdout, stderr, status = Open3.capture3(RbConfig.ruby, "-I", lib, cli, *args, "-o", out)
+      expect(status.success?).to be(true), stderr
+      Dir[File.join(out, "*")].flat_map { |f| File.readlines(f) }.map { |l| JSON.parse(l) }.to_h { |r| [r["title"], r] }
+    end
+
+    it "adds the Wikidata ID and lead terms to JSON, identically on both extraction paths" do
+      dump, db = build_dump
+      Wp2txt::PagePropsImporter.new(db).import!(page_props_file(PAGE_PROPS_SQL.b))
+      common = ["-i", dump, "--cache-dir", @dir, "--format", "json", "--summary-only", "--lead-terms"]
+      turbo = records(*common)
+      streamed = records(*common, "--no-turbo")
+
+      expect(turbo["東京"].keys.first(4)).to eq(%w[title page_id revision_id qid])
+      expect(turbo["東京"]["qid"]).to eq("Q1490")
+      expect(turbo["記事B"].slice("qid", "sort_key", "disambiguation")).to eq("qid" => nil, "sort_key" => nil, "disambiguation" => false)
+      expect(turbo["東京"]["lead_terms"].first.slice("text", "notes"))
+        .to eq("text" => "東京", "notes" => %w[とうきょう Tokyo])
+      # (the default path skips articles with empty text; compare what both emit)
+      common_titles = turbo.keys & streamed.keys
+      expect(common_titles.size).to eq(turbo.size)
+      expect(streamed.slice(*common_titles).transform_values { |r| r["lead_terms"] })
+        .to eq(turbo.transform_values { |r| r["lead_terms"] })
+    end
+
+    it "requires JSON output for --lead-terms" do
+      input = File.join(@dir, "x.xml.bz2")
+      File.write(input, "")
+      _stdout, stderr, status = Open3.capture3(RbConfig.ruby, "-I", lib, cli, "-i", input, "--lead-terms")
+      expect(status.success?).to be(false)
+      expect(stderr).to include("--lead-terms requires --format json")
+    end
+  end
+end

data/spec/output_integrity_spec.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+require "open3"
+require "json"
+require "tmpdir"
+require "parallel"
+require_relative "support/multistream_fixture"
+require "wp2txt/multistream"
+require "wp2txt/output_writer"
+require "wp2txt/stream_processor"
+
+RSpec.describe "output integrity" do
+  include MultistreamFixture
+
+  around do |example|
+    Dir.mktmpdir("wp2txt-integrity-") do |dir|
+      @dir = dir
+      example.run
+    end
+  end
+
+  let(:cli) { File.expand_path("../bin/wp2txt", __dir__) }
+  let(:lib) { File.expand_path("../lib", __dir__) }
+
+  def run_cli(*args)
+    Open3.capture3(RbConfig.ruby, "-I", lib, cli, *args)
+  end
+
+  def json_records(dir)
+    Dir[File.join(dir, "*")].flat_map { |f| File.readlines(f) }.map { |line| JSON.parse(line) }
+  end
+
+  def write_pages(path, count)
+    body = (1..count).map { |i| page_xml(id: i, ns: 0, title: "記事#{i}", text: "本文#{i}。日本語の文。\n") }.join
+    File.write(path, "<mediawiki>\n#{body}</mediawiki>\n")
+  end
+
+  describe "forked workers" do
+    it "do not write the parent's buffered output again when they exit" do
+      writer = Wp2txt::OutputWriter.new(output_dir: @dir, base_name: "out", format: :text, file_size_mb: 10)
+      writer.write("one line still sitting in the buffer")
+      writer.flush
+      Parallel.map([1, 2, 3], in_processes: 3) { |x| x }
+      files = writer.close
+      expect(File.readlines(files.first).size).to eq(1)
+    end
+
+    it "leave every article exactly once in streaming output across batch boundaries" do
+      input = File.join(@dir, "pages.xml")
+      out = File.join(@dir, "out")
+      Dir.mkdir(out)
+      write_pages(input, 450) # batches of 200 with -n 2
+      _stdout, stderr, status = run_cli("-i", input, "-o", out, "--format", "json", "-n", "2")
+      expect(status.success?).to be(true), stderr
+
+      titles = json_records(out).map { |r| r["title"] }
+      expect(titles.size).to eq(450)
+      expect(titles.tally.select { |_, n| n > 1 }).to be_empty
+    end
+  end
+
+  describe "--num-procs" do
+    it "uses the requested process count" do
+      input = File.join(@dir, "pages.xml")
+      out = File.join(@dir, "out")
+      Dir.mkdir(out)
+      write_pages(input, 3)
+      stdout, stderr, status = run_cli("-i", input, "-o", out, "--format", "json", "-n", "1")
+      expect(status.success?).to be(true), stderr
+      expect(stdout.gsub(/\e\[[\d;]*m/, "")).to match(/CPU cores:\s*1\b/)
+    end
+  end
+
+  describe "page and revision IDs" do
+    it "appear in JSON records from the streaming path, including summaries" do
+      input = File.join(@dir, "pages.xml")
+      write_pages(input, 3)
+      [[], ["--summary-only"]].each do |extra|
+        out = File.join(@dir, "out#{extra.size}")
+        Dir.mkdir(out)
+        _stdout, stderr, status = run_cli("-i", input, "-o", out, "--format", "json", *extra)
+        expect(status.success?).to be(true), stderr
+        record = json_records(out).find { |r| r["title"] == "記事2" }
+        expect(record).to include("page_id" => 2, "revision_id" => 200)
+        expect(record.keys.first(3)).to eq(%w[title page_id revision_id])
+      end
+    end
+
+    it "appear in JSON records from the default path for bz2 dumps" do
+      dump, = create_fixture(@dir)
+      out = File.join(@dir, "out")
+      Dir.mkdir(out)
+      _stdout, stderr, status = run_cli("-i", dump, "-o", out, "--format", "json")
+      expect(status.success?).to be(true), stderr
+      record = json_records(out).find { |r| r["title"] == "Film B" }
+      expect(record).to include("page_id" => 2, "revision_id" => 200)
+    end
+
+    it "are read from the page header, not from the contributor" do
+      xml = "<page><title>X</title><ns>0</ns><id>12</id><revision><id>34</id>" \
+            "<contributor><id>9</id></contributor><text>t</text></revision></page>"
+      expect(Wp2txt.page_ids(xml)).to eq(page_id: 12, revision_id: 34)
+    end
+
+    it "are offered by StreamProcessor only on request, keeping each_page's shape" do
+      input = File.join(@dir, "pages.xml")
+      write_pages(input, 1)
+      processor = Wp2txt::StreamProcessor.new(input, adaptive_buffer: false)
+      expect(processor.each_page.to_a).to eq([["記事1", "本文1。日本語の文。\n"]])
+      with_ids = Wp2txt::StreamProcessor.new(input, adaptive_buffer: false).each_page(with_ids: true).to_a
+      expect(with_ids.first.last).to eq(page_id: 1, revision_id: 100)
+    end
+  end
+
+  describe "targeted extraction after an early index stop" do
+    it "knows where the last found article's stream ends" do
+      _dump, index_path = create_fixture(@dir)
+      index = Wp2txt::MultistreamIndex.new(index_path, use_cache: false, target_titles: ["Film A"],
+                                                       show_progress: false)
+      expect(index.early_terminated?).to be(true)
+      expect(index.stream_offsets).to eq([0])
+      second_stream = File.readlines(index_path).map { |line| line.split(":").first.to_i }.uniq[1]
+      expect(index.stream_end_offset).to eq(second_stream)
+    end
+
+    it "reads only that stream" do
+      dump, index_path = create_fixture(@dir)
+      index = Wp2txt::MultistreamIndex.new(index_path, use_cache: false, target_titles: ["Film A"],
+                                                       show_progress: false)
+      reader = Wp2txt::MultistreamReader.new(dump, index)
+      stub_const("Wp2txt::MultistreamReader::MAX_TAIL_STREAM_BYTES", 0)
+      expect(reader.extract_article("Film A")).to include(title: "Film A", id: 1, revision_id: 100)
+    end
+
+    it "refuses to read a large remainder in one call when the end is unknown" do
+      dump, index_path = create_fixture(@dir)
+      index = Wp2txt::MultistreamIndex.new(index_path, use_cache: false, target_titles: ["Film A"],
+                                                       show_progress: false)
+      index.instance_variable_set(:@stream_end_offset, nil)
+      reader = Wp2txt::MultistreamReader.new(dump, index)
+      stub_const("Wp2txt::MultistreamReader::MAX_TAIL_STREAM_BYTES", 0)
+      expect { reader.extract_article("Film A") }.to raise_error(Wp2txt::Error, /cannot locate the end/)
+    end
+  end
+end

data/spec/p1_correctness_spec.rb

@@ -40,10 +40,25 @@ RSpec.describe "P1 correctness contracts" do
       end
     end
 
-    it "scrubs invalid bytes and incomplete EOF without losing adjacent valid text" do
-      processor = processor_for("é\xFFあ😀\xE3\x81".b, 1)
-      nil while processor.send(:fill_buffer)
-      expect(processor.instance_variable_get(:@buffer)).to eq("éあ😀")
+    it "stops on an invalid byte and reports where it is" do
+      processor = processor_for("éあ\xFF😀".b, 1)
+      expect { nil while processor.send(:fill_buffer) }
+        .to raise_error(Wp2txt::EncodingError, /byte 5\b/)
+    end
+
+    it "stops when the input ends partway through a character" do
+      processor = processor_for("éあ\xE3\x81".b, 1)
+      expect { nil while processor.send(:fill_buffer) }
+        .to raise_error(Wp2txt::EncodingError, /middle of a UTF-8 character/)
+    end
+
+    it "passes valid text through unchanged at every buffer size" do
+      text = "é\nあ😀終\n" * 7
+      [1, 2, 3, 5, 64].each do |size|
+        processor = processor_for(text.b, size)
+        nil while processor.send(:fill_buffer)
+        expect(processor.instance_variable_get(:@buffer)).to eq(text)
+      end
     end
 
     it "handles empty input" do
@@ -105,7 +120,7 @@ RSpec.describe "P1 correctness contracts" do
     it "retains the historical ns=0 default for missing ns in both parsers" do
       xml = page_xml(id: 1, ns: 0, title: "作品: 東京", text: "本文").sub(/<ns>.*?<\/ns>/, "")
       processor = Wp2txt::StreamProcessor.new("unused.xml", adaptive_buffer: false)
-      expect(processor.send(:parse_page_xml, xml)).to eq(["作品: 東京", "本文"])
+      expect(processor.send(:parse_page_xml, xml)).to eq(["作品: 東京", "本文", { page_id: 1, revision_id: 100 }])
       rows = { pages: [], categories: [], sections: [], hierarchy: [] }
       Wp2txt::MetadataIndexBuilder.scan_page(xml, rows)
       expect(rows[:pages].first[2]).to eq(0)
@@ -341,17 +356,20 @@ RSpec.describe "P1 correctness contracts" do
       Rake.application = previous
     end
 
-    [[false, "Cannot connect to daemon"], [false, "image missing"], [true, "LEAK:/wp2txt/tmp\n"]].each do |success, output|
-      it "fails verification for #{output.strip}" do
-        allow(Open3).to receive(:capture2e).and_return([output, double(success?: success)])
-        expect { suppress_stderr { Rake::Task[:verify_image].invoke("test-image") } }.to raise_error(SystemExit)
-      end
+    it "builds from a clean clone and hands the image to the CI gate with that clone as context" do
+      commands = []
+      allow(TOPLEVEL_BINDING.receiver).to receive(:sh) { |*args| commands << args }
+      Rake::Task[:check_image].invoke
+      clone, build, gate = commands
+      expect(clone.first(4)).to eq(%w[git clone --quiet --no-local])
+      dir = clone.last
+      expect(build).to eq(["docker", "build", "-t", "wp2txt-verify:local", dir])
+      expect(gate).to eq(["ruby", "scripts/verify_image.rb", "wp2txt-verify:local", "--context", dir])
     end
 
-    it "passes only after a successful clean container inspection" do
-      expect(Open3).to receive(:capture2e).with("docker", "run", "--rm", "test-image", "sh", "-c", kind_of(String))
-        .and_return(["", double(success?: true)])
-      expect { Rake::Task[:verify_image].invoke("test-image") }.to output(/OK:/).to_stdout
+    it "no longer carries a hand-written list of forbidden paths" do
+      expect(Rake::Task.task_defined?(:verify_image)).to be(false)
+      expect(defined?(IMAGE_FORBIDDEN_PATHS)).to be_nil
     end
   end
 end

data/spec/page_properties_spec.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+require "wp2txt/page_props_importer"
+require "wp2txt/metadata_index"
+require "wp2txt/corpus"
+require "wp2txt/link_counter"
+require "tmpdir"
+require "open3"
+require "json"
+require "zlib"
+require_relative "support/multistream_fixture"
+
+RSpec.describe "page properties" do
+  include MultistreamFixture
+  TITLES = %w[東京 QID パイプ Sort Invalid Other Empty Absent].freeze
+  FIELDS = %w[qid sort_key disambiguation].freeze
+
+  around do |example|
+    Dir.mktmpdir do |dir|
+      @dir = dir
+      @dump = File.join(dir, "testwiki-20260101-pages-articles-multistream.xml.bz2")
+      xml = TITLES.each_with_index.map { |t, i| page_xml(id: i + 1, ns: 0, title: t, text: "'''#{t}'''です。") }.join
+      File.binwrite(@dump, bzip2("<mediawiki>\n#{xml}</mediawiki>"))
+      File.write(@dump.sub(".xml.bz2", "-index.txt"), TITLES.each_with_index.map { |t, i| "0:#{i + 1}:#{t}\n" }.join)
+      @db_path = Wp2txt::MetadataIndex.path_for(@dump, cache_dir: dir)
+      Wp2txt::MetadataIndexBuilder.new(@dump, [0], db_path: @db_path, num_processes: 0).build.close
+      @source = File.join(dir, "testwiki-20260101-page_props.sql.gz")
+      @sql = <<~SQL
+        INSERT INTO `page_props` VALUES
+        (1,'defaultsort','とうきよう',NULL),
+        (1,'disambiguation','ignored',NULL),
+        (1,'wikibase_item','Q1490',NULL),
+        (2,'wikibase_item','Q2',NULL),
+        (3,'disambiguation','',NULL),
+        (4,'defaultsort','O\\'Brien あ',NULL),
+        (5,'defaultsort','\xFF\xFE',NULL),
+        (6,'other','ignored',NULL),
+        (7,'defaultsort','',NULL),
+        (8,'wikibase_item','bad',NULL);
+        INSERT INTO `other_table` VALUES (8,'wikibase_item','Q999',NULL);
+      SQL
+      Zlib::GzipWriter.open(@source) { |gz| gz.write(@sql.b) }
+      example.run
+    end
+  end
+
+  def import(**options)
+    Wp2txt::PagePropsImporter.new(@db_path).import!(@source, **options)
+  end
+
+  def with_db
+    db = SQLite3::Database.new(@db_path)
+    yield db
+  ensure
+    db&.close
+  end
+
+  it "merges all three properties across batches and records counts and invalid sort keys" do
+    stub_const("Wp2txt::PagePropsImporter::BATCH_SIZE", 1)
+    result = import
+    expect(result[:row_count]).to eq(5)
+    with_db do |db|
+      expect(db.execute("SELECT page_id,qid,disambiguation,sort_key FROM page_properties ORDER BY page_id"))
+        .to eq([[1, "Q1490", 1, "とうきよう"], [2, "Q2", 0, nil], [3, nil, 1, nil],
+                [4, nil, 0, "O'Brien あ"], [7, nil, 0, ""]])
+    end
+    expect(result[:provenance]).to include(page_count: 5, qid_count: 2, disambiguation_count: 2,
+                                         sort_key_count: 3, skipped_invalid_sort_keys: 1,
+                                         source_sha256: Digest::SHA256.file(@source).hexdigest)
+    expect(import).to include(status: :already_imported, row_count: 5, provenance: result[:provenance])
+  end
+
+  it "replaces the unreleased QID table without requiring force" do
+    with_db do |db|
+      db.execute("CREATE TABLE page_qids (page_id INTEGER PRIMARY KEY, qid TEXT)")
+      db.execute("INSERT INTO metadata VALUES ('page_props_imported_at','old')")
+    end
+    expect(import[:status]).to eq(:imported)
+    with_db { |db| expect(db.get_first_value("SELECT 1 FROM sqlite_master WHERE name='page_qids'")).to be_nil }
+  end
+
+  def records(*flags)
+    out = Dir.mktmpdir("out-", @dir)
+    _, stderr, status = Open3.capture3(RbConfig.ruby, "-I", File.expand_path("../lib", __dir__),
+                                      File.expand_path("../bin/wp2txt", __dir__), "-i", @dump,
+                                      "--cache-dir", @dir, "--format", "json", "-n", "2", "-o", out, *flags)
+    expect(status.success?).to be(true), stderr
+    Dir[File.join(out, "*.jsonl")].flat_map { |f| File.readlines(f).map { |s| JSON.parse(s) } }.to_h { |r| [r["title"], r] }
+  end
+
+  it "always emits the three fields after import on both CLI paths, including null and false" do
+    import
+    turbo, stream = records, records("--no-turbo")
+    expect(turbo.size).to eq(8)
+    expect(turbo).to eq(stream)
+    turbo.each_value { |r| expect(r.keys.first(6)).to eq(%w[title page_id revision_id qid sort_key disambiguation]) }
+    expect(turbo["東京"].slice(*FIELDS)).to eq("qid" => "Q1490", "sort_key" => "とうきよう", "disambiguation" => true)
+    expect(turbo["パイプ"].slice(*FIELDS)).to eq("qid" => nil, "sort_key" => nil, "disambiguation" => true)
+    expect(turbo["Absent"].slice(*FIELDS)).to eq("qid" => nil, "sort_key" => nil, "disambiguation" => false)
+    expect(turbo["Empty"]["sort_key"]).to eq("")
+  end
+
+  it "omits all three fields before import and after a failed force import" do
+    [false, true].each do |fail_import|
+      if fail_import
+        import
+        Zlib::GzipWriter.open(@source) { |gz| gz.write("-- empty\n") }
+        expect { import(force: true) }.to raise_error(Wp2txt::Error, /no page_props rows/)
+      end
+      [[], ["--no-turbo"]].each do |flags|
+        records(*flags).each_value { |r| expect(r.keys & FIELDS).to eq([]) }
+      end
+    end
+  end
+
+  it "attaches properties to Ractor JSON in the parent without omitting nulls" do
+    import
+    result = records("--no-turbo", "--ractor")
+    expect(result["東京"].slice(*FIELDS)).to eq("qid" => "Q1490", "sort_key" => "とうきよう", "disambiguation" => true)
+    expect(result["Absent"].slice(*FIELDS)).to eq("qid" => nil, "sort_key" => nil, "disambiguation" => false)
+  end
+
+  it "distinguishes an imported dump with no relevant properties from an unimported dump" do
+    Zlib::GzipWriter.open(@source) { |gz| gz.write("INSERT INTO `page_props` VALUES (1,'other','',NULL);\n") }
+    expect(import[:row_count]).to eq(0)
+    expect(records["東京"].slice(*FIELDS)).to eq("qid" => nil, "sort_key" => nil, "disambiguation" => false)
+  end
+
+  it "applies the same contract to MCP's Corpus methods and provenance" do
+    corpus = Wp2txt::Corpus.for_input(@dump, cache_dir: @dir)
+    expect(corpus.get_article("東京").keys & FIELDS.map(&:to_sym)).to eq([])
+    unimported = File.join(@dir, "unimported.jsonl")
+    corpus.extract_corpus(output_path: unimported, titles: ["東京", "Absent"], content: "full", num_processes: 0)
+    File.readlines(unimported).each { |line| expect(JSON.parse(line).keys & FIELDS).to eq([]) }
+    corpus.close
+    import
+    corpus = Wp2txt::Corpus.for_input(@dump, cache_dir: @dir)
+    expect(corpus.get_article("東京")).to include(qid: "Q1490", sort_key: "とうきよう", disambiguation: true)
+    expect(corpus.get_article("Absent")).to include(qid: nil, sort_key: nil, disambiguation: false)
+    output = File.join(@dir, "corpus.jsonl")
+    corpus.extract_corpus(output_path: output, titles: ["東京", "パイプ", "Absent"], content: "full", num_processes: 0)
+    rows = File.readlines(output).map { |s| JSON.parse(s) }
+    expect(rows.size).to eq(3)
+    rows.each { |r| expect(r.keys & FIELDS).to match_array(FIELDS) }
+    expect(rows.last.slice(*FIELDS)).to eq("qid" => nil, "sort_key" => nil, "disambiguation" => false)
+    expect(corpus.dump_info[:page_properties]).to include(qid_count: 2, disambiguation_count: 2, sort_key_count: 3,
+                                                       skipped_invalid_sort_keys: 1)
+  ensure
+    corpus&.close
+  end
+
+  it "decodes title references once, including escaped ampersands and numeric references" do
+    counter = Wp2txt::LinkCounter.new("", [], db_path: "")
+    counter.instance_variable_set(:@redirects, {})
+    direct, via = Hash.new(0), Hash.new(0)
+    text = "[[A&amp;amp;B]] [[Caf&#233;]] [[東京&#35;節]] [[X&amp;lt;Y]]"
+    counter.send(:count_page, page_xml(id: 1, ns: 0, title: "Source", text: text), direct, via)
+    expect(direct).to eq("A&amp;B" => 1, "Café" => 1, "東京" => 1, "X&lt;Y" => 1)
+  end
+end

data/spec/region_semantics_spec.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+require "wp2txt/lead_terms"
+require "wp2txt/link_counter"
+require_relative "support/multistream_fixture"
+
+RSpec.describe "literal regions and lead prose" do
+  include MultistreamFixture
+
+  def terms(text)
+    render = ->(s) { Object.new.extend(Wp2txt).format_wiki(s, expand_templates: true, markers: [:all]) }
+    Wp2txt::LeadTerms.extract(text, render: render)
+  end
+
+  def counts(text)
+    counter = Wp2txt::LinkCounter.new("", [], db_path: "")
+    counter.instance_variable_set(:@redirects, {})
+    direct, via = Hash.new(0), Hash.new(0)
+    counter.send(:count_page, page_xml(id: 1, ns: 0, title: "Source", text: text), direct, via)
+    direct
+  end
+
+  %w[gallery timeline].each do |tag|
+    it "counts links inside #{tag}, but excludes its bold text from the lead" do
+      text = "<#{tag}>[[東京]] '''偽'''（にせ）</#{tag}>\n\n'''大阪'''（おおさか）。"
+      expect(counts(text)).to eq("東京" => 1)
+      expect(terms(text).map { |term| term["text"] }).to eq(["大阪"])
+      expect(counts("<#{tag}>[[東京]]<nowiki>[[京都]]</nowiki>[[大阪]]</#{tag}>"))
+        .to eq("東京" => 1, "大阪" => 1)
+    end
+  end
+
+  it "counts links and extracts bold terms inside code as ordinary wikitext" do
+    text = "<code>'''東京'''（とうきょう） [[大阪]]</code>"
+    expect(counts(text)).to eq("大阪" => 1)
+    term = terms(text).first
+    expect(term).not_to be_nil
+    expect(term.slice("text", "notes")).to eq("text" => "東京", "notes" => ["とうきょう"])
+    s, e = term["span"]["bold"]
+    expect(text[s...e]).to eq("'''東京'''")
+    s, e = term["span"]["paren"]
+    expect(text[s...e]).to eq("（とうきょう）")
+  end
+
+  %w[nowiki pre math chem ce score syntaxhighlight source graph mapframe templatedata].each do |tag|
+    it "excludes literal #{tag} contents from both consumers and protects its pipes" do
+      region = "<#{tag}>[[東京]] '''偽'''（にせ）A|B</#{tag}>"
+      expect(counts(region + "[[大阪]]")).to eq("大阪" => 1)
+      expect(terms(region + "\n\n'''大阪'''（おおさか）。").map { |term| term["text"] }).to eq(["大阪"])
+      expect(Wp2txt::WikitextRegions.split_pipes("ruby|#{region}|よみ")).to eq(["ruby", region, "よみ"])
+    end
+  end
+
+  %w[gallery timeline code].each do |tag|
+    it "splits template pipes inside #{tag} while still protecting nested literal regions" do
+      expect(Wp2txt::WikitextRegions.split_pipes("ruby|<#{tag}>A|B</#{tag}>|よみ"))
+        .to eq(["ruby", "<#{tag}>A", "B</#{tag}>", "よみ"])
+      expect(Wp2txt::WikitextRegions.split_pipes("ruby|<#{tag}>A<nowiki>|</nowiki>B|C</#{tag}>|よみ"))
+        .to eq(["ruby", "<#{tag}>A<nowiki>|</nowiki>B", "C</#{tag}>", "よみ"])
+    end
+  end
+
+  it "continues to exclude comments and keeps rule version 2" do
+    expect(counts("<!-- [[東京]] -->[[大阪]]")).to eq("大阪" => 1)
+    expect(terms("<!-- '''偽''' -->'''大阪'''（おおさか）").first["text"]).to eq("大阪")
+    expect(Wp2txt::WikitextRegions.split_pipes("ruby|<!-- a|b -->東京|よみ"))
+      .to eq(["ruby", "<!-- a|b -->東京", "よみ"])
+    expect(Wp2txt::LinkCounter::RULE_VERSION).to eq("2")
+  end
+end