wp2txt 2.3.3 → 2.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 051b5501ecf2aad35ca972af6527c5afbce88222c17e9c93dc76f806b91a8e9d
-  data.tar.gz: 02efabb7102da40c144a3c8204f564a8a40dfa1e77a54813e40bd6740693a264
+  metadata.gz: 3972afe52897df60befbb96bed211e4fd05e33cce380ed46ced223637aed6954
+  data.tar.gz: 6674325567d9b21cfb8819110827a16327aaee2e7e0e509115953e3b97229b74
 SHA512:
-  metadata.gz: d89ff3e231200c36f647284fee43565021c69c73ac8d9f23ca42b58008d2d0d3ac5598fe63aa3736a45ce933622dd28f649f721635a63e782d638f444602d08b
-  data.tar.gz: c70fccdb332688f805e77dbefd291563045e2f53c07e7cc994dbff0a7f8858dded28f18bce4bcb1296e7594f12ee54eea12a4573beee04a8944d4e39b7d35620
+  metadata.gz: b7e36ac5cc6ab08699f3765768d9629b8cf193d1c134fcaeb9925228739b960bbc71c9ac5056c0bda5add9d1110cf993220b0a5a508e29caa9529eac84830278
+  data.tar.gz: abba849021598f7003a6fce9798517b2f497486853ce65522f687ea767eea1a1b91b87046c715f90d7908bff228cc441beec6cf4907c4885adeddd9967335165

data/CHANGELOG.md

@@ -8,6 +8,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 Entries were rewritten in August 2026 to describe what changed for people using
 wp2txt, rather than how it was implemented. The changes themselves are unaltered.
 
+## [2.3.4] - 2026-09-30
+
+- **Records are no longer duplicated in `--no-turbo` output or in large `extract_corpus` runs**: the last record written before each batch could appear once more for every worker process — up to eight identical lines in a row. The default mode for `.bz2` dumps was not affected. If you extracted more than 200 articles with `extract_corpus`, or used `--no-turbo`, check earlier output for repeated lines; the repeats are byte-for-byte identical, so removing exact duplicate lines restores it
+- **`--num-procs` is now honoured**: any value smaller than the number of CPU cores was silently replaced with the automatic choice. The value you give is used, and one outside 1 to the core count is adjusted with a warning
+- **`--articles` no longer fails on large dumps**: extracting specific articles could stop with `Invalid argument` (seen on macOS) when the last requested article sat early in a multi-gigabyte dump
+- **JSON records include `page_id` and `revision_id`**: the dump's own IDs for the page and its revision, right after `title`, in every mode — so each record can be traced to the exact version it came from. From Ruby, `StreamProcessor#each_page(with_ids: true)` yields them as a third value; plain `each_page` is unchanged
+- **Corrupt input now stops processing instead of being cleaned silently**: invalid UTF-8 raises `Wp2txt::EncodingError` naming where it was found. Official dumps are valid UTF-8, so this only triggers on damaged files
+- **Articles that fail to render are reported**: the default mode skipped them without a word; each one is now named on stderr with the reason
+
 ## [2.3.3] - 2026-09-08
 
 - **Short searches no longer report a false zero**: in Japanese, Chinese, and Korean indexes a phrase of one or two characters could match nothing and come back as `0 matches`, which reads exactly like a term that is genuinely absent from the dump. Such a search now fails with an explicit error instead. If you recorded a zero result for a short term with an earlier version, re-check it

data/README.md

@@ -172,13 +172,16 @@ CATEGORIES: Category1, Category2, Category3
 Each line contains one JSON object:
 
 ```json
-{"title": "Article Title", "categories": ["Cat1", "Cat2"], "text": "...", "redirect": null}
+{"title": "Article Title", "page_id": 12345, "revision_id": 67890, "categories": ["Cat1", "Cat2"], "text": "...", "redirect": null}
 ```
 
+`page_id` and `revision_id` are the dump's own `<id>` values for the page and its
+revision, so a record can be traced back to the exact version of the article it came from.
+
 For redirect articles:
 
 ```json
-{"title": "NYC", "categories": [], "text": "", "redirect": "New York City"}
+{"title": "NYC", "page_id": 23456, "revision_id": 78901, "categories": [], "text": "", "redirect": "New York City"}
 ```
 
 ## Cache Management

data/README_ja.md

@@ -156,13 +156,16 @@ CATEGORIES: カテゴリ1, カテゴリ2, カテゴリ3
 各行に1つのJSONオブジェクト：
 
 ```json
-{"title": "記事タイトル", "categories": ["カテゴリ1", "カテゴリ2"], "text": "...", "redirect": null}
+{"title": "記事タイトル", "page_id": 12345, "revision_id": 67890, "categories": ["カテゴリ1", "カテゴリ2"], "text": "...", "redirect": null}
 ```
 
+`page_id` と `revision_id` はダンプに記録されたページと版の `<id>` です。各レコードが
+どの記事のどの版から取り出されたかを後から辿れます。
+
 リダイレクト記事の場合：
 
 ```json
-{"title": "NYC", "categories": [], "text": "", "redirect": "New York City"}
+{"title": "NYC", "page_id": 23456, "revision_id": 78901, "categories": [], "text": "", "redirect": "New York City"}
 ```
 
 ## キャッシュ管理

data/bin/wp2txt

@@ -45,14 +45,14 @@ class WpApp
   def calculate_num_processes(opts)
     optimal = Wp2txt::MemoryMonitor.optimal_processes
 
-    if opts[:num_procs]
-      # User specified a value - use it if reasonable
-      requested = opts[:num_procs].to_i
-      max_allowed = Etc.nprocessors
-      [requested, max_allowed, 1].max == requested ? requested : optimal
-    else
-      optimal
-    end.tap { |n| n = 1 if n < 1 }
+    return optimal unless opts[:num_procs]
+
+    requested = opts[:num_procs].to_i
+    used = requested.clamp(1, Etc.nprocessors)
+    if used != requested
+      print_warning("--num-procs #{requested} is outside 1..#{Etc.nprocessors}; using #{used}.")
+    end
+    used
   end
 
   # Process articles using turbo mode (split-first architecture from v1.x)
@@ -244,6 +244,9 @@ class WpApp
           next if redirect_page?(text)
 
           article = Article.new(text, title, strip_tmarker)
+          ids = Wp2txt.page_ids(page_xml)
+          article.page_id = ids[:page_id]
+          article.revision_id = ids[:revision_id]
           result = format_article(article, config)
           next unless result
 
@@ -254,7 +257,9 @@ class WpApp
             out.puts(result)
           end
           article_count += 1
-        rescue StandardError
+        rescue StandardError => e
+          # Keep going, but never lose an article silently
+          warn "wp2txt: skipped #{title.inspect}: #{e.class}: #{e.message}"
           next
         end
       end
@@ -355,8 +360,8 @@ class WpApp
     end
     puts
 
-    stream.each_page do |title, text|
-      pages << [title, text]
+    stream.each_page(with_ids: true) do |title, text, ids|
+      pages << [title, text, ids]
       page_count += 1
 
       # Process batch when full
@@ -449,8 +454,13 @@ class WpApp
                 )
               else
                 # Fall back to Parallel gem (process-based parallelism)
-                Parallel.map(pages, in_processes: num_processes) do |title, text|
+                # Anything the writer still buffers would be inherited by every
+                # forked worker and written again as each one exits.
+                writer.flush
+                Parallel.map(pages, in_processes: num_processes) do |title, text, ids|
                   article = Article.new(text, title, strip_tmarker)
+                  article.page_id = ids&.dig(:page_id)
+                  article.revision_id = ids&.dig(:revision_id)
                   format_article(article, config)
                 end
               end

data/lib/wp2txt/article.rb

@@ -26,7 +26,7 @@ module Wp2txt
   # an article contains elements, each of which is [TYPE, string]
   class Article
     include Wp2txt
-    attr_accessor :elements, :title, :categories
+    attr_accessor :elements, :title, :categories, :page_id, :revision_id
 
     def initialize(text, title = "", strip_tmarker = false)
       @title = title.strip

data/lib/wp2txt/constants.rb

@@ -6,6 +6,17 @@ module Wp2txt
     (value || "0").to_i
   end
 
+  # Page and revision IDs from one <page> element of a dump. The page's own
+  # <id> precedes <revision>; the revision's <id> is its first child (the
+  # contributor's <id> comes later). Only the header before <text> is read.
+  def self.page_ids(page_xml)
+    head = page_xml[0, page_xml.index("<text") || page_xml.size]
+    {
+      page_id: head[%r{<page>.*?<id>(\d+)</id>}m, 1]&.to_i,
+      revision_id: head[%r{<revision>\s*<id>(\d+)</id>}, 1]&.to_i
+    }
+  end
+
   # =========================================================================
   # Custom Exception Classes
   # =========================================================================

data/lib/wp2txt/corpus.rb

@@ -401,6 +401,8 @@ module Wp2txt
           titles.each_slice(EXTRACT_BATCH_SIZE) do |batch|
             raise Cancelled if cancel_check&.call
 
+            # Forked workers inherit this file's buffer and would write it again on exit
+            f.flush
             pages = reader.extract_articles_parallel(batch, num_processes: num_processes)
             batch.each do |t|
               page = pages[t]

data/lib/wp2txt/extractor.rb

@@ -111,6 +111,8 @@ module Wp2txt
 
         if page
           article = Article.new(page[:text], page[:title], !config[:marker])
+          article.page_id = page[:id]
+          article.revision_id = page[:revision_id]
           result = format_article(article, config)
           writer.write(result)
           extracted_count += 1
@@ -480,6 +482,8 @@ module Wp2txt
 
         pages.each do |page|
           article = Article.new(page[:text], page[:title], !config[:marker])
+          article.page_id = page[:id]
+          article.revision_id = page[:revision_id]
           result = format_article(article, config)
           writer.write(result)
           extracted_count += 1
@@ -493,6 +497,8 @@ module Wp2txt
 
           if page
             article = Article.new(page[:text], page[:title], !config[:marker])
+            article.page_id = page[:id]
+            article.revision_id = page[:revision_id]
             result = format_article(article, config)
             writer.write(result)
             extracted_count += 1

data/lib/wp2txt/formatter.rb

@@ -14,6 +14,24 @@ module Wp2txt
 
     # Format article based on configuration and output format
     def format_article(article, config)
+      with_page_ids(format_article_body(article, config), article)
+    end
+
+    # JSON records carry the dump's page and revision IDs right after the title,
+    # when the reader supplied them, so a record can be traced back to its source.
+    def with_page_ids(result, article)
+      return result unless result.is_a?(Hash) && (article.page_id || article.revision_id)
+
+      result.each_with_object({}) do |(key, value), out|
+        out[key] = value
+        next unless key == "title"
+
+        out["page_id"] = article.page_id
+        out["revision_id"] = article.revision_id
+      end
+    end
+
+    def format_article_body(article, config)
       # Store original title for magic word expansion in content
       original_title = article.title.dup
       article.title = format_wiki(article.title, config)

data/lib/wp2txt/multistream.rb

@@ -95,6 +95,11 @@ module Wp2txt
       @early_terminated == true
     end
 
+    # Where the stream holding the last found article ends. Scanning stops as
+    # soon as every target is found, so without this the reader cannot tell how
+    # far that stream extends and would read the dump to its end.
+    attr_reader :stream_end_offset
+
     def find_by_title(title)
       @entries_by_title[title]
     end
@@ -179,6 +184,14 @@ module Wp2txt
       end
     end
 
+    def next_stream_offset_after(io, offset)
+      io.each_line do |line|
+        next_offset = line.split(":", 2).first.to_i
+        return next_offset if next_offset > offset
+      end
+      nil
+    end
+
     def parse_index_stream(io)
       count = 0
       io.each_line do |line|
@@ -205,6 +218,7 @@ module Wp2txt
           @found_targets << title if @target_titles.include?(title)
           if @found_targets.size == @target_titles.size
             @early_terminated = true
+            @stream_end_offset = next_stream_offset_after(io, offset)
             print "\r  Found all #{@target_titles.size} target articles" if @show_progress
             puts if @show_progress
             break
@@ -223,6 +237,10 @@ module Wp2txt
 
   # Reads articles from multistream bz2 files
   class MultistreamReader
+    # A single multistream bz2 stream holds about 100 pages; anything this large
+    # past the last known offset is several streams, not one.
+    MAX_TAIL_STREAM_BYTES = 64 * 1024 * 1024
+
     attr_reader :multistream_path, :index
 
     # Initialize reader with multistream file and index
@@ -357,7 +375,14 @@ module Wp2txt
         if next_offset
           compressed_data = f.read(next_offset - offset)
         else
-          # Last stream - read to end
+          # Last stream of the dump: the rest of the file is that one stream.
+          # A large remainder means the end was never recorded, and reading it
+          # in one call fails outright on some platforms (EINVAL on macOS).
+          remaining = File.size(@multistream_path) - offset
+          if remaining > MAX_TAIL_STREAM_BYTES
+            raise Wp2txt::Error, "cannot locate the end of the stream at offset #{offset} " \
+                                 "(#{remaining} bytes to end of file); the index may be incomplete"
+          end
           compressed_data = f.read
         end
 
@@ -372,7 +397,9 @@ module Wp2txt
       # here would silently read gigabytes to EOF, so fail fast instead
       raise "Stream offset #{current_offset} not found in index (#{offsets.size} streams known)" unless idx
 
-      offsets[idx + 1]
+      return offsets[idx + 1] if idx + 1 < offsets.size
+
+      @index.respond_to?(:stream_end_offset) ? @index.stream_end_offset : nil
     end
 
     def decompress_bz2(data)
@@ -394,6 +421,7 @@ module Wp2txt
           return {
             title: page_title,
             id: page_node.at_xpath("id")&.text&.to_i,
+            revision_id: page_node.at_xpath("revision/id")&.text&.to_i,
             text: page_node.at_xpath(".//text")&.text || ""
           }
         end
@@ -409,6 +437,7 @@ module Wp2txt
         page = {
           title: page_node.at_xpath("title")&.text,
           id: page_node.at_xpath("id")&.text&.to_i,
+          revision_id: page_node.at_xpath("revision/id")&.text&.to_i,
           text: page_node.at_xpath(".//text")&.text || ""
         }
         yield page if page[:title]

data/lib/wp2txt/output_writer.rb

@@ -100,6 +100,14 @@ module Wp2txt
       raise Wp2txt::FileIOError, "Write failed: #{e.message}"
     end
 
+    # Push buffered output to the file. Call before forking: a child process
+    # inherits the buffer and writes it out again when it exits.
+    def flush
+      @mutex.synchronize do
+        @current_file.flush if @current_file && !@current_file.closed?
+      end
+    end
+
     # Close current file and finalize
     def close
       @mutex.synchronize do

data/lib/wp2txt/stream_processor.rb

@@ -64,20 +64,28 @@ module Wp2txt
 
     # Iterate over each page in the input
     # Yields [title, text] for each page
-    def each_page
-      return enum_for(:each_page) unless block_given?
+    # Yields title and text of each article. With with_ids: true, also yields
+    # { page_id:, revision_id: } taken from the dump.
+    def each_page(with_ids: false, &block)
+      return enum_for(:each_page, with_ids: with_ids) unless block
+
+      emit = if with_ids
+               ->(title, text, ids) { block.call(title, text, ids) }
+             else
+               ->(title, text, _ids) { block.call(title, text) }
+             end
 
       if File.directory?(@input_path)
         # Process XML files in directory
         Dir.glob(File.join(@input_path, "*.xml")).sort.each do |xml_file|
-          process_xml_file(xml_file) { |title, text| yield title, text }
+          process_xml_file(xml_file) { |title, text, ids| emit.call(title, text, ids) }
         end
       elsif @input_path.end_with?(".bz2")
         # Process bz2 compressed file with streaming
-        process_bz2_stream { |title, text| yield title, text }
+        process_bz2_stream { |title, text, ids| emit.call(title, text, ids) }
       elsif @input_path.end_with?(".xml")
         # Process single XML file
-        process_xml_file(@input_path) { |title, text| yield title, text }
+        process_xml_file(@input_path) { |title, text, ids| emit.call(title, text, ids) }
       else
         raise ArgumentError, "Unsupported input format: #{@input_path}"
       end
@@ -162,20 +170,32 @@ module Wp2txt
     def fill_buffer
       chunk = @file_pointer.read(@buffer_size)
       unless chunk
-        @buffer << @pending_bytes.to_s.dup.force_encoding(Encoding::UTF_8).scrub("")
-        @pending_bytes = +"".b
+        unless @pending_bytes.to_s.empty?
+          raise Wp2txt::EncodingError,
+                "input ends in the middle of a UTF-8 character (byte #{@bytes_read}): #{@input_path}"
+        end
         return false
       end
 
-      @bytes_read += chunk.bytesize
-      bytes = @pending_bytes.to_s.b + chunk.b
-      # Retain a trailing UTF-8 sequence until the next read. Only complete
-      # chunks are scrubbed, so valid characters split by read are preserved.
+      carried = @pending_bytes.to_s.b
+      start = @bytes_read - carried.bytesize # stream position of the first carried byte
+      bytes = carried + chunk.b
+      # A read can end partway through a multi-byte character; carry those
+      # bytes into the next read rather than judging half a character.
       tail = bytes[/[\xC2-\xF4][\x80-\xBF]{0,2}\z/n]
       width = tail && (tail.getbyte(0) < 0xE0 ? 2 : tail.getbyte(0) < 0xF0 ? 3 : 4)
       @pending_bytes = tail && tail.bytesize < width ? tail : +"".b
-      bytes = bytes.byteslice(0, bytes.bytesize - @pending_bytes.bytesize)
-      @buffer << bytes.force_encoding(Encoding::UTF_8).scrub("")
+      bytes = bytes.byteslice(0, bytes.bytesize - @pending_bytes.bytesize).force_encoding(Encoding::UTF_8)
+      # Anything still invalid is corrupt input. Dropping it would change the
+      # text without a trace, so stop and say where it is.
+      unless bytes.valid_encoding?
+        bad = bytes.each_char.find_index { |c| !c.valid_encoding? }.to_i
+        offset = start + bytes[0, bad].bytesize
+        raise Wp2txt::EncodingError,
+              "invalid UTF-8 near decompressed byte #{offset}: #{@input_path}"
+      end
+      @bytes_read += chunk.bytesize
+      @buffer << bytes
 
       # Adaptive buffer adjustment: if memory is low, reduce buffer size
       if @adaptive_buffer && MemoryMonitor.memory_low?
@@ -252,7 +272,7 @@ module Wp2txt
       end
 
       @pages_processed += 1
-      [title, text]
+      [title, text, Wp2txt.page_ids(page_xml)]
     rescue Nokogiri::XML::SyntaxError
       # Skip malformed XML
       nil

data/lib/wp2txt/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Wp2txt
-  VERSION = "2.3.3"
+  VERSION = "2.3.4"
 end

data/lib/wp2txt.rb

@@ -269,12 +269,14 @@ module Wp2txt
         end
         page << line if inside_page
       end
-      if page.empty?
-        false
-      else
-        page.force_encoding("utf-8")
+      return false if page.empty?
+
+      page.force_encoding("utf-8")
+      # Corrupt input must not be cleaned into different text without a trace
+      unless page.valid_encoding?
+        title = page[%r{<title>([^<]*)</title>}, 1]&.scrub("?")
+        raise Wp2txt::EncodingError, "invalid UTF-8 in page #{title.inspect} of #{@input_file}"
       end
-    rescue ::Encoding::InvalidByteSequenceError, ::Encoding::UndefinedConversionError
       page
     end
 

data/spec/output_integrity_spec.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+require "open3"
+require "json"
+require "tmpdir"
+require "parallel"
+require_relative "support/multistream_fixture"
+require "wp2txt/multistream"
+require "wp2txt/output_writer"
+require "wp2txt/stream_processor"
+
+RSpec.describe "output integrity" do
+  include MultistreamFixture
+
+  around do |example|
+    Dir.mktmpdir("wp2txt-integrity-") do |dir|
+      @dir = dir
+      example.run
+    end
+  end
+
+  let(:cli) { File.expand_path("../bin/wp2txt", __dir__) }
+  let(:lib) { File.expand_path("../lib", __dir__) }
+
+  def run_cli(*args)
+    Open3.capture3(RbConfig.ruby, "-I", lib, cli, *args)
+  end
+
+  def json_records(dir)
+    Dir[File.join(dir, "*")].flat_map { |f| File.readlines(f) }.map { |line| JSON.parse(line) }
+  end
+
+  def write_pages(path, count)
+    body = (1..count).map { |i| page_xml(id: i, ns: 0, title: "記事#{i}", text: "本文#{i}。日本語の文。\n") }.join
+    File.write(path, "<mediawiki>\n#{body}</mediawiki>\n")
+  end
+
+  describe "forked workers" do
+    it "do not write the parent's buffered output again when they exit" do
+      writer = Wp2txt::OutputWriter.new(output_dir: @dir, base_name: "out", format: :text, file_size_mb: 10)
+      writer.write("one line still sitting in the buffer")
+      writer.flush
+      Parallel.map([1, 2, 3], in_processes: 3) { |x| x }
+      files = writer.close
+      expect(File.readlines(files.first).size).to eq(1)
+    end
+
+    it "leave every article exactly once in streaming output across batch boundaries" do
+      input = File.join(@dir, "pages.xml")
+      out = File.join(@dir, "out")
+      Dir.mkdir(out)
+      write_pages(input, 450) # batches of 200 with -n 2
+      _stdout, stderr, status = run_cli("-i", input, "-o", out, "--format", "json", "-n", "2")
+      expect(status.success?).to be(true), stderr
+
+      titles = json_records(out).map { |r| r["title"] }
+      expect(titles.size).to eq(450)
+      expect(titles.tally.select { |_, n| n > 1 }).to be_empty
+    end
+  end
+
+  describe "--num-procs" do
+    it "uses the requested process count" do
+      input = File.join(@dir, "pages.xml")
+      out = File.join(@dir, "out")
+      Dir.mkdir(out)
+      write_pages(input, 3)
+      stdout, stderr, status = run_cli("-i", input, "-o", out, "--format", "json", "-n", "1")
+      expect(status.success?).to be(true), stderr
+      expect(stdout.gsub(/\e\[[\d;]*m/, "")).to match(/CPU cores:\s*1\b/)
+    end
+  end
+
+  describe "page and revision IDs" do
+    it "appear in JSON records from the streaming path, including summaries" do
+      input = File.join(@dir, "pages.xml")
+      write_pages(input, 3)
+      [[], ["--summary-only"]].each do |extra|
+        out = File.join(@dir, "out#{extra.size}")
+        Dir.mkdir(out)
+        _stdout, stderr, status = run_cli("-i", input, "-o", out, "--format", "json", *extra)
+        expect(status.success?).to be(true), stderr
+        record = json_records(out).find { |r| r["title"] == "記事2" }
+        expect(record).to include("page_id" => 2, "revision_id" => 200)
+        expect(record.keys.first(3)).to eq(%w[title page_id revision_id])
+      end
+    end
+
+    it "appear in JSON records from the default path for bz2 dumps" do
+      dump, = create_fixture(@dir)
+      out = File.join(@dir, "out")
+      Dir.mkdir(out)
+      _stdout, stderr, status = run_cli("-i", dump, "-o", out, "--format", "json")
+      expect(status.success?).to be(true), stderr
+      record = json_records(out).find { |r| r["title"] == "Film B" }
+      expect(record).to include("page_id" => 2, "revision_id" => 200)
+    end
+
+    it "are read from the page header, not from the contributor" do
+      xml = "<page><title>X</title><ns>0</ns><id>12</id><revision><id>34</id>" \
+            "<contributor><id>9</id></contributor><text>t</text></revision></page>"
+      expect(Wp2txt.page_ids(xml)).to eq(page_id: 12, revision_id: 34)
+    end
+
+    it "are offered by StreamProcessor only on request, keeping each_page's shape" do
+      input = File.join(@dir, "pages.xml")
+      write_pages(input, 1)
+      processor = Wp2txt::StreamProcessor.new(input, adaptive_buffer: false)
+      expect(processor.each_page.to_a).to eq([["記事1", "本文1。日本語の文。\n"]])
+      with_ids = Wp2txt::StreamProcessor.new(input, adaptive_buffer: false).each_page(with_ids: true).to_a
+      expect(with_ids.first.last).to eq(page_id: 1, revision_id: 100)
+    end
+  end
+
+  describe "targeted extraction after an early index stop" do
+    it "knows where the last found article's stream ends" do
+      _dump, index_path = create_fixture(@dir)
+      index = Wp2txt::MultistreamIndex.new(index_path, use_cache: false, target_titles: ["Film A"],
+                                                       show_progress: false)
+      expect(index.early_terminated?).to be(true)
+      expect(index.stream_offsets).to eq([0])
+      second_stream = File.readlines(index_path).map { |line| line.split(":").first.to_i }.uniq[1]
+      expect(index.stream_end_offset).to eq(second_stream)
+    end
+
+    it "reads only that stream" do
+      dump, index_path = create_fixture(@dir)
+      index = Wp2txt::MultistreamIndex.new(index_path, use_cache: false, target_titles: ["Film A"],
+                                                       show_progress: false)
+      reader = Wp2txt::MultistreamReader.new(dump, index)
+      stub_const("Wp2txt::MultistreamReader::MAX_TAIL_STREAM_BYTES", 0)
+      expect(reader.extract_article("Film A")).to include(title: "Film A", id: 1, revision_id: 100)
+    end
+
+    it "refuses to read a large remainder in one call when the end is unknown" do
+      dump, index_path = create_fixture(@dir)
+      index = Wp2txt::MultistreamIndex.new(index_path, use_cache: false, target_titles: ["Film A"],
+                                                       show_progress: false)
+      index.instance_variable_set(:@stream_end_offset, nil)
+      reader = Wp2txt::MultistreamReader.new(dump, index)
+      stub_const("Wp2txt::MultistreamReader::MAX_TAIL_STREAM_BYTES", 0)
+      expect { reader.extract_article("Film A") }.to raise_error(Wp2txt::Error, /cannot locate the end/)
+    end
+  end
+end

data/spec/p1_correctness_spec.rb

@@ -40,10 +40,25 @@ RSpec.describe "P1 correctness contracts" do
       end
     end
 
-    it "scrubs invalid bytes and incomplete EOF without losing adjacent valid text" do
-      processor = processor_for("é\xFFあ😀\xE3\x81".b, 1)
-      nil while processor.send(:fill_buffer)
-      expect(processor.instance_variable_get(:@buffer)).to eq("éあ😀")
+    it "stops on an invalid byte and reports where it is" do
+      processor = processor_for("éあ\xFF😀".b, 1)
+      expect { nil while processor.send(:fill_buffer) }
+        .to raise_error(Wp2txt::EncodingError, /byte 5\b/)
+    end
+
+    it "stops when the input ends partway through a character" do
+      processor = processor_for("éあ\xE3\x81".b, 1)
+      expect { nil while processor.send(:fill_buffer) }
+        .to raise_error(Wp2txt::EncodingError, /middle of a UTF-8 character/)
+    end
+
+    it "passes valid text through unchanged at every buffer size" do
+      text = "é\nあ😀終\n" * 7
+      [1, 2, 3, 5, 64].each do |size|
+        processor = processor_for(text.b, size)
+        nil while processor.send(:fill_buffer)
+        expect(processor.instance_variable_get(:@buffer)).to eq(text)
+      end
     end
 
     it "handles empty input" do
@@ -105,7 +120,7 @@ RSpec.describe "P1 correctness contracts" do
     it "retains the historical ns=0 default for missing ns in both parsers" do
       xml = page_xml(id: 1, ns: 0, title: "作品: 東京", text: "本文").sub(/<ns>.*?<\/ns>/, "")
       processor = Wp2txt::StreamProcessor.new("unused.xml", adaptive_buffer: false)
-      expect(processor.send(:parse_page_xml, xml)).to eq(["作品: 東京", "本文"])
+      expect(processor.send(:parse_page_xml, xml)).to eq(["作品: 東京", "本文", { page_id: 1, revision_id: 100 }])
       rows = { pages: [], categories: [], sections: [], hierarchy: [] }
       Wp2txt::MetadataIndexBuilder.scan_page(xml, rows)
       expect(rows[:pages].first[2]).to eq(0)

data/spec/titles_output_path_spec.rb

@@ -36,6 +36,18 @@ RSpec.describe "titles extraction and SQL file output" do
     File.readlines(path).map { |l| JSON.parse(l) }
   end
 
+  describe "extract_corpus across several batches with worker processes" do
+    it "writes each record once (workers must not re-flush the output buffer)" do
+      stub_const("Wp2txt::Corpus::EXTRACT_BATCH_SIZE", 1)
+      out = File.join(@dir, "batched.jsonl")
+      titles = ["Film A", "Film B", "Person X"]
+      result = @corpus.extract_corpus(output_path: out, content: "full", titles: titles, num_processes: 2)
+      records = read_jsonl(out)
+      expect(records.map { |r| r["title"] }.tally).to eq(titles.to_h { |t| [t, 1] })
+      expect(records.size).to eq(result[:records_written])
+    end
+  end
+
   describe "extract_corpus titles:" do
     it "extracts an explicit set with normalization, dedup, and input order" do
       out = File.join(@dir, "t.jsonl")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wp2txt
 version: !ruby/object:Gem::Version
-  version: 2.3.3
+  version: 2.3.4
 platform: ruby
 authors:
 - Yoichiro Hasebe
@@ -284,6 +284,7 @@ files:
 - spec/metadata_index_spec.rb
 - spec/multi_dump_attach_spec.rb
 - spec/multistream_spec.rb
+- spec/output_integrity_spec.rb
 - spec/output_writer_spec.rb
 - spec/p1_correctness_spec.rb
 - spec/parser_functions_spec.rb
@@ -351,6 +352,7 @@ test_files:
 - spec/metadata_index_spec.rb
 - spec/multi_dump_attach_spec.rb
 - spec/multistream_spec.rb
+- spec/output_integrity_spec.rb
 - spec/output_writer_spec.rb
 - spec/p1_correctness_spec.rb
 - spec/parser_functions_spec.rb