wp2txt 2.3.2 → 2.3.4

data/lib/wp2txt/stream_processor.rb

@@ -23,6 +23,7 @@ module Wp2txt
       @input_path = input_path
       @bz2_gem = bz2_gem
       @buffer = +""
+      @pending_bytes = +"".b
       @file_pointer = nil
       @adaptive_buffer = adaptive_buffer
       @buffer_size = adaptive_buffer ? calculate_optimal_buffer_size : DEFAULT_BUFFER_SIZE
@@ -63,20 +64,28 @@ module Wp2txt
 
     # Iterate over each page in the input
     # Yields [title, text] for each page
-    def each_page
-      return enum_for(:each_page) unless block_given?
+    # Yields title and text of each article. With with_ids: true, also yields
+    # { page_id:, revision_id: } taken from the dump.
+    def each_page(with_ids: false, &block)
+      return enum_for(:each_page, with_ids: with_ids) unless block
+
+      emit = if with_ids
+               ->(title, text, ids) { block.call(title, text, ids) }
+             else
+               ->(title, text, _ids) { block.call(title, text) }
+             end
 
       if File.directory?(@input_path)
         # Process XML files in directory
         Dir.glob(File.join(@input_path, "*.xml")).sort.each do |xml_file|
-          process_xml_file(xml_file) { |title, text| yield title, text }
+          process_xml_file(xml_file) { |title, text, ids| emit.call(title, text, ids) }
         end
       elsif @input_path.end_with?(".bz2")
         # Process bz2 compressed file with streaming
-        process_bz2_stream { |title, text| yield title, text }
+        process_bz2_stream { |title, text, ids| emit.call(title, text, ids) }
       elsif @input_path.end_with?(".xml")
         # Process single XML file
-        process_xml_file(@input_path) { |title, text| yield title, text }
+        process_xml_file(@input_path) { |title, text, ids| emit.call(title, text, ids) }
       else
         raise ArgumentError, "Unsupported input format: #{@input_path}"
       end
@@ -99,7 +108,8 @@ module Wp2txt
     # Process a single XML file
     def process_xml_file(xml_file)
       @buffer = +""
-      @file_pointer = File.open(xml_file, "r:UTF-8")
+      @pending_bytes = +"".b
+      @file_pointer = File.open(xml_file, "rb")
 
       while (page = extract_next_page)
         result = parse_page_xml(page)
@@ -120,6 +130,7 @@ module Wp2txt
       end
 
       @buffer = +""
+      @pending_bytes = +"".b
       @file_pointer = open_bz2_stream
 
       while (page = extract_next_page)
@@ -158,14 +169,33 @@ module Wp2txt
     # Fill buffer from file pointer
     def fill_buffer
       chunk = @file_pointer.read(@buffer_size)
-      return false unless chunk
+      unless chunk
+        unless @pending_bytes.to_s.empty?
+          raise Wp2txt::EncodingError,
+                "input ends in the middle of a UTF-8 character (byte #{@bytes_read}): #{@input_path}"
+        end
+        return false
+      end
 
+      carried = @pending_bytes.to_s.b
+      start = @bytes_read - carried.bytesize # stream position of the first carried byte
+      bytes = carried + chunk.b
+      # A read can end partway through a multi-byte character; carry those
+      # bytes into the next read rather than judging half a character.
+      tail = bytes[/[\xC2-\xF4][\x80-\xBF]{0,2}\z/n]
+      width = tail && (tail.getbyte(0) < 0xE0 ? 2 : tail.getbyte(0) < 0xF0 ? 3 : 4)
+      @pending_bytes = tail && tail.bytesize < width ? tail : +"".b
+      bytes = bytes.byteslice(0, bytes.bytesize - @pending_bytes.bytesize).force_encoding(Encoding::UTF_8)
+      # Anything still invalid is corrupt input. Dropping it would change the
+      # text without a trace, so stop and say where it is.
+      unless bytes.valid_encoding?
+        bad = bytes.each_char.find_index { |c| !c.valid_encoding? }.to_i
+        offset = start + bytes[0, bad].bytesize
+        raise Wp2txt::EncodingError,
+              "invalid UTF-8 near decompressed byte #{offset}: #{@input_path}"
+      end
       @bytes_read += chunk.bytesize
-
-      # Handle encoding for bz2 streams
-      chunk = chunk.force_encoding("UTF-8")
-      chunk = chunk.scrub("")
-      @buffer << chunk
+      @buffer << bytes
 
       # Adaptive buffer adjustment: if memory is low, reduce buffer size
       if @adaptive_buffer && MemoryMonitor.memory_low?
@@ -223,8 +253,8 @@ module Wp2txt
       return nil unless title_node
 
       title = title_node.content
-      # Skip special pages (containing colon in title like "Wikipedia:", "File:", etc.)
-      return nil if title.include?(":")
+      namespace = title_node.parent.at_css("ns")&.text
+      return nil unless Wp2txt.namespace_id(namespace).zero?
 
       text = text_node.content
 
@@ -242,7 +272,7 @@ module Wp2txt
       end
 
       @pages_processed += 1
-      [title, text]
+      [title, text, Wp2txt.page_ids(page_xml)]
     rescue Nokogiri::XML::SyntaxError
       # Skip malformed XML
       nil

data/lib/wp2txt/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Wp2txt
-  VERSION = "2.3.2"
+  VERSION = "2.3.4"
 end

data/lib/wp2txt.rb

@@ -269,12 +269,14 @@ module Wp2txt
         end
         page << line if inside_page
       end
-      if page.empty?
-        false
-      else
-        page.force_encoding("utf-8")
+      return false if page.empty?
+
+      page.force_encoding("utf-8")
+      # Corrupt input must not be cleaned into different text without a trace
+      unless page.valid_encoding?
+        title = page[%r{<title>([^<]*)</title>}, 1]&.scrub("?")
+        raise Wp2txt::EncodingError, "invalid UTF-8 in page #{title.inspect} of #{@input_file}"
       end
-    rescue ::Encoding::InvalidByteSequenceError, ::Encoding::UndefinedConversionError
       page
     end
 

data/spec/fts_index_spec.rb

@@ -171,6 +171,46 @@ RSpec.describe "Wp2txt Full-Text Search" do
         end
       end
 
+      ["", "東", "東京", "é", "😀"].each do |query|
+        it "rejects the short phrase #{query.inspect} with a stable code" do
+          expect { @fts.search(query, count: "exact") }.to raise_error(Wp2txt::FtsIndex::ShortQueryError) { |error|
+            expect(error.code).to eq("query_too_short")
+          }
+        end
+      end
+
+      it "leaves raw FTS syntax to SQLite even for short expressions" do
+        expect(@fts.search('"東"', mode: "query", count: "exact")[:total]).to eq(0)
+        expect(@fts.search("\xFF".b, mode: "query", count: "exact")[:total]).to eq(0)
+      end
+
+      it "accepts three Unicode characters" do
+        expect(@fts.search("東京都", count: "exact")[:total]).to eq(0)
+      end
+
+      it "returns a coded MCP tool error instead of a zero-result success" do
+        @fts.close
+        requests = [
+          { jsonrpc: "2.0", id: 1, method: "initialize", params: {
+            protocolVersion: "2025-03-26", capabilities: {}, clientInfo: { name: "regression", version: "1" }
+          } },
+          { jsonrpc: "2.0", method: "notifications/initialized" },
+          { jsonrpc: "2.0", id: 2, method: "tools/call", params: {
+            name: "search_text", arguments: { query: "東京", count: "exact" }
+          } }
+        ]
+        stdout, stderr, status = Open3.capture3(RbConfig.ruby,
+          File.expand_path("../bin/wp2txt-mcp", __dir__), "--input", @multistream_path,
+          "--cache-dir", File.dirname(@multistream_path),
+          stdin_data: requests.map { |request| JSON.generate(request) }.join("\n") + "\n")
+        expect(status.success?).to be(true), stderr
+        response = stdout.lines.map { |line| JSON.parse(line) }.find { |message| message["id"] == 2 }
+        expect(response.dig("result", "isError")).to be true
+        payload = JSON.parse(response.dig("result", "content", 0, "text"))
+        expect(payload["code"]).to eq("query_too_short")
+        expect(payload).not_to have_key("total")
+      end
+
       it "matches substrings of three or more characters" do
         result = @fts.search("tory", count: "exact")
         expect(result[:total]).to eq(2)

data/spec/output_integrity_spec.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+require "open3"
+require "json"
+require "tmpdir"
+require "parallel"
+require_relative "support/multistream_fixture"
+require "wp2txt/multistream"
+require "wp2txt/output_writer"
+require "wp2txt/stream_processor"
+
+RSpec.describe "output integrity" do
+  include MultistreamFixture
+
+  around do |example|
+    Dir.mktmpdir("wp2txt-integrity-") do |dir|
+      @dir = dir
+      example.run
+    end
+  end
+
+  let(:cli) { File.expand_path("../bin/wp2txt", __dir__) }
+  let(:lib) { File.expand_path("../lib", __dir__) }
+
+  def run_cli(*args)
+    Open3.capture3(RbConfig.ruby, "-I", lib, cli, *args)
+  end
+
+  def json_records(dir)
+    Dir[File.join(dir, "*")].flat_map { |f| File.readlines(f) }.map { |line| JSON.parse(line) }
+  end
+
+  def write_pages(path, count)
+    body = (1..count).map { |i| page_xml(id: i, ns: 0, title: "記事#{i}", text: "本文#{i}。日本語の文。\n") }.join
+    File.write(path, "<mediawiki>\n#{body}</mediawiki>\n")
+  end
+
+  describe "forked workers" do
+    it "do not write the parent's buffered output again when they exit" do
+      writer = Wp2txt::OutputWriter.new(output_dir: @dir, base_name: "out", format: :text, file_size_mb: 10)
+      writer.write("one line still sitting in the buffer")
+      writer.flush
+      Parallel.map([1, 2, 3], in_processes: 3) { |x| x }
+      files = writer.close
+      expect(File.readlines(files.first).size).to eq(1)
+    end
+
+    it "leave every article exactly once in streaming output across batch boundaries" do
+      input = File.join(@dir, "pages.xml")
+      out = File.join(@dir, "out")
+      Dir.mkdir(out)
+      write_pages(input, 450) # batches of 200 with -n 2
+      _stdout, stderr, status = run_cli("-i", input, "-o", out, "--format", "json", "-n", "2")
+      expect(status.success?).to be(true), stderr
+
+      titles = json_records(out).map { |r| r["title"] }
+      expect(titles.size).to eq(450)
+      expect(titles.tally.select { |_, n| n > 1 }).to be_empty
+    end
+  end
+
+  describe "--num-procs" do
+    it "uses the requested process count" do
+      input = File.join(@dir, "pages.xml")
+      out = File.join(@dir, "out")
+      Dir.mkdir(out)
+      write_pages(input, 3)
+      stdout, stderr, status = run_cli("-i", input, "-o", out, "--format", "json", "-n", "1")
+      expect(status.success?).to be(true), stderr
+      expect(stdout.gsub(/\e\[[\d;]*m/, "")).to match(/CPU cores:\s*1\b/)
+    end
+  end
+
+  describe "page and revision IDs" do
+    it "appear in JSON records from the streaming path, including summaries" do
+      input = File.join(@dir, "pages.xml")
+      write_pages(input, 3)
+      [[], ["--summary-only"]].each do |extra|
+        out = File.join(@dir, "out#{extra.size}")
+        Dir.mkdir(out)
+        _stdout, stderr, status = run_cli("-i", input, "-o", out, "--format", "json", *extra)
+        expect(status.success?).to be(true), stderr
+        record = json_records(out).find { |r| r["title"] == "記事2" }
+        expect(record).to include("page_id" => 2, "revision_id" => 200)
+        expect(record.keys.first(3)).to eq(%w[title page_id revision_id])
+      end
+    end
+
+    it "appear in JSON records from the default path for bz2 dumps" do
+      dump, = create_fixture(@dir)
+      out = File.join(@dir, "out")
+      Dir.mkdir(out)
+      _stdout, stderr, status = run_cli("-i", dump, "-o", out, "--format", "json")
+      expect(status.success?).to be(true), stderr
+      record = json_records(out).find { |r| r["title"] == "Film B" }
+      expect(record).to include("page_id" => 2, "revision_id" => 200)
+    end
+
+    it "are read from the page header, not from the contributor" do
+      xml = "<page><title>X</title><ns>0</ns><id>12</id><revision><id>34</id>" \
+            "<contributor><id>9</id></contributor><text>t</text></revision></page>"
+      expect(Wp2txt.page_ids(xml)).to eq(page_id: 12, revision_id: 34)
+    end
+
+    it "are offered by StreamProcessor only on request, keeping each_page's shape" do
+      input = File.join(@dir, "pages.xml")
+      write_pages(input, 1)
+      processor = Wp2txt::StreamProcessor.new(input, adaptive_buffer: false)
+      expect(processor.each_page.to_a).to eq([["記事1", "本文1。日本語の文。\n"]])
+      with_ids = Wp2txt::StreamProcessor.new(input, adaptive_buffer: false).each_page(with_ids: true).to_a
+      expect(with_ids.first.last).to eq(page_id: 1, revision_id: 100)
+    end
+  end
+
+  describe "targeted extraction after an early index stop" do
+    it "knows where the last found article's stream ends" do
+      _dump, index_path = create_fixture(@dir)
+      index = Wp2txt::MultistreamIndex.new(index_path, use_cache: false, target_titles: ["Film A"],
+                                                       show_progress: false)
+      expect(index.early_terminated?).to be(true)
+      expect(index.stream_offsets).to eq([0])
+      second_stream = File.readlines(index_path).map { |line| line.split(":").first.to_i }.uniq[1]
+      expect(index.stream_end_offset).to eq(second_stream)
+    end
+
+    it "reads only that stream" do
+      dump, index_path = create_fixture(@dir)
+      index = Wp2txt::MultistreamIndex.new(index_path, use_cache: false, target_titles: ["Film A"],
+                                                       show_progress: false)
+      reader = Wp2txt::MultistreamReader.new(dump, index)
+      stub_const("Wp2txt::MultistreamReader::MAX_TAIL_STREAM_BYTES", 0)
+      expect(reader.extract_article("Film A")).to include(title: "Film A", id: 1, revision_id: 100)
+    end
+
+    it "refuses to read a large remainder in one call when the end is unknown" do
+      dump, index_path = create_fixture(@dir)
+      index = Wp2txt::MultistreamIndex.new(index_path, use_cache: false, target_titles: ["Film A"],
+                                                       show_progress: false)
+      index.instance_variable_set(:@stream_end_offset, nil)
+      reader = Wp2txt::MultistreamReader.new(dump, index)
+      stub_const("Wp2txt::MultistreamReader::MAX_TAIL_STREAM_BYTES", 0)
+      expect { reader.extract_article("Film A") }.to raise_error(Wp2txt::Error, /cannot locate the end/)
+    end
+  end
+end