wp2txt 2.3.2 → 2.3.4

data/spec/p1_correctness_spec.rb

@@ -0,0 +1,372 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+require "tmpdir"
+require "timeout"
+require "open3"
+require "rake"
+require_relative "../lib/wp2txt/stream_processor"
+require_relative "../lib/wp2txt/corpus"
+require_relative "../lib/wp2txt/corpus_jobs"
+require_relative "../lib/wp2txt/output_path"
+require_relative "support/multistream_fixture"
+
+RSpec.describe "P1 correctness contracts" do
+  include MultistreamFixture
+
+  around do |example|
+    Dir.mktmpdir("wp2txt-p1-") do |dir|
+      @dir = dir
+      example.run
+    end
+  end
+
+  def processor_for(bytes, size)
+    processor = Wp2txt::StreamProcessor.new("unused.xml", adaptive_buffer: false)
+    processor.instance_variable_set(:@file_pointer, StringIO.new(bytes.b))
+    processor.instance_variable_set(:@buffer_size, size)
+    processor
+  end
+
+  describe "stream decoding" do
+    %w[é あ 😀].each do |character|
+      (1...character.bytesize).each do |boundary|
+        it "preserves #{character.inspect} split after byte #{boundary}" do
+          bytes = "x" * (8 - boundary) + character + "終"
+          processor = processor_for(bytes, 8)
+          nil while processor.send(:fill_buffer)
+          expect(processor.instance_variable_get(:@buffer)).to eq(bytes)
+        end
+      end
+    end
+
+    it "stops on an invalid byte and reports where it is" do
+      processor = processor_for("éあ\xFF😀".b, 1)
+      expect { nil while processor.send(:fill_buffer) }
+        .to raise_error(Wp2txt::EncodingError, /byte 5\b/)
+    end
+
+    it "stops when the input ends partway through a character" do
+      processor = processor_for("éあ\xE3\x81".b, 1)
+      expect { nil while processor.send(:fill_buffer) }
+        .to raise_error(Wp2txt::EncodingError, /middle of a UTF-8 character/)
+    end
+
+    it "passes valid text through unchanged at every buffer size" do
+      text = "é\nあ😀終\n" * 7
+      [1, 2, 3, 5, 64].each do |size|
+        processor = processor_for(text.b, size)
+        nil while processor.send(:fill_buffer)
+        expect(processor.instance_variable_get(:@buffer)).to eq(text)
+      end
+    end
+
+    it "handles empty input" do
+      processor = processor_for("", 1)
+      expect(processor.send(:fill_buffer)).to be false
+      expect(processor.instance_variable_get(:@buffer)).to eq("")
+    end
+
+    it "preserves multibyte XML content through one-byte reads" do
+      path = File.join(@dir, "boundary.xml")
+      File.binwrite(path, page_xml(id: 1, ns: 0, title: "作品: é😀", text: "あé😀"))
+      processor = Wp2txt::StreamProcessor.new(path, adaptive_buffer: false)
+      processor.instance_variable_set(:@buffer_size, 1)
+      expect(processor.each_page.to_a).to eq([["作品: é😀", "あé😀"]])
+    end
+  end
+
+  describe "namespace parity" do
+    [["作品: 東京", 0, true], ["No punctuation", 4, false], ["", 6, false]].each do |title, ns, included|
+      it "uses ns=#{ns} for #{title.inspect}" do
+        xml = page_xml(id: 1, ns: ns, title: title, text: "日本語")
+        processor = Wp2txt::StreamProcessor.new("unused.xml", adaptive_buffer: false)
+        expect(!processor.send(:parse_page_xml, xml).nil?).to eq(included)
+        rows = { pages: [], categories: [], sections: [], hierarchy: [] }
+        Wp2txt::MetadataIndexBuilder.scan_page(xml, rows)
+        expect(rows[:pages].first[2]).to eq(ns) unless title.empty?
+      end
+    end
+
+    it "uses namespace elements in the turbo extraction and section-stat paths" do
+      input = File.join(@dir, "pages.xml")
+      output = File.join(@dir, "output.jsonl")
+      File.write(input, [
+        page_xml(id: 1, ns: 0, title: "作品: 東京", text: "本文\n== Allowed ==\nあ"),
+        page_xml(id: 2, ns: 4, title: "No punctuation", text: "本文\n== Hidden ==\nあ"),
+        page_xml(id: 3, ns: 0, title: "Empty", text: "")
+      ].join)
+      script = <<~'CODE'
+        path, input, output = ARGV
+        source = File.read(path).split("\n# Handle Ctrl+C gracefully").first
+        eval(source, TOPLEVEL_BINDING, path)
+        require File.expand_path("../lib/wp2txt/corpus", File.dirname(path))
+        app = WpApp.new
+        count = app.send(:process_xml_file_and_write, input, output,
+                         Wp2txt::Corpus::RENDER_CONFIG.merge(format: :json), false, :json)
+        stats = app.send(:process_xml_file_for_stats, input)
+        puts JSON.generate(count: count, stats: stats)
+      CODE
+      stdout, stderr, status = Open3.capture3(RbConfig.ruby, "-e", script,
+        File.expand_path("../bin/wp2txt", __dir__), input, output)
+      expect(status.success?).to be(true), stderr
+      result = JSON.parse(stdout)
+      expect(result["count"]).to eq(1)
+      expect(result["stats"].to_json).to include("Allowed")
+      expect(result["stats"].to_json).not_to include("Hidden")
+      expect(File.read(output)).to include("作品: 東京")
+    end
+
+    it "retains the historical ns=0 default for missing ns in both parsers" do
+      xml = page_xml(id: 1, ns: 0, title: "作品: 東京", text: "本文").sub(/<ns>.*?<\/ns>/, "")
+      processor = Wp2txt::StreamProcessor.new("unused.xml", adaptive_buffer: false)
+      expect(processor.send(:parse_page_xml, xml)).to eq(["作品: 東京", "本文", { page_id: 1, revision_id: 100 }])
+      rows = { pages: [], categories: [], sections: [], hierarchy: [] }
+      Wp2txt::MetadataIndexBuilder.scan_page(xml, rows)
+      expect(rows[:pages].first[2]).to eq(0)
+    end
+  end
+
+  describe "SQL rows and cells" do
+    around do |example|
+      @corpus = Wp2txt::Corpus.allocate
+      @db = SQLite3::Database.new(":memory:")
+      example.run
+    ensure
+      @db.close
+    end
+
+    it "reserves original and generated column names without losing values" do
+      path = File.join(@dir, "q.jsonl")
+      result = @corpus.send(:run_sql_file_on, @db,
+                            'SELECT 1 AS x, 2 AS x, 3 AS x_2, 4 AS x, 5 AS "日本", 6 AS "日本", 7 AS ""', path)
+      record = JSON.parse(File.read(path))
+      expect(record).to eq("x" => 1, "x_3" => 2, "x_2" => 3, "x_4" => 4,
+                           "日本" => 5, "日本_2" => 6, "" => 7)
+      expect(result[:column_mapping][1]).to eq(ordinal: 1, original_name: "x", output_name: "x_3")
+    end
+
+    [0, 1, 2].each do |rows|
+      it "reports truncation only when more than the limit exists (#{rows} rows)" do
+        result = @corpus.send(:run_sql_on, @db, "SELECT 1 UNION ALL SELECT 2 LIMIT #{rows}", 1)
+        expect(result[:truncated]).to eq(rows > 1)
+        expect(result[:row_count]).to eq([rows, 1].min)
+      end
+    end
+
+    it "does not silently sanitize invalid UTF-8 cells while clipping" do
+      @db.execute("CREATE TABLE cells (value BLOB)")
+      @db.execute("INSERT INTO cells VALUES (?)", [SQLite3::Blob.new("\xFF".b + "a" * 70_000)])
+      expect do
+        @corpus.send(:run_sql_file_on, @db, "SELECT value FROM cells", File.join(@dir, "invalid.jsonl"))
+      end.to raise_error(JSON::GeneratorError)
+    end
+
+    it "clips by bytes including the ellipsis without splitting UTF-8" do
+      @db.execute("CREATE TABLE cells (value TEXT)")
+      values = ["あ" * 30_000, "😀" * 20_000, "é" * 40_000, "", "a" * 65_536]
+      values.each { |value| @db.execute("INSERT INTO cells VALUES (?)", [value]) }
+      path = File.join(@dir, "cells.jsonl")
+      result = @corpus.send(:run_sql_file_on, @db, "SELECT value FROM cells", path)
+      output = File.readlines(path).map { |line| JSON.parse(line)["value"] }
+      expect(result[:cells_clipped]).to eq(3)
+      expect(output.all?(&:valid_encoding?)).to be true
+      expect(output.map(&:bytesize).max).to be <= 65_536
+      expect(output.first(3).all? { |value| value.end_with?("…") }).to be true
+      expect(output.last(2)).to eq(values.last(2))
+    end
+  end
+
+  describe "exclusive staged output" do
+    let(:out) { File.join(@dir, "成果.jsonl") }
+
+    def publish(path, overwrite: false)
+      Wp2txt::OutputPath.write_pair(path, overwrite: overwrite) do |data, meta|
+        File.write(data, "本文")
+        File.write(meta, "{}")
+      end
+    end
+
+    it "rejects symlink directories, output files, dangling links, and sidecars even with overwrite" do
+      target = File.join(@dir, "target")
+      Dir.mkdir(target)
+      link = File.join(@dir, "link")
+      File.symlink(target, link)
+      expect { publish(File.join(link, "out"), overwrite: true) }.to raise_error(ArgumentError, /symbolic/)
+      [out, "#{out}.meta.json"].each do |destination|
+        File.symlink(File.join(target, "missing"), destination)
+        expect { publish(out, overwrite: true) }.to raise_error(ArgumentError, /symbolic/)
+        expect { Wp2txt::OutputPath.confine(out, @dir, overwrite: true) }.to raise_error(ArgumentError, /symbolic/)
+        File.unlink(destination)
+      end
+      expect(Dir.children(target)).to be_empty
+    end
+
+    it "rejects an existing sidecar without creating the output" do
+      File.write("#{out}.meta.json", "old")
+      expect { publish(out) }.to raise_error(ArgumentError, /already exists/)
+      expect(File.exist?(out)).to be false
+      expect(File.read("#{out}.meta.json")).to eq("old")
+    end
+
+    it "uses EXCL even if a file arrives after validation" do
+      allow(Wp2txt::OutputPath).to receive(:validate_pair!).and_wrap_original do |method, *args, **kwargs|
+        method.call(*args, **kwargs)
+        File.write(out, "competing writer")
+      end
+      expect { publish(out) }.to raise_error(ArgumentError, /already exists/)
+      expect(File.read(out)).to eq("competing writer")
+      expect(File.exist?("#{out}.meta.json")).to be false
+    end
+
+    it "cleans its own reservation when sidecar reservation loses a race" do
+      allow(Wp2txt::OutputPath).to receive(:validate_pair!).and_wrap_original do |method, *args, **kwargs|
+        method.call(*args, **kwargs)
+        File.write("#{out}.meta.json", "competing writer")
+      end
+      expect { publish(out) }.to raise_error(ArgumentError, /already exists/)
+      expect(File.exist?(out)).to be false
+      expect(File.read("#{out}.meta.json")).to eq("competing writer")
+    end
+
+    it "publishes both files and leaves no temporary files" do
+      publish(out)
+      expect(File.read(out)).to eq("本文")
+      expect(File.read("#{out}.meta.json")).to eq("{}")
+      expect(Dir.children(@dir).sort).to eq([File.basename(out), File.basename(out) + ".meta.json"].sort)
+    end
+
+    it "preserves old files if staging fails during an overwrite" do
+      File.write(out, "old data")
+      File.write("#{out}.meta.json", "old meta")
+      expect do
+        Wp2txt::OutputPath.write_pair(out, overwrite: true) do |data, _meta|
+          File.write(data, "incomplete")
+          raise "failed"
+        end
+      end.to raise_error("failed")
+      expect(File.read(out)).to eq("old data")
+      expect(File.read("#{out}.meta.json")).to eq("old meta")
+      expect(Dir.glob(File.join(@dir, ".wp2txt-*"))).to be_empty
+      publish(out, overwrite: true)
+      expect(File.read(out)).to eq("本文")
+    end
+
+    it "allows only one concurrent writer without overwrite" do
+      entered = Queue.new
+      release = Queue.new
+      writer = Thread.new do
+        Wp2txt::OutputPath.write_pair(out) do |data, meta|
+          entered << true
+          release.pop
+          File.write(data, "first")
+          File.write(meta, "{}")
+        end
+      end
+      Timeout.timeout(3) { entered.pop }
+      expect { publish(out) }.to raise_error(ArgumentError, /already exists/)
+      release << true
+      writer.value
+      expect(File.read(out)).to eq("first")
+    ensure
+      release << true
+      writer&.join
+    end
+  end
+
+  describe "Parallel finish contract" do
+    [0, 2].each do |workers|
+      it "delivers each payload to finish but retains none with #{workers} workers" do
+        skip "fork unavailable" if workers.positive? && !Process.respond_to?(:fork)
+        payloads = ["日本語", "\xFF".b, ""]
+        received = []
+        returned = Parallel.each(payloads, in_processes: workers,
+                                 finish: ->(_item, _i, value) { received << value }) { |value| "rendered:#{value.bytesize}" }
+        expect(received).to match_array(payloads.map { |value| "rendered:#{value.bytesize}" })
+        expect(Array(returned).flatten.grep(/\Arendered:/)).to be_empty
+      end
+    end
+
+    it "builds both indexes while disabling retained batch results" do
+      dump, index_path = create_fixture(@dir)
+      index = Wp2txt::MultistreamIndex.new(index_path, use_cache: false, show_progress: false)
+      calls = []
+      allow(Parallel).to receive(:each).and_wrap_original do |method, source, options, &block|
+        calls << options
+        method.call(source, options, &block)
+      end
+      meta_path = Wp2txt::MetadataIndex.path_for(dump, cache_dir: @dir)
+      meta = Wp2txt::MetadataIndexBuilder.new(dump, index.stream_offsets, db_path: meta_path, num_processes: 0).build
+      expect(meta.stats[:article_count]).to eq(3)
+      meta.close
+      fts = Wp2txt::FtsIndexBuilder.new(dump, index.stream_offsets,
+        db_path: File.join(@dir, "fts.sqlite3"), meta_db_path: meta_path, num_processes: 0).build
+      expect(fts.search("Story", count: "exact")[:total]).to eq(2)
+      expect(calls.size).to eq(2)
+      expect(calls.all? { |options| options[:finish] && !options.key?(:preserve_results) }).to be true
+      fts.close
+    end
+  end
+
+  describe "job lifecycle" do
+    it "records factory failure and permits a subsequent job" do
+      manager = Wp2txt::CorpusJobManager.new(-> { raise "factory failure" })
+      first = manager.start_extract({})
+      Timeout.timeout(3) { Thread.pass until manager.status(first[:job_id])[:status] == "error" }
+      expect(manager.status(first[:job_id])[:error]).to include("factory failure")
+      second = manager.start_extract({})
+      expect(second[:job_id]).not_to eq(first[:job_id])
+      Timeout.timeout(3) { Thread.pass until manager.status(second[:job_id])[:status] == "error" }
+    end
+
+    it "atomically admits one job from simultaneous callers" do
+      release = Queue.new
+      fake = Object.new
+      fake.define_singleton_method(:extract_corpus) { |**_params| release.pop; {} }
+      fake.define_singleton_method(:close) {}
+      manager = Wp2txt::CorpusJobManager.new(-> { fake })
+      # Yield during state construction, precisely where the old implementation
+      # had released its check lock but had not registered the running job.
+      allow(Time).to receive(:now).and_wrap_original do |method|
+        sleep 0.005
+        method.call
+      end
+      gate = Queue.new
+      callers = Array.new(12) { Thread.new { gate.pop; manager.start_extract({}) } }
+      12.times { gate << true }
+      results = callers.map(&:value)
+      expect(results.count { |result| result[:job_id] }).to eq(1)
+      expect(results.count { |result| result[:error] }).to eq(11)
+      release << true
+      Timeout.timeout(3) { Thread.pass until manager.list.first[:status] == "completed" }
+    ensure
+      release << true
+      callers&.each(&:join)
+    end
+  end
+
+  describe "image verification" do
+    around do |example|
+      previous = Rake.application
+      Rake.application = Rake::Application.new
+      Rake::Task.define_task(:build)
+      suppress_stderr { load File.expand_path("../Rakefile", __dir__) }
+      example.run
+    ensure
+      Rake.application = previous
+    end
+
+    [[false, "Cannot connect to daemon"], [false, "image missing"], [true, "LEAK:/wp2txt/tmp\n"]].each do |success, output|
+      it "fails verification for #{output.strip}" do
+        allow(Open3).to receive(:capture2e).and_return([output, double(success?: success)])
+        expect { suppress_stderr { Rake::Task[:verify_image].invoke("test-image") } }.to raise_error(SystemExit)
+      end
+    end
+
+    it "passes only after a successful clean container inspection" do
+      expect(Open3).to receive(:capture2e).with("docker", "run", "--rm", "test-image", "sh", "-c", kind_of(String))
+        .and_return(["", double(success?: true)])
+      expect { Rake::Task[:verify_image].invoke("test-image") }.to output(/OK:/).to_stdout
+    end
+  end
+end

data/spec/stream_processor_spec.rb

@@ -112,12 +112,14 @@ RSpec.describe Wp2txt::StreamProcessor do
           </page>
           <page>
             <title>Wikipedia:Help</title>
+            <ns>4</ns>
             <revision>
               <text>Help content.</text>
             </revision>
           </page>
           <page>
             <title>File:Image.jpg</title>
+            <ns>6</ns>
             <revision>
               <text>File description.</text>
             </revision>
@@ -131,7 +133,7 @@ RSpec.describe Wp2txt::StreamProcessor do
         File.write(xml_file, xml_content)
       end
 
-      it "skips pages with colon in title (special pages)" do
+      it "skips non-article namespaces" do
         processor = described_class.new(xml_file)
         pages = processor.each_page.to_a
 

data/spec/titles_output_path_spec.rb

@@ -36,6 +36,18 @@ RSpec.describe "titles extraction and SQL file output" do
     File.readlines(path).map { |l| JSON.parse(l) }
   end
 
+  describe "extract_corpus across several batches with worker processes" do
+    it "writes each record once (workers must not re-flush the output buffer)" do
+      stub_const("Wp2txt::Corpus::EXTRACT_BATCH_SIZE", 1)
+      out = File.join(@dir, "batched.jsonl")
+      titles = ["Film A", "Film B", "Person X"]
+      result = @corpus.extract_corpus(output_path: out, content: "full", titles: titles, num_processes: 2)
+      records = read_jsonl(out)
+      expect(records.map { |r| r["title"] }.tally).to eq(titles.to_h { |t| [t, 1] })
+      expect(records.size).to eq(result[:records_written])
+    end
+  end
+
   describe "extract_corpus titles:" do
     it "extracts an explicit set with normalization, dedup, and input order" do
       out = File.join(@dir, "t.jsonl")
@@ -237,6 +249,46 @@ RSpec.describe "titles extraction and SQL file output" do
       expect(File.exist?("#{out}.partial")).to be false
     end
 
+    it "records ordinal to original name mappings in the sidecar for colliding names" do
+      out = File.join(@dir, "names.jsonl")
+      @corpus.query_sql('SELECT 1 AS x, 2 AS x, 3 AS x_2', output_path: out)
+      expect(read_jsonl(out)).to eq([{ "x" => 1, "x_3" => 2, "x_2" => 3 }])
+      meta = JSON.parse(File.read("#{out}.meta.json"))
+      expect(meta["column_mapping"]).to eq([
+        { "ordinal" => 0, "original_name" => "x", "output_name" => "x" },
+        { "ordinal" => 1, "original_name" => "x", "output_name" => "x_3" },
+        { "ordinal" => 2, "original_name" => "x_2", "output_name" => "x_2" }
+      ])
+      expect(meta["cell_byte_limit"]).to eq(65_536)
+    end
+
+    it "refuses sidecar collisions through both public output APIs" do
+      out = File.join(@dir, "sidecar.jsonl")
+      File.write("#{out}.meta.json", "old")
+      expect { @corpus.query_sql("SELECT 1", output_path: out) }.to raise_error(ArgumentError, /already exists/)
+      expect do
+        @corpus.extract_corpus(output_path: out, content: "summary", titles: ["Film A"], num_processes: 0)
+      end.to raise_error(ArgumentError, /already exists/)
+      expect(File.exist?(out)).to be false
+      expect(File.read("#{out}.meta.json")).to eq("old")
+    end
+
+    it "supports explicit overwrite for extraction and cleans up cancelled staging" do
+      out = File.join(@dir, "extract.jsonl")
+      File.write(out, "old")
+      File.write("#{out}.meta.json", "old metadata")
+      expect do
+        @corpus.extract_corpus(output_path: out, content: "summary", titles: ["Film A"],
+                               num_processes: 0, overwrite: true, cancel_check: -> { true })
+      end.to raise_error(Wp2txt::Corpus::Cancelled)
+      expect(File.read(out)).to eq("old")
+      expect(File.read("#{out}.meta.json")).to eq("old metadata")
+      expect(Dir.glob(File.join(@dir, ".wp2txt-*"))).to be_empty
+      @corpus.extract_corpus(output_path: out, content: "summary", titles: ["Film A"],
+                             num_processes: 0, overwrite: true)
+      expect(read_jsonl(out).first["title"]).to eq("Film A")
+    end
+
     it "ignores limit in file-output mode" do
       out = File.join(@dir, "q.jsonl")
       result = @corpus.query_sql("SELECT title FROM pages", output_path: out, limit: 1)
@@ -255,6 +307,14 @@ RSpec.describe "titles extraction and SQL file output" do
       expect(read_jsonl(out)).to eq([{ "one" => 1 }])
     end
 
+    it "does not touch another writer's fixed .partial path" do
+      out = File.join(@dir, "q.jsonl")
+      File.write("#{out}.partial", "another writer")
+      @corpus.query_sql("SELECT 1", output_path: out)
+      expect(File.read("#{out}.partial")).to eq("another writer")
+      expect(Dir.glob(File.join(@dir, ".wp2txt-*"))).to be_empty
+    end
+
     it "removes .partial and leaves no output on SQL error" do
       out = File.join(@dir, "q.jsonl")
       expect { @corpus.query_sql("SELECT * FROM no_such_table", output_path: out) }
@@ -291,7 +351,7 @@ RSpec.describe "titles extraction and SQL file output" do
 
       expect(result[:cells_clipped]).to eq(1)
       value = read_jsonl(out).first["big"]
-      expect(value.length).to eq(Wp2txt::Corpus::SQL_FILE_CELL_LIMIT + 1)
+      expect(value.bytesize).to eq(Wp2txt::Corpus::SQL_FILE_CELL_LIMIT)
     end
 
     it "writes a .meta.json sidecar with SQL, attach provenance, and counts" do

data/wp2txt.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |s|
   s.license = "MIT"
   s.required_ruby_version = Gem::Requirement.new(">= 3.0")
   s.files = `git ls-files`.split("\n")
-  s.files -= ["data/*", "image/*"]
+                          .reject { |f| f.start_with?("data/", "image/", "scripts/") }
   s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
   s.require_paths = ["lib"]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wp2txt
 version: !ruby/object:Gem::Version
-  version: 2.3.2
+  version: 2.3.4
 platform: ruby
 authors:
 - Yoichiro Hasebe
@@ -204,6 +204,7 @@ files:
 - ".dockerignore"
 - ".github/FUNDING.yml"
 - ".github/workflows/ci.yml"
+- ".github/workflows/publish-image.yml"
 - ".gitignore"
 - ".solargraph.yml"
 - CHANGELOG.md
@@ -218,8 +219,6 @@ files:
 - bin/wp2txt
 - bin/wp2txt-mcp
 - docs/INDEXES.md
-- image/wp2txt-logo.svg
-- image/wp2txt.svg
 - lib/wp2txt.rb
 - lib/wp2txt/article.rb
 - lib/wp2txt/bz2_validator.rb
@@ -259,12 +258,6 @@ files:
 - lib/wp2txt/text_processing.rb
 - lib/wp2txt/utils.rb
 - lib/wp2txt/version.rb
-- scripts/benchmark_regex.rb
-- scripts/fetch_html_entities.rb
-- scripts/fetch_language_metadata.rb
-- scripts/fetch_mediawiki_data.rb
-- scripts/fetch_template_data.rb
-- scripts/profile_memory.rb
 - spec/article_spec.rb
 - spec/auto_download_spec.rb
 - spec/bz2_validator_spec.rb
@@ -291,7 +284,9 @@ files:
 - spec/metadata_index_spec.rb
 - spec/multi_dump_attach_spec.rb
 - spec/multistream_spec.rb
+- spec/output_integrity_spec.rb
 - spec/output_writer_spec.rb
+- spec/p1_correctness_spec.rb
 - spec/parser_functions_spec.rb
 - spec/ractor_worker_spec.rb
 - spec/regex_spec.rb
@@ -357,7 +352,9 @@ test_files:
 - spec/metadata_index_spec.rb
 - spec/multi_dump_attach_spec.rb
 - spec/multistream_spec.rb
+- spec/output_integrity_spec.rb
 - spec/output_writer_spec.rb
+- spec/p1_correctness_spec.rb
 - spec/parser_functions_spec.rb
 - spec/ractor_worker_spec.rb
 - spec/regex_spec.rb

data/image/wp2txt-logo.svg

@@ -1,16 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<svg viewBox="46.275 153.235 274.248 70.635" xmlns="http://www.w3.org/2000/svg">
-  <g style="" transform="matrix(0.140807, 0, 0, 0.140807, 46.270237, 160.188004)">
-    <path style="fill:#194F82;" d="M359.854,0H32.743C14.707,0,0.032,14.675,0.032,32.711v241.972 c0,18.036,14.675,32.711,32.711,32.711h91.798l-5.495,21.786H91.83c-6.012,0-10.925,4.848-10.925,10.925v41.503 c0,6.012,4.848,10.925,10.925,10.925h208.873c6.012,0,10.925-4.848,10.925-10.925v-41.503c0-6.012-4.848-10.925-10.925-10.925 h-27.216l-5.495-21.786h91.798c18.036,0,32.711-14.675,32.711-32.711V32.711C392.566,14.675,377.891,0,359.854,0z M289.778,351.03 v19.717H102.82V351.03h24.824c5.042,0,9.374-3.426,10.602-8.275l8.792-35.362h98.586l8.792,35.362 c1.228,4.848,5.56,8.275,10.602,8.275H289.778z M359.854,285.608H32.743c-6.012,0-10.925-4.848-10.925-10.925V32.711 c0-6.012,4.848-10.925,10.925-10.925h327.111c6.012,0,10.925,4.848,10.925,10.925v242.036h0.065 C370.78,280.695,365.931,285.608,359.854,285.608z"/>
-    <path style="fill:#194F82;" d="M338.004,43.572H54.594c-6.012,0-10.925,4.848-10.925,10.925v198.335 c0,6.012,4.848,10.925,10.925,10.925h283.539c6.012,0,10.925-4.848,10.925-10.925V54.562 C348.994,48.549,344.016,43.572,338.004,43.572z M327.208,241.907h-0.065H65.454v-88.566h13.188 c6.012,0,10.925-4.848,10.925-10.925c0-6.012-4.848-10.925-10.925-10.925H65.454V109.64h34.974c6.012,0,10.925-4.848,10.925-10.925 c0-6.012-4.848-10.925-10.925-10.925H65.454V65.293h261.754V241.907L327.208,241.907z"/>
-    <path style="fill:#194F82;" d="M109.155,161.099l34.521,34.521c6.723,6.206,13.317,2.133,15.451,0 c4.267-4.267,4.267-11.119,0-15.451l-26.828-26.764l26.828-26.828c4.267-4.267,4.267-11.119,0-15.451 c-4.267-4.267-11.119-4.267-15.451,0l-34.521,34.521C104.889,149.915,104.889,156.768,109.155,161.099z"/>
-    <path style="fill:#194F82;" d="M233.535,195.62c4.978,5.042,12.283,3.556,15.451,0l34.521-34.521 c4.267-4.267,4.267-11.119,0-15.451l-34.521-34.521c-4.267-4.267-11.119-4.267-15.451,0c-4.267,4.267-4.267,11.119,0,15.451 l26.828,26.828l-26.828,26.828C229.269,184.436,229.269,191.289,233.535,195.62z"/>
-    <path style="fill:#194F82;" d="M174.254,219.152c6.853,2.069,12.283-2.263,13.834-6.788l37.883-110.222 c1.939-5.689-1.099-11.895-6.788-13.834c-5.689-1.939-11.895,1.099-13.834,6.788l-37.883,110.222 C165.527,210.941,168.501,217.341,174.254,219.152z"/>
-  </g>
-  <path d="M 187.019 174.178 L 187.537 174.178 Q 188.054 174.178 188.266 174.484 Q 188.477 174.79 188.477 175.001 Q 188.477 175.213 188.43 175.354 L 180.012 205.593 Q 179.542 207.145 178.061 207.145 Q 176.579 207.145 176.156 205.593 L 170.371 184.901 L 164.587 205.687 Q 164.07 207.145 162.682 207.145 Q 161.295 207.145 160.778 205.687 L 151.89 175.401 Q 151.842 175.213 151.842 174.978 Q 151.842 174.743 152.054 174.461 Q 152.266 174.178 152.783 174.178 L 153.253 174.178 Q 153.818 174.178 154.147 174.837 L 162.706 203.9 L 168.773 182.22 Q 169.149 180.903 170.442 180.903 Q 171.735 180.903 172.017 182.267 Q 173.005 185.841 175.051 193.06 Q 177.096 200.279 178.084 203.853 L 186.126 174.884 Q 186.22 174.649 186.479 174.414 Q 186.737 174.178 187.019 174.178 Z" transform="matrix(1.352871, 0, 0, 1.231972, -93.713486, -46.78933)" style="fill: rgb(25, 79, 130); line-height: 73.9473px; white-space: pre;"/>
-  <path d="M 193.83 174.178 L 201.12 174.178 Q 204.412 174.178 207.186 175.966 Q 208.644 176.906 209.538 178.693 Q 210.431 180.48 210.431 182.926 Q 210.431 186.641 208.268 189.157 Q 206.105 191.673 200.743 191.673 L 197.64 191.673 Q 197.216 191.673 196.958 191.391 Q 196.699 191.109 196.699 190.732 L 196.699 190.309 Q 196.699 189.369 197.64 189.369 L 200.179 189.369 Q 205.305 189.369 206.81 187.205 Q 208.08 185.418 208.08 182.879 Q 208.08 179.54 205.94 178.035 Q 203.8 176.53 200.884 176.53 L 194.112 176.53 L 194.112 206.157 Q 194.112 206.534 193.83 206.816 Q 193.548 207.098 193.172 207.098 L 192.702 207.098 Q 192.325 207.098 192.043 206.816 Q 191.761 206.534 191.761 206.157 L 191.761 176.248 Q 191.761 175.025 192.184 174.602 Q 192.608 174.178 193.83 174.178 Z" transform="matrix(1.352871, 0, 0, 1.231972, -93.713486, -46.78933)" style="fill: rgb(25, 79, 130); line-height: 73.9473px; white-space: pre;"/>
-  <path d="M 224.167 174.178 Q 228.258 174.178 230.468 176.318 Q 232.679 178.458 232.679 181.985 Q 232.679 183.96 231.456 186.523 Q 230.233 189.086 229.057 190.732 Q 226.33 194.636 221.721 199.95 L 220.31 201.643 Q 219.135 203.054 218.5 203.9 Q 217.865 204.747 217.724 204.888 Q 217.818 204.841 218.006 204.841 L 232.443 204.841 Q 233.384 204.841 233.384 205.734 L 233.384 206.157 Q 233.384 207.098 232.443 207.098 L 217.959 207.098 Q 217.206 207.098 216.76 207.004 Q 216.313 206.91 215.866 206.369 Q 215.419 205.828 215.419 204.864 Q 215.419 203.9 217.442 201.361 L 219.417 199.103 Q 219.84 198.586 220.357 197.975 Q 221.58 196.564 222.568 195.294 L 223.837 193.789 Q 224.119 193.413 224.99 192.308 Q 225.86 191.203 226.189 190.779 Q 226.518 190.356 227.247 189.369 Q 227.976 188.381 228.281 187.84 Q 228.587 187.299 229.081 186.429 Q 229.575 185.559 229.81 184.948 Q 230.374 183.49 230.374 182.22 Q 230.374 179.258 228.54 177.847 Q 226.706 176.436 224.167 176.436 Q 219.652 176.436 217.912 180.668 Q 217.63 181.233 217.065 181.233 L 216.642 181.233 Q 215.701 181.233 215.701 180.386 Q 215.701 180.151 215.749 180.057 Q 216.689 177.282 218.97 175.73 Q 221.251 174.178 224.167 174.178 Z" transform="matrix(1.352871, 0, 0, 1.231972, -93.713486, -46.78933)" style="fill: rgb(25, 79, 130); line-height: 73.9473px; white-space: pre;"/>
-  <path d="M 247.486 177.753 L 247.956 177.753 Q 248.897 177.753 248.897 178.646 L 248.897 206.157 Q 248.897 206.534 248.638 206.816 Q 248.38 207.098 247.956 207.098 L 247.486 207.098 Q 247.063 207.098 246.804 206.816 Q 246.545 206.534 246.545 206.157 L 246.545 178.646 Q 246.545 177.753 247.486 177.753 Z M 237.328 174.178 L 258.208 174.178 Q 258.632 174.178 258.89 174.461 Q 259.149 174.743 259.149 175.119 L 259.149 175.636 Q 259.149 176.483 258.208 176.483 L 237.328 176.483 Q 236.387 176.483 236.387 175.636 L 236.387 175.119 Q 236.387 174.743 236.67 174.461 Q 236.952 174.178 237.328 174.178 Z" transform="matrix(1.352871, 0, 0, 1.231972, -93.713486, -46.78933)" style="fill: rgb(25, 79, 130); line-height: 73.9473px; white-space: pre;"/>
-  <path d="M 279.324 174.178 L 279.841 174.178 Q 280.405 174.178 280.664 174.649 Q 280.923 175.119 280.64 175.589 L 280.64 175.542 L 272.693 190.027 Q 272.646 190.074 272.528 190.262 Q 272.411 190.45 272.317 190.544 L 272.599 190.92 L 281.111 205.734 Q 281.252 205.969 281.252 206.204 Q 281.252 206.44 281.064 206.769 Q 280.876 207.098 280.311 207.098 L 279.747 207.098 Q 279.23 207.098 279.042 206.816 L 270.906 192.707 Q 270.435 193.507 267.167 198.751 Q 263.899 203.994 262.253 206.675 Q 261.923 207.098 261.453 207.098 L 260.889 207.098 Q 260.277 207.098 260.042 206.628 Q 259.807 206.157 260.089 205.64 L 269.307 190.92 L 269.307 190.967 Q 269.495 190.591 269.636 190.45 Q 269.448 190.262 269.401 190.121 L 259.666 175.636 Q 259.525 175.401 259.525 175.166 Q 259.525 174.931 259.713 174.555 Q 259.901 174.178 260.465 174.178 L 260.983 174.178 Q 261.453 174.178 261.782 174.602 L 271 188.287 L 278.571 174.461 Q 278.759 174.178 279.324 174.178 Z" transform="matrix(1.352871, 0, 0, 1.231972, -93.713486, -46.78933)" style="fill: rgb(25, 79, 130); line-height: 73.9473px; white-space: pre;"/>
-  <path d="M 293.994 177.753 L 294.464 177.753 Q 295.405 177.753 295.405 178.646 L 295.405 206.157 Q 295.405 206.534 295.146 206.816 Q 294.888 207.098 294.464 207.098 L 293.994 207.098 Q 293.571 207.098 293.312 206.816 Q 293.054 206.534 293.054 206.157 L 293.054 178.646 Q 293.054 177.753 293.994 177.753 Z M 283.836 174.178 L 304.716 174.178 Q 305.14 174.178 305.398 174.461 Q 305.657 174.743 305.657 175.119 L 305.657 175.636 Q 305.657 176.483 304.716 176.483 L 283.836 176.483 Q 282.896 176.483 282.896 175.636 L 282.896 175.119 Q 282.896 174.743 283.178 174.461 Q 283.46 174.178 283.836 174.178 Z" transform="matrix(1.352871, 0, 0, 1.231972, -93.713486, -46.78933)" style="fill: rgb(25, 79, 130); line-height: 73.9473px; white-space: pre;"/>
-</svg>

data/image/wp2txt.svg

@@ -1,31 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<svg version="1.1" id="Layer_1" x="0px" y="0px" style="enable-background:new 0 0 392.533 392.533;" xmlns="http://www.w3.org/2000/svg" xmlns:bx="https://boxy-svg.com">
-  <defs>
-    <bx:guide x="212.88" y="167.188" angle="90"/>
-    <bx:guide x="327.22" y="208.912" angle="90"/>
-    <style bx:fonts="Text Me One">@import url(https://fonts.googleapis.com/css2?family=Text+Me+One%3Aital%2Cwght%400%2C400&amp;display=swap);</style>
-  </defs>
-  <g style="" transform="matrix(0.140807, 0, 0, 0.140807, 46.270237, 160.188004)">
-    <path style="fill:#194F82;" d="M359.854,0H32.743C14.707,0,0.032,14.675,0.032,32.711v241.972 c0,18.036,14.675,32.711,32.711,32.711h91.798l-5.495,21.786H91.83c-6.012,0-10.925,4.848-10.925,10.925v41.503 c0,6.012,4.848,10.925,10.925,10.925h208.873c6.012,0,10.925-4.848,10.925-10.925v-41.503c0-6.012-4.848-10.925-10.925-10.925 h-27.216l-5.495-21.786h91.798c18.036,0,32.711-14.675,32.711-32.711V32.711C392.566,14.675,377.891,0,359.854,0z M289.778,351.03 v19.717H102.82V351.03h24.824c5.042,0,9.374-3.426,10.602-8.275l8.792-35.362h98.586l8.792,35.362 c1.228,4.848,5.56,8.275,10.602,8.275H289.778z M359.854,285.608H32.743c-6.012,0-10.925-4.848-10.925-10.925V32.711 c0-6.012,4.848-10.925,10.925-10.925h327.111c6.012,0,10.925,4.848,10.925,10.925v242.036h0.065 C370.78,280.695,365.931,285.608,359.854,285.608z"/>
-    <path style="fill:#194F82;" d="M338.004,43.572H54.594c-6.012,0-10.925,4.848-10.925,10.925v198.335 c0,6.012,4.848,10.925,10.925,10.925h283.539c6.012,0,10.925-4.848,10.925-10.925V54.562 C348.994,48.549,344.016,43.572,338.004,43.572z M327.208,241.907h-0.065H65.454v-88.566h13.188 c6.012,0,10.925-4.848,10.925-10.925c0-6.012-4.848-10.925-10.925-10.925H65.454V109.64h34.974c6.012,0,10.925-4.848,10.925-10.925 c0-6.012-4.848-10.925-10.925-10.925H65.454V65.293h261.754V241.907L327.208,241.907z"/>
-    <path style="fill:#194F82;" d="M109.155,161.099l34.521,34.521c6.723,6.206,13.317,2.133,15.451,0 c4.267-4.267,4.267-11.119,0-15.451l-26.828-26.764l26.828-26.828c4.267-4.267,4.267-11.119,0-15.451 c-4.267-4.267-11.119-4.267-15.451,0l-34.521,34.521C104.889,149.915,104.889,156.768,109.155,161.099z"/>
-    <path style="fill:#194F82;" d="M233.535,195.62c4.978,5.042,12.283,3.556,15.451,0l34.521-34.521 c4.267-4.267,4.267-11.119,0-15.451l-34.521-34.521c-4.267-4.267-11.119-4.267-15.451,0c-4.267,4.267-4.267,11.119,0,15.451 l26.828,26.828l-26.828,26.828C229.269,184.436,229.269,191.289,233.535,195.62z"/>
-    <path style="fill:#194F82;" d="M174.254,219.152c6.853,2.069,12.283-2.263,13.834-6.788l37.883-110.222 c1.939-5.689-1.099-11.895-6.788-13.834c-5.689-1.939-11.895,1.099-13.834,6.788l-37.883,110.222 C165.527,210.941,168.501,217.341,174.254,219.152z"/>
-  </g>
-  <g/>
-  <g/>
-  <g/>
-  <g/>
-  <g/>
-  <g/>
-  <g/>
-  <g/>
-  <g/>
-  <g/>
-  <g/>
-  <g/>
-  <g/>
-  <g/>
-  <g/>
-  <text style="fill: rgb(25, 79, 130); font-family: &quot;Text Me One&quot;; font-size: 47.0279px; line-height: 73.9473px; white-space: pre;" transform="matrix(1.352871, 0, 0, 1.231972, -93.713486, -46.78933)" x="152.689" y="207.098">WP2TXT</text>
-</svg>