wp2txt 2.3.3 → 2.4.0

data/spec/template_passthrough_spec.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+require "wp2txt"
+require "wp2txt/formatter"
+
+# Templates the cleaning stage renders itself must survive template expansion;
+# deleting them earlier silently dropped readings and link text from articles.
+RSpec.describe "templates rendered by the cleaning stage" do
+  include Wp2txt
+  include Wp2txt::Formatter
+
+  def summary_of(wikitext)
+    article = Wp2txt::Article.new("#{wikitext}\n\n== 節 ==\n本文\n", "T", true)
+    format_article(article, { format: :json, summary_only: true, expand_templates: true,
+                              markers: [:all] })["text"].strip
+  end
+
+  {
+    "'''{{読み仮名|言語|げんご}}'''は記号体系。" => "言語（げんご）は記号体系。",
+    "{{読み仮名_ruby不使用|東京|とうきょう}}都" => "東京（とうきょう）都",
+    "作曲は{{仮リンク|ジョン・ドウ|en|John Doe}}が担当した。" => "作曲はジョン・ドウが担当した。",
+    "{{ruby|漢字|かんじ}}を読む。" => "漢字（かんじ）を読む。",
+    "作曲は{{仮リンク|{{lang|en|John Doe}}|en|John Doe}}。" => "作曲はJohn Doe。"
+  }.each do |source, expected|
+    it "renders #{source.inspect}" do
+      expect(summary_of(source)).to eq(expected)
+    end
+  end
+
+  it "keeps an image caption that contains a link template" do
+    source = "[[ファイル:Stele.jpg|thumb|200px|[[ナラム・シン]]。画像は{{仮リンク|戦勝記念碑|en|Victory Stele}}。]]\n'''紀元前23世紀'''は世紀。"
+    expect(summary_of(source)).to include("ナラム・シン。画像は戦勝記念碑。")
+  end
+
+  it "still removes templates nothing downstream knows" do
+    expect(summary_of("前{{Otheruses|x|y}}後")).to eq("前後")
+  end
+
+  it "adds no empty brackets for an empty reading" do
+    expect(summary_of("{{読み仮名|語|}}は")).to eq("語は")
+  end
+end

data/spec/titles_output_path_spec.rb

@@ -36,6 +36,18 @@ RSpec.describe "titles extraction and SQL file output" do
     File.readlines(path).map { |l| JSON.parse(l) }
   end
 
+  describe "extract_corpus across several batches with worker processes" do
+    it "writes each record once (workers must not re-flush the output buffer)" do
+      stub_const("Wp2txt::Corpus::EXTRACT_BATCH_SIZE", 1)
+      out = File.join(@dir, "batched.jsonl")
+      titles = ["Film A", "Film B", "Person X"]
+      result = @corpus.extract_corpus(output_path: out, content: "full", titles: titles, num_processes: 2)
+      records = read_jsonl(out)
+      expect(records.map { |r| r["title"] }.tally).to eq(titles.to_h { |t| [t, 1] })
+      expect(records.size).to eq(result[:records_written])
+    end
+  end
+
   describe "extract_corpus titles:" do
     it "extracts an explicit set with normalization, dedup, and input order" do
       out = File.join(@dir, "t.jsonl")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wp2txt
 version: !ruby/object:Gem::Version
-  version: 2.3.3
+  version: 2.4.0
 platform: ruby
 authors:
 - Yoichiro Hasebe
@@ -243,21 +243,26 @@ files:
 - lib/wp2txt/index_cache.rb
 - lib/wp2txt/index_commands.rb
 - lib/wp2txt/langlinks_importer.rb
+- lib/wp2txt/lead_terms.rb
+- lib/wp2txt/link_counter.rb
 - lib/wp2txt/magic_words.rb
 - lib/wp2txt/memory_monitor.rb
 - lib/wp2txt/metadata_index.rb
 - lib/wp2txt/multistream.rb
 - lib/wp2txt/output_path.rb
 - lib/wp2txt/output_writer.rb
+- lib/wp2txt/page_props_importer.rb
 - lib/wp2txt/parser_functions.rb
 - lib/wp2txt/ractor_worker.rb
 - lib/wp2txt/regex.rb
 - lib/wp2txt/section_extractor.rb
+- lib/wp2txt/sql_dump_reader.rb
 - lib/wp2txt/stream_processor.rb
 - lib/wp2txt/template_expander.rb
 - lib/wp2txt/text_processing.rb
 - lib/wp2txt/utils.rb
 - lib/wp2txt/version.rb
+- lib/wp2txt/wikitext_regions.rb
 - spec/article_spec.rb
 - spec/auto_download_spec.rb
 - spec/bz2_validator_spec.rb
@@ -278,17 +283,22 @@ files:
 - spec/index_cache_spec.rb
 - spec/integration_spec.rb
 - spec/langlinks_importer_spec.rb
+- spec/lead_terms_edge_cases_spec.rb
+- spec/lead_terms_links_qids_spec.rb
 - spec/magic_words_spec.rb
 - spec/markers_spec.rb
 - spec/memory_monitor_spec.rb
 - spec/metadata_index_spec.rb
 - spec/multi_dump_attach_spec.rb
 - spec/multistream_spec.rb
+- spec/output_integrity_spec.rb
 - spec/output_writer_spec.rb
 - spec/p1_correctness_spec.rb
+- spec/page_properties_spec.rb
 - spec/parser_functions_spec.rb
 - spec/ractor_worker_spec.rb
 - spec/regex_spec.rb
+- spec/region_semantics_spec.rb
 - spec/section_extractor_spec.rb
 - spec/spec_helper.rb
 - spec/stream_processor_spec.rb
@@ -296,6 +306,7 @@ files:
 - spec/support/multistream_fixture.rb
 - spec/template_data_spec.rb
 - spec/template_expander_spec.rb
+- spec/template_passthrough_spec.rb
 - spec/template_processing_spec.rb
 - spec/text_processing_spec.rb
 - spec/titles_output_path_spec.rb
@@ -345,17 +356,22 @@ test_files:
 - spec/index_cache_spec.rb
 - spec/integration_spec.rb
 - spec/langlinks_importer_spec.rb
+- spec/lead_terms_edge_cases_spec.rb
+- spec/lead_terms_links_qids_spec.rb
 - spec/magic_words_spec.rb
 - spec/markers_spec.rb
 - spec/memory_monitor_spec.rb
 - spec/metadata_index_spec.rb
 - spec/multi_dump_attach_spec.rb
 - spec/multistream_spec.rb
+- spec/output_integrity_spec.rb
 - spec/output_writer_spec.rb
 - spec/p1_correctness_spec.rb
+- spec/page_properties_spec.rb
 - spec/parser_functions_spec.rb
 - spec/ractor_worker_spec.rb
 - spec/regex_spec.rb
+- spec/region_semantics_spec.rb
 - spec/section_extractor_spec.rb
 - spec/spec_helper.rb
 - spec/stream_processor_spec.rb
@@ -363,6 +379,7 @@ test_files:
 - spec/support/multistream_fixture.rb
 - spec/template_data_spec.rb
 - spec/template_expander_spec.rb
+- spec/template_passthrough_spec.rb
 - spec/template_processing_spec.rb
 - spec/text_processing_spec.rb
 - spec/titles_output_path_spec.rb