wp2txt 2.3.0 → 2.3.2

data/spec/utils_spec.rb

@@ -59,6 +59,135 @@ RSpec.describe "Wp2txt Utils" do
     end
   end
 
+  # Regression: process_external_links used to strip the brackets of the
+  # [ref]/[/ref] markers (contents "ref" / "/ref" take the parts.size == 1
+  # branch), so remove_ref could no longer find them and the tag names plus
+  # reference body leaked into the extracted text. These tests exercise the
+  # composed make_reference -> format_wiki path, since testing remove_ref in
+  # isolation passes even with the bug.
+  describe "reference markers through format_wiki" do
+    it "removes <ref>...</ref> including the body" do
+      input = "Paris was founded.<ref>Patrick Boucheron, France in the World (2019) pp 81-86.</ref> The city grew."
+      result = format_wiki(make_reference(input))
+      expect(result).to eq "Paris was founded. The city grew."
+    end
+
+    it "removes named <ref name=\"x\">...</ref> including the body" do
+      input = "Fine dining.<ref name=\"lemonde\">Le Monde, 2 February 2015</ref> Paris has."
+      result = format_wiki(make_reference(input))
+      expect(result).to eq "Fine dining. Paris has."
+    end
+
+    it "removes self-closing <ref name=\"x\"/>" do
+      input = "An asteroid,<ref name=\"x\"/> and a building."
+      result = format_wiki(make_reference(input))
+      expect(result).to eq "An asteroid, and a building."
+    end
+
+    it "removes consecutive <ref>A</ref><ref>B</ref> without joining tag names" do
+      input = "Text<ref>A</ref><ref>B</ref> more."
+      result = format_wiki(make_reference(input))
+      expect(result).to eq "Text more."
+    end
+
+    it "keeps [ref]...[/ref] markers when config[:ref] is true" do
+      input = "Paris was founded.<ref>Patrick Boucheron, France in the World (2019) pp 81-86.</ref> The city grew."
+      result = format_wiki(make_reference(input), ref: true)
+      expect(result).to eq "Paris was founded.[ref]Patrick Boucheron, France in the World (2019) pp 81-86.[/ref] The city grew."
+    end
+
+    # Guards against the rejected fix of returning "[ref]" from the scanner
+    # block: that makes process_nested_single_pass re-detect the same spot
+    # until MAX_NESTING_ITERATIONS, leaving external links unprocessed.
+    it "still processes external links outside [ref] markers" do
+      input = "Claim.<ref>See [http://example.com the site] for detail.</ref> Next [http://foo.com Foo] end."
+      expect(format_wiki(make_reference(input))).to eq "Claim. Next Foo end."
+      expect(format_wiki(make_reference(input), ref: true)).to eq "Claim.[ref]See the site for detail.[/ref] Next Foo end."
+    end
+  end
+
+  # Regression: element splitting (Article#parse) breaks paragraphs at
+  # newlines, so a reference written across lines landed in separate elements
+  # with [ref] and [/ref] never visible to remove_ref at the same time. This
+  # also meant --extract-citations never fired for multi-line cite templates.
+  # These tests go through Article.new -> format_wiki per element, since
+  # passing a string to format_wiki directly skips element splitting and does
+  # not reproduce the bug.
+  describe "multi-line references through element splitting" do
+    def render_elements(wikitext, config = {})
+      Wp2txt::Article.new(wikitext).elements.map { |e| format_wiki(e[1], config) }.join
+    end
+
+    it "removes an empty reference spanning a blank line" do
+      result = render_elements("...in the Super League.<ref>\n\n</ref> In 2006, Catalans Dragons became...")
+      expect(result).not_to include("[ref]")
+      expect(result).not_to include("[/ref]")
+      expect(result).to include("Super League. In 2006,")
+    end
+
+    it "removes a named empty reference spanning a blank line" do
+      result = render_elements("Claim.<ref name=\"x\">\n\n</ref> Next.")
+      expect(result).not_to include("[ref]")
+      expect(result).not_to include("[/ref]")
+      expect(result).to include("Claim. Next.")
+    end
+
+    it "removes a non-empty reference spanning a blank line" do
+      result = render_elements("Claim.<ref>Author\n\nPublisher</ref> Next.")
+      expect(result).not_to include("[ref]")
+      expect(result).not_to include("[/ref]")
+      expect(result).to include("Claim. Next.")
+    end
+
+    it "removes a multi-line cite template reference (with and without leading space)" do
+      ["Claim.<ref> {{cite book\n |title=Foo\n |year=2019}}</ref> Next.",
+       "Claim.<ref>{{cite book\n|title=Foo\n|year=2019}}</ref> Next."].each do |input|
+        result = render_elements(input)
+        expect(result).not_to include("[ref]")
+        expect(result).not_to include("[/ref]")
+        expect(result).not_to include("cite book")
+        expect(result).to include("Claim. Next.")
+      end
+    end
+
+    # The core of this fix: before, only the multi-line form leaked raw
+    # markup, so --extract-citations silently did nothing for it.
+    it "extracts citations identically from single-line and multi-line cite templates" do
+      config = { extract_citations: true, ref: true }
+      single = render_elements("Claim.<ref>{{cite book|title=Foo|year=2019}}</ref> Next.", config)
+      multi  = render_elements("Claim.<ref>{{cite book\n|title=Foo\n|year=2019}}</ref> Next.", config)
+      expect(multi).to eq single
+      expect(single).to include("[ref]\"Foo\". 2019.[/ref]")
+    end
+
+    # Control: newlines outside references must still split paragraphs.
+    it "still splits ordinary paragraph boundaries at blank lines" do
+      elements = Wp2txt::Article.new("First para.\n\nSecond para.").elements
+      paragraphs = elements.select { |e| e[0] == :mw_paragraph }
+      expect(paragraphs.size).to eq 2
+      expect(paragraphs[0][1]).to include("First para.")
+      expect(paragraphs[1][1]).to include("Second para.")
+    end
+
+    # An unclosed <ref> must not pair with a later </ref> across paragraphs;
+    # the body text stays (a floating [ref] is the pre-existing behavior for
+    # this malformed markup).
+    it "does not swallow paragraphs when a <ref> is left unclosed" do
+      wikitext = "First para with <ref>unclosed reference.\n\nSecond para here.\n\n" \
+                 "Third para with <ref>closed</ref> end."
+      result = render_elements(wikitext)
+      expect(result).to include("Second para here.")
+      expect(result).to include("First para with")
+    end
+
+    it "handles a multi-line reference with a group attribute" do
+      result = render_elements("Claim.<ref group=\"note\">Author\nTitle 2019</ref> Next.")
+      expect(result).not_to include("[ref]")
+      expect(result).not_to include("[/ref]")
+      expect(result).to include("Claim. Next.")
+    end
+  end
+
   describe "remove_table" do
     it "removes table formated parts" do
       str_before = "{| ... \n{| ... \n ...|}\n ...|}"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wp2txt
 version: !ruby/object:Gem::Version
-  version: 2.3.0
+  version: 2.3.2
 platform: ruby
 authors:
 - Yoichiro Hasebe
@@ -217,7 +217,7 @@ files:
 - Rakefile
 - bin/wp2txt
 - bin/wp2txt-mcp
-- docs/RESEARCH.md
+- docs/INDEXES.md
 - image/wp2txt-logo.svg
 - image/wp2txt.svg
 - lib/wp2txt.rb
@@ -276,6 +276,7 @@ files:
 - spec/config_spec.rb
 - spec/constants_spec.rb
 - spec/corpus_spec.rb
+- spec/docs_sync_spec.rb
 - spec/file_utils_spec.rb
 - spec/fixtures/samples.rb
 - spec/formatter_sections_spec.rb
@@ -341,6 +342,7 @@ test_files:
 - spec/config_spec.rb
 - spec/constants_spec.rb
 - spec/corpus_spec.rb
+- spec/docs_sync_spec.rb
 - spec/file_utils_spec.rb
 - spec/fixtures/samples.rb
 - spec/formatter_sections_spec.rb