wp2txt 1.1.3 → 2.1.0

data/spec/regex_spec.rb

@@ -0,0 +1,281 @@
+# frozen_string_literal: true
+
+require_relative "spec_helper"
+
+RSpec.describe "Wp2txt MediaWiki Data Loading" do
+  describe "load_mediawiki_data" do
+    it "loads data from JSON file" do
+      data = Wp2txt.load_mediawiki_data
+      expect(data).to be_a(Hash)
+      expect(data).to have_key("magic_words")
+      expect(data).to have_key("namespaces")
+    end
+
+    it "contains redirect keywords" do
+      data = Wp2txt.load_mediawiki_data
+      redirects = data.dig("magic_words", "redirect")
+      expect(redirects).to be_an(Array)
+      expect(redirects).to include("REDIRECT")
+      expect(redirects.size).to be > 100  # Should have many languages
+    end
+
+    it "contains category namespaces" do
+      data = Wp2txt.load_mediawiki_data
+      categories = data.dig("namespaces", "category")
+      expect(categories).to be_an(Array)
+      expect(categories).to include("Category")
+      expect(categories).to include("カテゴリ")  # Japanese
+      expect(categories).to include("분류")      # Korean (Hangul)
+      expect(categories).to include("分類")      # Chinese Traditional
+    end
+
+    it "contains file namespaces" do
+      data = Wp2txt.load_mediawiki_data
+      files = data.dig("namespaces", "file")
+      expect(files).to be_an(Array)
+      expect(files).to include("File")
+      expect(files).to include("Image")
+    end
+  end
+end
+
+RSpec.describe "Wp2txt Regex Patterns" do
+  # Define local references to module constants
+  let(:remove_hr_regex) { Wp2txt::REMOVE_HR_REGEX }
+  let(:in_heading_regex) { Wp2txt::IN_HEADING_REGEX }
+  let(:redirect_regex) { Wp2txt::REDIRECT_REGEX }
+  let(:category_regex) { Wp2txt::CATEGORY_REGEX }
+  let(:in_link_regex) { Wp2txt::IN_LINK_REGEX }
+  let(:ml_template_onset_regex) { Wp2txt::ML_TEMPLATE_ONSET_REGEX }
+  let(:ml_template_end_regex) { Wp2txt::ML_TEMPLATE_END_REGEX }
+  let(:blank_line_regex) { Wp2txt::BLANK_LINE_REGEX }
+  let(:isolated_template_regex) { Wp2txt::ISOLATED_TEMPLATE_REGEX }
+  let(:chrref_to_utf_regex) { Wp2txt::CHRREF_TO_UTF_REGEX }
+  let(:in_table_regex1) { Wp2txt::IN_TABLE_REGEX1 }
+  let(:in_table_regex2) { Wp2txt::IN_TABLE_REGEX2 }
+  let(:in_unordered_regex) { Wp2txt::IN_UNORDERED_REGEX }
+  let(:in_ordered_regex) { Wp2txt::IN_ORDERED_REGEX }
+  let(:in_definition_regex) { Wp2txt::IN_DEFINITION_REGEX }
+
+  describe "REMOVE_HR_REGEX" do
+    it "matches horizontal rules with 4+ hyphens" do
+      expect("----").to match(remove_hr_regex)
+      expect("----------").to match(remove_hr_regex)
+      expect("  ----  ").to match(remove_hr_regex)
+    end
+
+    it "does NOT match fewer than 4 hyphens" do
+      # These tests will FAIL with current implementation (exposing the bug)
+      expect("-").not_to match(remove_hr_regex)
+      expect("--").not_to match(remove_hr_regex)
+      expect("---").not_to match(remove_hr_regex)
+    end
+  end
+
+  describe "IN_HEADING_REGEX" do
+    it "matches valid headings with equal = counts" do
+      expect("== Title ==").to match(in_heading_regex)
+      expect("=== Section ===").to match(in_heading_regex)
+      expect("==== Subsection ====").to match(in_heading_regex)
+    end
+
+    # These tests document the expected behavior after fix
+    # Current implementation may not enforce matching = counts
+    it "handles headings with trailing whitespace" do
+      expect("== Title ==  ").to match(in_heading_regex)
+    end
+  end
+
+  describe "REDIRECT_REGEX" do
+    it "captures English redirect target correctly" do
+      match = "#REDIRECT [[Target Page]]".match(redirect_regex)
+      expect(match).not_to be_nil
+      expect(match[1]).to eq "Target Page"
+    end
+
+    it "handles Japanese redirect" do
+      match = "#転送 [[日本語ページ]]".match(redirect_regex)
+      expect(match).not_to be_nil
+      expect(match[1]).to eq "日本語ページ"
+    end
+
+    it "is case-insensitive for REDIRECT" do
+      match = "#redirect [[Page]]".match(redirect_regex)
+      expect(match).not_to be_nil
+    end
+
+    it "handles German redirect" do
+      match = "#WEITERLEITUNG [[Zielseite]]".match(redirect_regex)
+      expect(match).not_to be_nil
+      expect(match[1]).to eq "Zielseite"
+    end
+
+    it "handles French redirect" do
+      match = "#REDIRECTION [[Page cible]]".match(redirect_regex)
+      expect(match).not_to be_nil
+      expect(match[1]).to eq "Page cible"
+    end
+
+    it "handles Russian redirect" do
+      match = "#ПЕРЕНАПРАВЛЕНИЕ [[Целевая страница]]".match(redirect_regex)
+      expect(match).not_to be_nil
+      expect(match[1]).to eq "Целевая страница"
+    end
+
+    it "handles Chinese redirect" do
+      match = "#重定向 [[目标页面]]".match(redirect_regex)
+      expect(match).not_to be_nil
+      expect(match[1]).to eq "目标页面"
+    end
+
+    it "handles Korean redirect" do
+      match = "#넘겨주기 [[대상 문서]]".match(redirect_regex)
+      expect(match).not_to be_nil
+      expect(match[1]).to eq "대상 문서"
+    end
+
+    it "handles full-width # in Japanese redirect" do
+      match = "＃転送 [[日本語ページ]]".match(redirect_regex)
+      expect(match).not_to be_nil
+      expect(match[1]).to eq "日本語ページ"
+    end
+
+    it "handles Arabic redirect" do
+      match = "#تحويل [[الصفحة المستهدفة]]".match(redirect_regex)
+      expect(match).not_to be_nil
+    end
+
+    it "handles Hindi redirect" do
+      match = "#पुनर्प्रेषित [[लक्ष्य पृष्ठ]]".match(redirect_regex)
+      expect(match).not_to be_nil
+    end
+  end
+
+  describe "CATEGORY_REGEX" do
+    it "matches English categories" do
+      expect("[[Category:Science]]").to match(category_regex)
+    end
+
+    it "matches Italian/Spanish categories" do
+      expect("[[Categoria:Scienza]]").to match(category_regex)
+    end
+
+    it "matches Japanese categories" do
+      expect("[[カテゴリ:科学]]").to match(category_regex)
+    end
+
+    it "matches German categories" do
+      expect("[[Kategorie:Wissenschaft]]").to match(category_regex)
+    end
+
+    it "matches French categories" do
+      expect("[[Catégorie:Science]]").to match(category_regex)
+    end
+
+    it "matches Chinese categories" do
+      expect("[[分类:科学]]").to match(category_regex)
+      expect("[[分類:科學]]").to match(category_regex)
+    end
+
+    it "matches Russian categories" do
+      expect("[[Категория:Наука]]").to match(category_regex)
+    end
+
+    it "matches Korean categories" do
+      expect("[[분류:과학]]").to match(category_regex)
+    end
+
+    it "matches Arabic categories" do
+      expect("[[تصنيف:علم]]").to match(category_regex)
+    end
+  end
+
+  describe "IN_LINK_REGEX" do
+    it "matches wikilinks on their own line" do
+      expect("[[Article]]").to match(in_link_regex)
+    end
+
+    it "matches wikilinks with leading/trailing whitespace" do
+      expect("  [[Page|Text]]  ").to match(in_link_regex)
+    end
+  end
+
+  describe "ML_TEMPLATE_ONSET_REGEX" do
+    it "matches opening of multi-line templates" do
+      expect("{{Infobox").to match(ml_template_onset_regex)
+      expect("{{Template name").to match(ml_template_onset_regex)
+    end
+
+    it "does not match complete templates" do
+      expect("{{Complete}}").not_to match(ml_template_onset_regex)
+    end
+  end
+
+  describe "ML_TEMPLATE_END_REGEX" do
+    it "matches closing of multi-line templates" do
+      expect("}}").to match(ml_template_end_regex)
+      expect("}}  ").to match(ml_template_end_regex)
+      expect("content}}").to match(ml_template_end_regex)
+    end
+  end
+
+  describe "BLANK_LINE_REGEX" do
+    it "matches empty lines" do
+      expect("").to match(blank_line_regex)
+      expect("   ").to match(blank_line_regex)
+      expect("\t").to match(blank_line_regex)
+    end
+
+    it "does not match lines with content" do
+      expect("text").not_to match(blank_line_regex)
+    end
+  end
+
+  describe "ISOLATED_TEMPLATE_REGEX" do
+    it "matches single-line templates" do
+      expect("{{Template}}").to match(isolated_template_regex)
+      expect("  {{Template|param}}  ").to match(isolated_template_regex)
+    end
+  end
+
+  describe "CHRREF_TO_UTF_REGEX" do
+    it "matches decimal character references" do
+      expect("A").to match(chrref_to_utf_regex)
+      expect("♪").to match(chrref_to_utf_regex)
+    end
+
+    it "matches hexadecimal character references" do
+      expect("A").to match(chrref_to_utf_regex)
+      expect("♪").to match(chrref_to_utf_regex)
+      expect("😀").to match(chrref_to_utf_regex)
+    end
+  end
+
+  describe "IN_TABLE_REGEX1 and IN_TABLE_REGEX2" do
+    it "matches MediaWiki table start" do
+      expect("{|").to match(in_table_regex1)
+      expect("  {|").to match(in_table_regex1)
+    end
+
+    it "matches MediaWiki table end" do
+      expect("|}").to match(in_table_regex2)
+    end
+  end
+
+  describe "List detection regexes" do
+    it "detects unordered list items" do
+      expect("* Item").to match(in_unordered_regex)
+      expect("** Nested").to match(in_unordered_regex)
+    end
+
+    it "detects ordered list items" do
+      expect("# Item").to match(in_ordered_regex)
+      expect("## Nested").to match(in_ordered_regex)
+    end
+
+    it "detects definition list items" do
+      expect("; Term").to match(in_definition_regex)
+      expect(": Definition").to match(in_definition_regex)
+    end
+  end
+end

data/spec/section_extractor_spec.rb

@@ -0,0 +1,397 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+require "tmpdir"
+require "fileutils"
+require_relative "../lib/wp2txt/article"
+require_relative "../lib/wp2txt/section_extractor"
+
+RSpec.describe Wp2txt::SectionExtractor do
+  let(:sample_wiki_text) do
+    <<~WIKI
+      This is the summary text before any headings.
+
+      == Early life ==
+      Born in Tokyo, Japan.
+
+      == Career ==
+      Started working in 2010.
+
+      === Publications ===
+      First paper (2010)
+      Second paper (2015)
+
+      == Reception ==
+      The work was well received.
+
+      == References ==
+      <ref>Citation</ref>
+
+      [[Category:Scientists]]
+      [[Category:1990 births]]
+    WIKI
+  end
+
+  let(:article) { Wp2txt::Article.new(sample_wiki_text, "Test Person") }
+  let(:extractor) { described_class.new }
+
+  describe "#extract_headings" do
+    it "extracts all section headings" do
+      headings = extractor.extract_headings(article)
+      expect(headings).to eq(["Early life", "Career", "Publications", "Reception", "References"])
+    end
+
+    it "returns empty array for article without headings" do
+      simple_article = Wp2txt::Article.new("Just some text.", "Simple")
+      headings = extractor.extract_headings(simple_article)
+      expect(headings).to eq([])
+    end
+  end
+
+  describe "#extract_headings_with_levels" do
+    it "extracts headings with their levels" do
+      headings = extractor.extract_headings_with_levels(article)
+
+      expect(headings).to include(
+        { name: "Early life", level: 2 },
+        { name: "Career", level: 2 },
+        { name: "Publications", level: 3 },
+        { name: "Reception", level: 2 }
+      )
+    end
+
+    it "correctly identifies level 3 subsections" do
+      headings = extractor.extract_headings_with_levels(article)
+      publications = headings.find { |h| h[:name] == "Publications" }
+      expect(publications[:level]).to eq(3)
+    end
+  end
+
+  describe "#extract_summary" do
+    it "extracts text before first heading" do
+      summary = extractor.extract_summary(article)
+      expect(summary).to include("This is the summary text")
+    end
+
+    it "returns nil for article starting with heading" do
+      no_summary_article = Wp2txt::Article.new("== Heading ==\nContent", "No Summary")
+      summary = extractor.extract_summary(no_summary_article)
+      expect(summary).to be_nil
+    end
+  end
+
+  describe "#extract_sections with targets" do
+    context "when extracting summary and specific sections" do
+      let(:extractor) { described_class.new(["summary", "Career", "Plot"]) }
+
+      it "includes summary when requested" do
+        sections = extractor.extract_sections(article)
+        expect(sections["summary"]).to include("summary text")
+      end
+
+      it "includes matching sections" do
+        sections = extractor.extract_sections(article)
+        expect(sections["Career"]).to include("Started working")
+      end
+
+      it "returns nil for non-existent sections" do
+        sections = extractor.extract_sections(article)
+        expect(sections["Plot"]).to be_nil
+      end
+
+      it "includes subsections in parent section" do
+        sections = extractor.extract_sections(article)
+        expect(sections["Career"]).to include("First paper")
+      end
+    end
+
+    context "with minimum length filter" do
+      let(:extractor) { described_class.new(["summary", "Career"], min_length: 50) }
+
+      it "filters out short sections but keeps long ones" do
+        sections = extractor.extract_sections(article)
+        # Career section with subsections should be long enough (>50 chars)
+        expect(sections["Career"]).not_to be_nil
+      end
+
+      it "filters out sections shorter than min_length" do
+        strict_extractor = described_class.new(["summary", "Early life"], min_length: 100)
+        sections = strict_extractor.extract_sections(article)
+        # Early life section is short ("Born in Tokyo, Japan.")
+        expect(sections["Early life"]).to be_nil
+      end
+    end
+  end
+
+  describe "alias matching" do
+    context "with default aliases" do
+      let(:wiki_with_synopsis) do
+        <<~WIKI
+          Summary.
+
+          == Synopsis ==
+          The story follows...
+        WIKI
+      end
+      let(:synopsis_article) { Wp2txt::Article.new(wiki_with_synopsis, "Movie") }
+      let(:extractor) { described_class.new(["Plot"]) }
+
+      it "matches Synopsis as alias for Plot" do
+        sections = extractor.extract_sections(synopsis_article)
+        expect(sections["Plot"]).to include("story follows")
+      end
+    end
+
+    context "with aliases disabled" do
+      let(:wiki_with_synopsis) do
+        <<~WIKI
+          Summary.
+
+          == Synopsis ==
+          The story follows...
+        WIKI
+      end
+      let(:synopsis_article) { Wp2txt::Article.new(wiki_with_synopsis, "Movie") }
+      let(:extractor) { described_class.new(["Plot"], use_aliases: false) }
+
+      it "does not match Synopsis when aliases are disabled" do
+        sections = extractor.extract_sections(synopsis_article)
+        expect(sections["Plot"]).to be_nil
+      end
+    end
+  end
+
+  describe "case-insensitive matching" do
+    let(:extractor) { described_class.new(["early life", "CAREER"]) }
+
+    it "matches sections regardless of case" do
+      sections = extractor.extract_sections(article)
+      expect(sections["early life"]).to include("Born in Tokyo")
+      expect(sections["CAREER"]).to include("Started working")
+    end
+  end
+
+  describe "#has_matching_sections?" do
+    context "with matching sections" do
+      let(:extractor) { described_class.new(["Career"]) }
+
+      it "returns true" do
+        expect(extractor.has_matching_sections?(article)).to be true
+      end
+    end
+
+    context "with no matching sections" do
+      let(:extractor) { described_class.new(["Plot", "Gameplay"]) }
+
+      it "returns false" do
+        expect(extractor.has_matching_sections?(article)).to be false
+      end
+    end
+
+    context "when summary is requested and exists" do
+      let(:extractor) { described_class.new(["summary"]) }
+
+      it "returns true" do
+        expect(extractor.has_matching_sections?(article)).to be true
+      end
+    end
+  end
+
+  describe "#should_skip?" do
+    context "with skip_empty: false (default)" do
+      let(:extractor) { described_class.new(["Plot"], skip_empty: false) }
+
+      it "returns false even when no sections match" do
+        expect(extractor.should_skip?(article)).to be false
+      end
+    end
+
+    context "with skip_empty: true" do
+      let(:extractor) { described_class.new(["Plot"], skip_empty: true) }
+
+      it "returns true when no sections match" do
+        expect(extractor.should_skip?(article)).to be true
+      end
+
+      it "returns false when sections match" do
+        extractor_with_match = described_class.new(["Career"], skip_empty: true)
+        expect(extractor_with_match.should_skip?(article)).to be false
+      end
+    end
+  end
+
+  describe "alias file loading" do
+    let(:temp_dir) { Dir.mktmpdir }
+    let(:alias_file) { File.join(temp_dir, "aliases.yml") }
+
+    after { FileUtils.remove_entry(temp_dir) }
+
+    context "with valid YAML alias file" do
+      before do
+        File.write(alias_file, <<~YAML)
+          Career:
+            - Work history
+            - Employment
+          Plot:
+            - Synopsis
+            - Story
+        YAML
+      end
+
+      it "loads aliases from file" do
+        aliases = described_class.load_aliases_from_file(alias_file)
+        expect(aliases["Career"]).to eq(["Work history", "Employment"])
+        expect(aliases["Plot"]).to eq(["Synopsis", "Story"])
+      end
+
+      it "merges file aliases with defaults" do
+        extractor = described_class.new(["Plot"], alias_file: alias_file)
+        # Should have both default "Synopsis" and file "Story" as aliases
+        wiki_with_story = <<~WIKI
+          == Story ==
+          The story begins...
+        WIKI
+        story_article = Wp2txt::Article.new(wiki_with_story, "Film")
+        sections = extractor.extract_sections(story_article)
+        expect(sections["Plot"]).to include("story begins")
+      end
+    end
+
+    context "with non-existent file" do
+      it "returns empty hash" do
+        aliases = described_class.load_aliases_from_file("/nonexistent/file.yml")
+        expect(aliases).to eq({})
+      end
+    end
+
+    context "with invalid YAML" do
+      before { File.write(alias_file, "invalid: yaml: syntax: {{") }
+
+      it "returns empty hash" do
+        aliases = described_class.load_aliases_from_file(alias_file)
+        expect(aliases).to eq({})
+      end
+    end
+  end
+
+  describe "matched sections tracking" do
+    let(:wiki_with_synopsis) do
+      <<~WIKI
+        Summary text.
+        == Synopsis ==
+        The story begins...
+      WIKI
+    end
+    let(:synopsis_article) { Wp2txt::Article.new(wiki_with_synopsis, "Movie") }
+
+    context "with track_matches enabled" do
+      let(:extractor) { described_class.new(["Plot"], track_matches: true) }
+
+      it "records alias matches" do
+        extractor.extract_sections(synopsis_article)
+        expect(extractor.matched_sections["Plot"]).to eq("Synopsis")
+      end
+    end
+
+    context "with track_matches disabled (default)" do
+      let(:extractor) { described_class.new(["Plot"], track_matches: false) }
+
+      it "does not record matches" do
+        extractor.extract_sections(synopsis_article)
+        expect(extractor.matched_sections).to be_empty
+      end
+    end
+
+    context "with direct match (different case)" do
+      let(:wiki_text) { "== plot ==\nContent here." }
+      let(:plot_article) { Wp2txt::Article.new(wiki_text, "Film") }
+      let(:extractor) { described_class.new(["Plot"], track_matches: true) }
+
+      it "records case-different direct matches" do
+        extractor.extract_sections(plot_article)
+        expect(extractor.matched_sections["Plot"]).to eq("plot")
+      end
+    end
+  end
+end
+
+RSpec.describe Wp2txt::SectionStatsCollector do
+  let(:sample_wiki_text) do
+    <<~WIKI
+      Summary.
+      == Early life ==
+      Content.
+      == Career ==
+      Content.
+    WIKI
+  end
+
+  let(:another_wiki_text) do
+    <<~WIKI
+      Summary.
+      == Career ==
+      Content.
+      == Reception ==
+      Content.
+    WIKI
+  end
+
+  let(:article1) { Wp2txt::Article.new(sample_wiki_text, "Person 1") }
+  let(:article2) { Wp2txt::Article.new(another_wiki_text, "Work 1") }
+
+  describe "#process" do
+    it "counts articles" do
+      collector = described_class.new
+      collector.process(article1)
+      collector.process(article2)
+      expect(collector.total_articles).to eq(2)
+    end
+
+    it "counts section occurrences" do
+      collector = described_class.new
+      collector.process(article1)
+      collector.process(article2)
+
+      expect(collector.section_counts["Career"]).to eq(2)
+      expect(collector.section_counts["Early life"]).to eq(1)
+      expect(collector.section_counts["Reception"]).to eq(1)
+    end
+  end
+
+  describe "#top_sections" do
+    it "returns sections sorted by count" do
+      collector = described_class.new
+      collector.process(article1)
+      collector.process(article2)
+
+      top = collector.top_sections(2)
+      expect(top.first["name"]).to eq("Career")
+      expect(top.first["count"]).to eq(2)
+      expect(top.length).to eq(2)
+    end
+  end
+
+  describe "#to_hash" do
+    it "returns statistics as hash" do
+      collector = described_class.new
+      collector.process(article1)
+      collector.process(article2)
+
+      result = collector.to_hash(top_n: 5)
+      expect(result["total_articles"]).to eq(2)
+      expect(result["section_counts"]).to be_a(Hash)
+      expect(result["top_sections"]).to be_an(Array)
+    end
+  end
+
+  describe "#to_json" do
+    it "returns valid JSON" do
+      collector = described_class.new
+      collector.process(article1)
+
+      json = collector.to_json
+      parsed = JSON.parse(json)
+      expect(parsed["total_articles"]).to eq(1)
+    end
+  end
+end