wp2txt 1.1.3 → 2.1.0

data/spec/category_fetcher_spec.rb

@@ -0,0 +1,504 @@
+# frozen_string_literal: true
+
+require_relative "spec_helper"
+require "webmock/rspec"
+require "tmpdir"
+require "fileutils"
+
+RSpec.describe Wp2txt::CategoryFetcher do
+  let(:lang) { "en" }
+  let(:category) { "Japanese cities" }
+  let(:fetcher) { described_class.new(lang, category) }
+
+  before do
+    WebMock.enable!
+    WebMock.disable_net_connect!
+  end
+
+  after do
+    WebMock.disable!
+  end
+
+  describe "#initialize" do
+    it "accepts language and category name" do
+      expect(fetcher.lang).to eq "en"
+      expect(fetcher.category).to eq "Japanese cities"
+    end
+
+    it "normalizes category name by removing Category: prefix" do
+      f = described_class.new("en", "Category:Test Category")
+      expect(f.category).to eq "Test Category"
+    end
+
+    it "handles lowercase category prefix" do
+      f = described_class.new("en", "category:Another Test")
+      expect(f.category).to eq "Another Test"
+    end
+
+    it "trims whitespace from category name" do
+      f = described_class.new("en", "  Test Category  ")
+      expect(f.category).to eq "Test Category"
+    end
+
+    it "defaults max_depth to 0" do
+      expect(fetcher.max_depth).to eq 0
+    end
+
+    it "accepts custom max_depth" do
+      f = described_class.new("ja", "Test", max_depth: 3)
+      expect(f.max_depth).to eq 3
+    end
+  end
+
+  describe "#fetch_preview" do
+    it "returns statistics without full article list" do
+      stub_request(:get, /en\.wikipedia\.org/)
+        .to_return(
+          status: 200,
+          body: {
+            query: {
+              categorymembers: [
+                { ns: 0, title: "Tokyo" },
+                { ns: 0, title: "Osaka" },
+                { ns: 0, title: "Kyoto" }
+              ]
+            }
+          }.to_json
+        )
+
+      preview = fetcher.fetch_preview
+
+      expect(preview[:category]).to eq "Japanese cities"
+      expect(preview[:depth]).to eq 0
+      expect(preview[:total_articles]).to eq 3
+      expect(preview[:subcategories]).to be_an(Array)
+    end
+
+    it "includes subcategory statistics when depth > 0" do
+      f = described_class.new("en", "Japanese cities", max_depth: 1)
+
+      # Parent category
+      stub_request(:get, /cmtitle=Category:Japanese%20cities/)
+        .to_return(
+          status: 200,
+          body: {
+            query: {
+              categorymembers: [
+                { ns: 0, title: "Tokyo" },
+                { ns: 14, title: "Category:Cities in Kanto" }
+              ]
+            }
+          }.to_json
+        )
+
+      # Subcategory
+      stub_request(:get, /cmtitle=Category:Cities%20in%20Kanto/)
+        .to_return(
+          status: 200,
+          body: {
+            query: {
+              categorymembers: [
+                { ns: 0, title: "Yokohama" },
+                { ns: 0, title: "Chiba" }
+              ]
+            }
+          }.to_json
+        )
+
+      preview = f.fetch_preview
+
+      expect(preview[:total_articles]).to eq 3
+      expect(preview[:total_subcategories]).to eq 1
+      expect(preview[:subcategories].size).to eq 2
+    end
+  end
+
+  describe "#fetch_articles" do
+    it "fetches articles from single page response" do
+      stub_request(:get, /en\.wikipedia\.org/)
+        .to_return(
+          status: 200,
+          body: {
+            query: {
+              categorymembers: [
+                { ns: 0, title: "Tokyo" },
+                { ns: 0, title: "Osaka" },
+                { ns: 0, title: "Kyoto" }
+              ]
+            }
+          }.to_json
+        )
+
+      articles = fetcher.fetch_articles
+      expect(articles).to contain_exactly("Tokyo", "Osaka", "Kyoto")
+    end
+
+    it "handles pagination with cmcontinue token" do
+      # First request
+      stub_request(:get, /en\.wikipedia\.org/)
+        .with(query: hash_excluding("cmcontinue"))
+        .to_return(
+          status: 200,
+          body: {
+            query: {
+              categorymembers: [
+                { ns: 0, title: "Article1" },
+                { ns: 0, title: "Article2" }
+              ]
+            },
+            continue: { cmcontinue: "page2token" }
+          }.to_json
+        )
+
+      # Second request with continue token
+      stub_request(:get, /en\.wikipedia\.org/)
+        .with(query: hash_including("cmcontinue" => "page2token"))
+        .to_return(
+          status: 200,
+          body: {
+            query: {
+              categorymembers: [
+                { ns: 0, title: "Article3" }
+              ]
+            }
+          }.to_json
+        )
+
+      articles = fetcher.fetch_articles
+      expect(articles.size).to eq 3
+      expect(articles).to include("Article1", "Article2", "Article3")
+    end
+
+    it "returns unique articles when duplicates exist" do
+      stub_request(:get, /en\.wikipedia\.org/)
+        .to_return(
+          status: 200,
+          body: {
+            query: {
+              categorymembers: [
+                { ns: 0, title: "Tokyo" },
+                { ns: 0, title: "Tokyo" },
+                { ns: 0, title: "Osaka" }
+              ]
+            }
+          }.to_json
+        )
+
+      articles = fetcher.fetch_articles
+      expect(articles).to contain_exactly("Tokyo", "Osaka")
+    end
+
+    it "returns empty array for non-existent category" do
+      stub_request(:get, /en\.wikipedia\.org/)
+        .to_return(
+          status: 200,
+          body: { query: { categorymembers: [] } }.to_json
+        )
+
+      articles = fetcher.fetch_articles
+      expect(articles).to be_empty
+    end
+
+    it "handles API errors gracefully" do
+      stub_request(:get, /en\.wikipedia\.org/)
+        .to_return(status: 500)
+
+      articles = fetcher.fetch_articles
+      expect(articles).to be_empty
+    end
+
+    it "handles network timeout gracefully" do
+      stub_request(:get, /en\.wikipedia\.org/)
+        .to_timeout
+
+      articles = fetcher.fetch_articles
+      expect(articles).to be_empty
+    end
+
+    it "handles malformed JSON response gracefully" do
+      stub_request(:get, /en\.wikipedia\.org/)
+        .to_return(
+          status: 200,
+          body: "not valid json"
+        )
+
+      articles = fetcher.fetch_articles
+      expect(articles).to be_empty
+    end
+  end
+
+  describe "subcategory recursion" do
+    it "does not recurse into subcategories when max_depth is 0" do
+      stub_request(:get, /cmtitle=Category:Japanese%20cities/)
+        .to_return(
+          status: 200,
+          body: {
+            query: {
+              categorymembers: [
+                { ns: 0, title: "Tokyo" },
+                { ns: 14, title: "Category:Cities in Kanto" }
+              ]
+            }
+          }.to_json
+        )
+
+      articles = fetcher.fetch_articles
+      expect(articles).to eq ["Tokyo"]
+      expect(WebMock).not_to have_requested(:get, /cmtitle=Category:Cities%20in%20Kanto/)
+    end
+
+    it "recurses into subcategories when max_depth > 0" do
+      f = described_class.new("en", "Japanese cities", max_depth: 1)
+
+      # Parent category
+      stub_request(:get, /cmtitle=Category:Japanese%20cities/)
+        .to_return(
+          status: 200,
+          body: {
+            query: {
+              categorymembers: [
+                { ns: 0, title: "Tokyo" },
+                { ns: 14, title: "Category:Cities in Kanto" }
+              ]
+            }
+          }.to_json
+        )
+
+      # Subcategory
+      stub_request(:get, /cmtitle=Category:Cities%20in%20Kanto/)
+        .to_return(
+          status: 200,
+          body: {
+            query: {
+              categorymembers: [
+                { ns: 0, title: "Yokohama" },
+                { ns: 0, title: "Chiba" }
+              ]
+            }
+          }.to_json
+        )
+
+      articles = f.fetch_articles
+      expect(articles).to contain_exactly("Tokyo", "Yokohama", "Chiba")
+    end
+
+    it "prevents infinite loops with circular category references" do
+      f = described_class.new("en", "Category A", max_depth: 5)
+
+      stub_request(:get, /cmtitle=Category:Category%20A/)
+        .to_return(
+          status: 200,
+          body: {
+            query: {
+              categorymembers: [
+                { ns: 0, title: "Article1" },
+                { ns: 14, title: "Category:Category B" }
+              ]
+            }
+          }.to_json
+        )
+
+      stub_request(:get, /cmtitle=Category:Category%20B/)
+        .to_return(
+          status: 200,
+          body: {
+            query: {
+              categorymembers: [
+                { ns: 0, title: "Article2" },
+                { ns: 14, title: "Category:Category A" }
+              ]
+            }
+          }.to_json
+        )
+
+      # Should complete without infinite loop
+      expect { f.fetch_articles }.not_to raise_error
+      articles = f.fetch_articles
+      expect(articles).to include("Article1", "Article2")
+    end
+
+    it "respects max_depth limit" do
+      f = described_class.new("en", "Root", max_depth: 1)
+
+      # Root
+      stub_request(:get, /cmtitle=Category:Root/)
+        .to_return(
+          status: 200,
+          body: {
+            query: {
+              categorymembers: [
+                { ns: 14, title: "Category:Level1" }
+              ]
+            }
+          }.to_json
+        )
+
+      # Level 1
+      stub_request(:get, /cmtitle=Category:Level1/)
+        .to_return(
+          status: 200,
+          body: {
+            query: {
+              categorymembers: [
+                { ns: 0, title: "Article1" },
+                { ns: 14, title: "Category:Level2" }
+              ]
+            }
+          }.to_json
+        )
+
+      # Level 2 should not be called
+      stub_request(:get, /cmtitle=Category:Level2/)
+        .to_return(
+          status: 200,
+          body: {
+            query: { categorymembers: [{ ns: 0, title: "Article2" }] }
+          }.to_json
+        )
+
+      articles = f.fetch_articles
+      expect(articles).to eq ["Article1"]
+      expect(WebMock).not_to have_requested(:get, /cmtitle=Category:Level2/)
+    end
+  end
+
+  describe "caching" do
+    let(:cache_dir) { Dir.mktmpdir("wp2txt_test_") }
+
+    after do
+      FileUtils.rm_rf(cache_dir)
+    end
+
+    it "caches category members to SQLite database" do
+      f = described_class.new("en", "Test Category")
+      f.enable_cache(cache_dir)
+
+      stub_request(:get, /en\.wikipedia\.org/)
+        .to_return(
+          status: 200,
+          body: {
+            query: {
+              categorymembers: [{ ns: 0, title: "Cached Article" }]
+            }
+          }.to_json
+        )
+
+      f.fetch_articles
+
+      # SQLite cache file should exist
+      cache_files = Dir.glob(File.join(cache_dir, "categories_*.sqlite3"))
+      expect(cache_files.size).to eq 1
+    end
+
+    it "uses cached data on subsequent calls" do
+      f = described_class.new("en", "Test Category")
+      f.enable_cache(cache_dir)
+
+      # Pre-populate cache using CategoryCache
+      cache = Wp2txt::CategoryCache.new("en", cache_dir: cache_dir)
+      cache.save("Test Category", ["Cached Article"], [])
+      cache.close
+
+      # Should not make API request
+      articles = f.fetch_articles
+      expect(articles).to eq ["Cached Article"]
+      expect(WebMock).not_to have_requested(:get, /wikipedia\.org/)
+    end
+
+    it "ignores stale cache" do
+      f = described_class.new("en", "Test Category")
+      f.enable_cache(cache_dir)
+
+      # Pre-populate cache using CategoryCache
+      cache = Wp2txt::CategoryCache.new("en", cache_dir: cache_dir, expiry_days: 7)
+      cache.save("Test Category", ["Old Article"], [])
+
+      # Manually update cached_at to make it old (8 days ago)
+      cache.instance_variable_get(:@db).execute(
+        "UPDATE categories SET cached_at = ? WHERE name = ?",
+        [Time.now.to_i - (8 * 24 * 3600), "Test Category"]
+      )
+      cache.close
+
+      stub_request(:get, /en\.wikipedia\.org/)
+        .to_return(
+          status: 200,
+          body: {
+            query: {
+              categorymembers: [{ ns: 0, title: "Fresh Article" }]
+            }
+          }.to_json
+        )
+
+      articles = f.fetch_articles
+      expect(articles).to eq ["Fresh Article"]
+    end
+  end
+
+  describe "special characters in category names" do
+    it "handles spaces in category names" do
+      f = described_class.new("en", "Japanese cities")
+
+      stub_request(:get, /cmtitle=Category:Japanese%20cities/)
+        .to_return(
+          status: 200,
+          body: { query: { categorymembers: [{ ns: 0, title: "Tokyo" }] } }.to_json
+        )
+
+      articles = f.fetch_articles
+      expect(articles).to eq ["Tokyo"]
+    end
+
+    it "handles Unicode category names" do
+      f = described_class.new("ja", "日本の都市")
+
+      stub_request(:get, /ja\.wikipedia\.org/)
+        .to_return(
+          status: 200,
+          body: { query: { categorymembers: [{ ns: 0, title: "東京" }] } }.to_json
+        )
+
+      articles = f.fetch_articles
+      expect(articles).to eq ["東京"]
+    end
+
+    it "handles special characters in category names" do
+      f = described_class.new("en", "Rock & Roll")
+
+      stub_request(:get, /cmtitle=Category:Rock%20%26%20Roll/)
+        .to_return(
+          status: 200,
+          body: { query: { categorymembers: [{ ns: 0, title: "Elvis" }] } }.to_json
+        )
+
+      articles = f.fetch_articles
+      expect(articles).to eq ["Elvis"]
+    end
+  end
+
+  describe "multilingual support" do
+    it "works with Japanese Wikipedia" do
+      f = described_class.new("ja", "日本の都市")
+
+      stub_request(:get, /ja\.wikipedia\.org/)
+        .to_return(
+          status: 200,
+          body: { query: { categorymembers: [{ ns: 0, title: "東京" }] } }.to_json
+        )
+
+      expect(f.fetch_articles).to eq ["東京"]
+    end
+
+    it "works with German Wikipedia" do
+      f = described_class.new("de", "Stadt in Deutschland")
+
+      stub_request(:get, /de\.wikipedia\.org/)
+        .to_return(
+          status: 200,
+          body: { query: { categorymembers: [{ ns: 0, title: "Berlin" }] } }.to_json
+        )
+
+      expect(f.fetch_articles).to eq ["Berlin"]
+    end
+  end
+end

data/spec/cleanup_spec.rb

@@ -0,0 +1,197 @@
+# frozen_string_literal: true
+
+require_relative "spec_helper"
+require_relative "../lib/wp2txt/utils"
+
+RSpec.describe "Wp2txt Cleanup" do
+  include Wp2txt
+
+  describe "MediaWiki magic words" do
+    it "removes DEFAULTSORT lines" do
+      input = "Some text\nDEFAULTSORT:にんちけんこかく\nMore text"
+      result = cleanup(input)
+      expect(result).not_to include("DEFAULTSORT")
+      expect(result).to include("Some text")
+      expect(result).to include("More text")
+    end
+
+    it "removes DISPLAYTITLE lines" do
+      input = "Some text\nDISPLAYTITLE:Custom Title\nMore text"
+      result = cleanup(input)
+      expect(result).not_to include("DISPLAYTITLE")
+    end
+
+    it "removes __NOTOC__ and similar" do
+      input = "Some text\n__NOTOC__\n__TOC__\n__FORCETOC__\nMore text"
+      result = cleanup(input)
+      expect(result).not_to include("__NOTOC__")
+      expect(result).not_to include("__TOC__")
+      expect(result).not_to include("__FORCETOC__")
+    end
+
+    it "removes __NOEDITSECTION__" do
+      input = "Some text\n__NOEDITSECTION__\nMore text"
+      result = cleanup(input)
+      expect(result).not_to include("__NOEDITSECTION__")
+    end
+  end
+
+  describe "Interwiki links" do
+    it "removes :en: prefixed links" do
+      input = "See :en:Force dynamics for more"
+      result = cleanup(input)
+      expect(result).to include("Force dynamics")
+      expect(result).not_to include(":en:")
+    end
+
+    it "removes :fr: prefixed links" do
+      input = "See :fr:Société de Linguistique de Paris"
+      result = cleanup(input)
+      expect(result).to include("Société de Linguistique de Paris")
+      expect(result).not_to include(":fr:")
+    end
+
+    it "removes :de: prefixed links" do
+      input = "Related: :de:Sprachwissenschaft"
+      result = cleanup(input)
+      expect(result).to include("Sprachwissenschaft")
+      expect(result).not_to include(":de:")
+    end
+
+    it "handles multiple interwiki links" do
+      input = "See :en:Article1 and :fr:Article2 for details"
+      result = cleanup(input)
+      expect(result).to include("Article1")
+      expect(result).to include("Article2")
+      expect(result).not_to match(/:[a-z]{2}:/)
+    end
+  end
+
+  describe "Authority control templates" do
+    it "removes Normdaten line" do
+      input = "Some text\nNormdaten\nMore text"
+      result = cleanup(input)
+      expect(result).not_to include("Normdaten")
+    end
+
+    it "removes Authority control line" do
+      input = "Some text\nAuthority control\nMore text"
+      result = cleanup(input)
+      expect(result).not_to include("Authority control")
+    end
+
+    it "removes Persondata line" do
+      input = "Some text\nPersondata\nMore text"
+      result = cleanup(input)
+      expect(result).not_to include("Persondata")
+    end
+  end
+
+  describe "Category line cleanup" do
+    it "removes standalone Category: lines (English)" do
+      input = "Text\nCategory:Linguistics\nCategory:Science\nMore"
+      result = cleanup(input)
+      expect(result).not_to match(/^Category:/)
+    end
+
+    it "removes standalone カテゴリ lines (Japanese)" do
+      input = "Text\nカテゴリ:言語学\nMore"
+      result = cleanup(input)
+      expect(result).not_to match(/^カテゴリ:/)
+    end
+
+    it "removes standalone Kategorie lines (German)" do
+      input = "Text\nKategorie:Sprachwissenschaft\nMore"
+      result = cleanup(input)
+      expect(result).not_to match(/^Kategorie:/)
+    end
+
+    it "removes standalone Catégorie lines (French)" do
+      input = "Text\nCatégorie:Linguistique\nMore"
+      result = cleanup(input)
+      expect(result).not_to match(/^Catégorie:/)
+    end
+
+    it "removes Category lines with asterisk prefix" do
+      input = "Text\n*\nCategory:Main\nMore"
+      result = cleanup(input)
+      expect(result).not_to match(/^Category:/)
+    end
+
+    it "preserves CATEGORIES summary line" do
+      input = "Text\nCATEGORIES: Foo, Bar, Baz\nMore"
+      result = cleanup(input)
+      expect(result).to include("CATEGORIES: Foo, Bar, Baz")
+    end
+  end
+
+  describe "Template artifact cleanup" do
+    it "removes stub template markers" do
+      # Common stub patterns across languages
+      input = "Text\n節スタブ\nMore"
+      result = cleanup(input)
+      # This might be Japanese-specific, but the pattern should be general
+    end
+
+    it "removes reference help markers" do
+      input = "Text\n脚注ヘルプ\nMore"
+      result = cleanup(input)
+      # Japanese-specific, need general approach
+    end
+
+    it "removes lines that are just asterisk + single word" do
+      input = "Text\n*和書\n*洋書\nMore"
+      result = cleanup(input)
+      # Pattern: ^\*[^\s\*]+$ (single word after asterisk)
+    end
+
+    it "removes Wikibooks/Wikiversity markers" do
+      input = "Text\nWikibooks\nSchool:言語学\nMore"
+      result = cleanup(input)
+      expect(result).not_to match(/^Wikibooks$/)
+      expect(result).not_to match(/^School:/)
+    end
+
+    it "removes commons/wikimedia markers" do
+      input = "Text\nCommons\nWikimedia Commons\nMore"
+      result = cleanup(input)
+      expect(result).not_to match(/^Commons$/)
+    end
+  end
+
+  describe "Combined cleanup" do
+    it "cleans up a realistic Wikipedia article footer" do
+      input = <<~TEXT
+        This is the main content.
+
+        == References ==
+        脚注ヘルプ
+
+        == External links ==
+        Wikibooks
+        School:言語学
+
+        Normdaten
+        DEFAULTSORT:けんこかく
+        Category:言語学
+        Category:人文科学
+        *
+
+        CATEGORIES: 言語学, 人文科学
+      TEXT
+
+      result = cleanup(input)
+
+      expect(result).to include("This is the main content")
+      expect(result).to include("== References ==")
+      expect(result).to include("== External links ==")
+      expect(result).to include("CATEGORIES: 言語学, 人文科学")
+
+      expect(result).not_to include("Normdaten")
+      expect(result).not_to include("DEFAULTSORT")
+      expect(result).not_to match(/^Category:/)
+      expect(result).not_to match(/^Wikibooks$/)
+      expect(result).not_to match(/^School:/)
+    end
+  end
+end