wp2txt 1.1.3 → 2.1.0

data/spec/config_spec.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+require_relative "spec_helper"
+require_relative "../lib/wp2txt/config"
+require "tmpdir"
+require "fileutils"
+
+RSpec.describe Wp2txt::Config do
+  let(:tmpdir) { Dir.mktmpdir("wp2txt_config_test_") }
+  let(:config_path) { File.join(tmpdir, "config.yml") }
+
+  after do
+    FileUtils.rm_rf(tmpdir)
+  end
+
+  describe ".default_path" do
+    it "returns path in home directory" do
+      expect(described_class.default_path).to include(".wp2txt")
+      expect(described_class.default_path).to end_with("config.yml")
+    end
+  end
+
+  describe ".load" do
+    context "when config file does not exist" do
+      it "returns default configuration" do
+        config = described_class.load(config_path)
+
+        expect(config.dump_expiry_days).to eq 30
+        expect(config.category_expiry_days).to eq 7
+        expect(config.cache_directory).to eq Wp2txt::Config::DEFAULT_CACHE_DIR
+        expect(config.default_format).to eq "text"
+        expect(config.default_depth).to eq 0
+      end
+    end
+
+    context "when config file exists" do
+      it "loads settings from file" do
+        File.write(config_path, <<~YAML)
+          cache:
+            dump_expiry_days: 60
+            category_expiry_days: 14
+            directory: /custom/cache
+          defaults:
+            format: json
+            depth: 2
+        YAML
+
+        config = described_class.load(config_path)
+
+        expect(config.dump_expiry_days).to eq 60
+        expect(config.category_expiry_days).to eq 14
+        expect(config.cache_directory).to eq "/custom/cache"
+        expect(config.default_format).to eq "json"
+        expect(config.default_depth).to eq 2
+      end
+
+      it "uses defaults for missing keys" do
+        File.write(config_path, <<~YAML)
+          cache:
+            dump_expiry_days: 45
+        YAML
+
+        config = described_class.load(config_path)
+
+        expect(config.dump_expiry_days).to eq 45
+        expect(config.category_expiry_days).to eq 7  # default
+        expect(config.default_format).to eq "text"   # default
+      end
+
+      it "handles empty file" do
+        File.write(config_path, "")
+
+        config = described_class.load(config_path)
+
+        expect(config.dump_expiry_days).to eq 30
+      end
+
+      it "handles malformed YAML gracefully" do
+        File.write(config_path, "invalid: yaml: content: [")
+
+        config = described_class.load(config_path)
+
+        # Should return defaults on parse error
+        expect(config.dump_expiry_days).to eq 30
+      end
+    end
+  end
+
+  describe "#save" do
+    it "writes configuration to file" do
+      config = described_class.new(
+        dump_expiry_days: 45,
+        category_expiry_days: 10,
+        cache_directory: "/my/cache",
+        default_format: "json",
+        default_depth: 1
+      )
+
+      config.save(config_path)
+
+      expect(File.exist?(config_path)).to be true
+      loaded = described_class.load(config_path)
+      expect(loaded.dump_expiry_days).to eq 45
+      expect(loaded.category_expiry_days).to eq 10
+      expect(loaded.cache_directory).to eq "/my/cache"
+      expect(loaded.default_format).to eq "json"
+      expect(loaded.default_depth).to eq 1
+    end
+
+    it "creates parent directories if needed" do
+      nested_path = File.join(tmpdir, "nested", "dir", "config.yml")
+      config = described_class.new
+
+      config.save(nested_path)
+
+      expect(File.exist?(nested_path)).to be true
+    end
+  end
+
+  describe ".create_default" do
+    it "creates a config file with default values" do
+      described_class.create_default(config_path)
+
+      expect(File.exist?(config_path)).to be true
+      content = File.read(config_path)
+      expect(content).to include("dump_expiry_days: 30")
+      expect(content).to include("category_expiry_days: 7")
+    end
+
+    it "does not overwrite existing file" do
+      File.write(config_path, "custom: value")
+
+      result = described_class.create_default(config_path)
+
+      expect(result).to be false
+      expect(File.read(config_path)).to eq "custom: value"
+    end
+
+    it "can force overwrite with force option" do
+      File.write(config_path, "custom: value")
+
+      result = described_class.create_default(config_path, force: true)
+
+      expect(result).to be true
+      expect(File.read(config_path)).to include("dump_expiry_days: 30")
+    end
+  end
+
+  describe "#to_h" do
+    it "returns hash representation" do
+      config = described_class.new(dump_expiry_days: 45)
+
+      hash = config.to_h
+
+      expect(hash).to be_a(Hash)
+      expect(hash[:cache][:dump_expiry_days]).to eq 45
+      expect(hash[:defaults][:format]).to eq "text"
+    end
+  end
+
+  describe "validation" do
+    it "clamps dump_expiry_days to valid range" do
+      config = described_class.new(dump_expiry_days: -5)
+      expect(config.dump_expiry_days).to eq 1
+
+      config = described_class.new(dump_expiry_days: 400)
+      expect(config.dump_expiry_days).to eq 365
+    end
+
+    it "clamps category_expiry_days to valid range" do
+      config = described_class.new(category_expiry_days: 0)
+      expect(config.category_expiry_days).to eq 1
+
+      config = described_class.new(category_expiry_days: 100)
+      expect(config.category_expiry_days).to eq 90
+    end
+
+    it "clamps default_depth to valid range" do
+      config = described_class.new(default_depth: -1)
+      expect(config.default_depth).to eq 0
+
+      config = described_class.new(default_depth: 20)
+      expect(config.default_depth).to eq 10
+    end
+
+    it "validates default_format" do
+      config = described_class.new(default_format: "invalid")
+      expect(config.default_format).to eq "text"
+
+      config = described_class.new(default_format: "json")
+      expect(config.default_format).to eq "json"
+    end
+  end
+end

data/spec/constants_spec.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+require "tempfile"
+
+RSpec.describe "Wp2txt Constants" do
+  describe "Time Constants" do
+    it "defines SECONDS_PER_DAY" do
+      expect(Wp2txt::SECONDS_PER_DAY).to eq(86_400)
+    end
+
+    it "defines SECONDS_PER_HOUR" do
+      expect(Wp2txt::SECONDS_PER_HOUR).to eq(3_600)
+    end
+
+    it "defines SECONDS_PER_MINUTE" do
+      expect(Wp2txt::SECONDS_PER_MINUTE).to eq(60)
+    end
+  end
+
+  describe "Cache Settings" do
+    it "defines DEFAULT_DUMP_EXPIRY_DAYS" do
+      expect(Wp2txt::DEFAULT_DUMP_EXPIRY_DAYS).to eq(30)
+    end
+
+    it "defines DEFAULT_CATEGORY_CACHE_EXPIRY_DAYS" do
+      expect(Wp2txt::DEFAULT_CATEGORY_CACHE_EXPIRY_DAYS).to eq(7)
+    end
+
+  end
+
+  describe "Network Settings" do
+    it "defines DEFAULT_HTTP_TIMEOUT" do
+      expect(Wp2txt::DEFAULT_HTTP_TIMEOUT).to eq(30)
+    end
+
+    it "defines DEFAULT_PROGRESS_INTERVAL" do
+      expect(Wp2txt::DEFAULT_PROGRESS_INTERVAL).to eq(10)
+    end
+
+    it "defines INDEX_PROGRESS_THRESHOLD" do
+      expect(Wp2txt::INDEX_PROGRESS_THRESHOLD).to eq(500_000)
+    end
+
+    it "defines DEFAULT_TOP_N_SECTIONS" do
+      expect(Wp2txt::DEFAULT_TOP_N_SECTIONS).to eq(50)
+    end
+
+    it "defines RESUME_METADATA_MAX_AGE_DAYS" do
+      expect(Wp2txt::RESUME_METADATA_MAX_AGE_DAYS).to eq(7)
+    end
+
+    it "defines MAX_HTTP_RETRIES" do
+      expect(Wp2txt::MAX_HTTP_RETRIES).to eq(3)
+    end
+  end
+
+  describe "Processing Limits" do
+    it "defines MAX_NESTING_ITERATIONS" do
+      expect(Wp2txt::MAX_NESTING_ITERATIONS).to eq(50_000)
+    end
+
+    it "defines DEFAULT_BUFFER_SIZE" do
+      expect(Wp2txt::DEFAULT_BUFFER_SIZE).to eq(10_485_760) # 10 MB
+    end
+  end
+
+  describe "File Size Units" do
+    it "defines BYTES_PER_KB" do
+      expect(Wp2txt::BYTES_PER_KB).to eq(1024)
+    end
+
+    it "defines BYTES_PER_MB" do
+      expect(Wp2txt::BYTES_PER_MB).to eq(1024 * 1024)
+    end
+
+    it "defines BYTES_PER_GB" do
+      expect(Wp2txt::BYTES_PER_GB).to eq(1024 * 1024 * 1024)
+    end
+  end
+
+  describe ".days_to_seconds" do
+    it "converts days to seconds" do
+      expect(Wp2txt.days_to_seconds(1)).to eq(86_400)
+      expect(Wp2txt.days_to_seconds(7)).to eq(7 * 86_400)
+      expect(Wp2txt.days_to_seconds(0.5)).to eq(43_200)
+    end
+  end
+
+  describe ".file_fresh?" do
+    let(:temp_file) { Tempfile.new("test_file") }
+
+    after { temp_file.unlink }
+
+    it "returns true for recently created file" do
+      expect(Wp2txt.file_fresh?(temp_file.path, 1)).to be true
+    end
+
+    it "returns false for non-existent file" do
+      expect(Wp2txt.file_fresh?("/nonexistent/path", 1)).to be false
+    end
+  end
+
+  describe ".file_age_days" do
+    let(:temp_file) { Tempfile.new("test_file") }
+
+    after { temp_file.unlink }
+
+    it "returns age in days for existing file" do
+      age = Wp2txt.file_age_days(temp_file.path)
+      expect(age).to be_a(Float)
+      expect(age).to be >= 0
+      expect(age).to be < 1 # File just created
+    end
+
+    it "returns nil for non-existent file" do
+      expect(Wp2txt.file_age_days("/nonexistent/path")).to be_nil
+    end
+  end
+
+  describe ".format_file_size" do
+    it "formats bytes" do
+      expect(Wp2txt.format_file_size(500)).to eq("500 B")
+    end
+
+    it "formats kilobytes" do
+      expect(Wp2txt.format_file_size(2048)).to eq("2.0 KB")
+    end
+
+    it "formats megabytes" do
+      expect(Wp2txt.format_file_size(5_242_880)).to eq("5.0 MB")
+    end
+
+    it "formats gigabytes" do
+      expect(Wp2txt.format_file_size(2_147_483_648)).to eq("2.0 GB")
+    end
+  end
+end

data/spec/file_utils_spec.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+require "tempfile"
+require "fileutils"
+
+RSpec.describe "Wp2txt FileUtils" do
+  include Wp2txt
+
+  describe "collect_files" do
+    let(:temp_dir) { Dir.mktmpdir }
+
+    after { FileUtils.remove_entry(temp_dir) }
+
+    it "collects all files in a directory" do
+      # Create test files
+      File.write(File.join(temp_dir, "file1.txt"), "content1")
+      File.write(File.join(temp_dir, "file2.txt"), "content2")
+
+      files = collect_files(temp_dir)
+      expect(files).to include(a_string_ending_with("file1.txt"))
+      expect(files).to include(a_string_ending_with("file2.txt"))
+    end
+
+    it "filters files by regex" do
+      File.write(File.join(temp_dir, "file1.txt"), "content1")
+      File.write(File.join(temp_dir, "file2.rb"), "content2")
+
+      files = collect_files(temp_dir, /\.txt$/)
+      expect(files).to include(a_string_ending_with("file1.txt"))
+      expect(files).not_to include(a_string_ending_with("file2.rb"))
+    end
+
+    it "returns sorted list" do
+      File.write(File.join(temp_dir, "z_file.txt"), "")
+      File.write(File.join(temp_dir, "a_file.txt"), "")
+
+      files = collect_files(temp_dir, /\.txt$/)
+      txt_files = files.select { |f| f.end_with?(".txt") }
+      expect(txt_files).to eq(txt_files.sort)
+    end
+  end
+
+  describe "correct_separator" do
+    it "converts backslashes to forward slashes on non-Windows" do
+      skip "Only runs on non-Windows" if RUBY_PLATFORM.index("win32")
+      expect(correct_separator("path\\to\\file")).to eq("path/to/file")
+    end
+
+    it "handles arrays of paths" do
+      skip "Only runs on non-Windows" if RUBY_PLATFORM.index("win32")
+      result = correct_separator(["path\\to\\file1", "path\\to\\file2"])
+      expect(result).to eq(["path/to/file1", "path/to/file2"])
+    end
+
+    it "returns nil for nil input" do
+      expect(correct_separator(nil)).to be_nil
+    end
+  end
+
+  describe "sec_to_str" do
+    it "converts seconds to HH:MM:SS format" do
+      expect(sec_to_str(3661)).to eq("01:01:01")
+    end
+
+    it "handles zero" do
+      expect(sec_to_str(0)).to eq("00:00:00")
+    end
+
+    it "handles large values" do
+      expect(sec_to_str(86400)).to eq("24:00:00")  # 1 day
+    end
+
+    it "handles nil input" do
+      expect(sec_to_str(nil)).to eq("--:--:--")
+    end
+
+    it "formats with leading zeros" do
+      expect(sec_to_str(61)).to eq("00:01:01")
+    end
+  end
+
+  describe "file_mod" do
+    let(:temp_file) { Tempfile.new("test_file") }
+
+    after do
+      temp_file.close
+      temp_file.unlink
+      File.unlink("temp") if File.exist?("temp")
+    end
+
+    it "modifies file content using block" do
+      temp_file.write("original content")
+      temp_file.close
+
+      file_mod(temp_file.path) do |content|
+        content.upcase
+      end
+
+      expect(File.read(temp_file.path)).to eq("ORIGINAL CONTENT")
+    end
+
+    it "keeps backup when requested" do
+      temp_file.write("original")
+      temp_file.close
+
+      file_mod(temp_file.path, true) do |content|
+        "modified"
+      end
+
+      expect(File.exist?(temp_file.path + ".bak")).to be true
+      File.unlink(temp_file.path + ".bak")
+    end
+  end
+
+  describe "batch_file_mod" do
+    let(:temp_dir) { Dir.mktmpdir }
+
+    after { FileUtils.remove_entry(temp_dir) }
+
+    it "yields each file in directory" do
+      File.write(File.join(temp_dir, "file1.txt"), "")
+      File.write(File.join(temp_dir, "file2.txt"), "")
+
+      files_processed = []
+      batch_file_mod(temp_dir) do |file|
+        files_processed << File.basename(file)
+      end
+
+      expect(files_processed).to include("file1.txt", "file2.txt")
+    end
+
+    it "yields single file if path is a file" do
+      file_path = File.join(temp_dir, "single.txt")
+      File.write(file_path, "")
+
+      files_processed = []
+      batch_file_mod(file_path) do |file|
+        files_processed << file
+      end
+
+      expect(files_processed).to eq([file_path])
+    end
+  end
+
+  describe "rename" do
+    let(:temp_dir) { Dir.mktmpdir }
+
+    after { FileUtils.remove_entry(temp_dir) }
+
+    it "renames files with extension" do
+      file1 = File.join(temp_dir, "test-1")
+      file2 = File.join(temp_dir, "test-2")
+      File.write(file1, "")
+      File.write(file2, "")
+
+      rename([file1, file2], "txt")
+
+      expect(File.exist?(File.join(temp_dir, "test-1.txt"))).to be true
+      expect(File.exist?(File.join(temp_dir, "test-2.txt"))).to be true
+    end
+
+    it "returns true on success" do
+      file1 = File.join(temp_dir, "test-1")
+      File.write(file1, "")
+
+      expect(rename([file1])).to be true
+    end
+  end
+end

data/spec/fixtures/samples.rb

@@ -0,0 +1,181 @@
+# frozen_string_literal: true
+
+# Test samples for multilingual content
+module Wp2txt
+  module TestSamples
+    ENGLISH_ARTICLE = <<~WIKI
+      '''Test Article''' is an [[English language|English]] article.
+      == Section ==
+      [[File:Example.jpg|thumb|A description]]
+      [[Category:Tests]]
+      [[Category:Examples]]
+    WIKI
+
+    JAPANESE_ARTICLE = <<~WIKI
+      '''テスト記事'''は[[日本語]]の記事です。
+      == セクション ==
+      [[ファイル:Example.jpg|thumb|説明文]]
+      [[カテゴリ:テスト]]
+    WIKI
+
+    GERMAN_ARTICLE = <<~WIKI
+      '''Testartikel''' ist ein [[Deutsch|deutscher]] Artikel.
+      == Abschnitt ==
+      [[Datei:Bild.png|miniatur|Beschreibung]]
+      [[Kategorie:Test]]
+      #WEITERLEITUNG [[Andere Seite]]
+    WIKI
+
+    FRENCH_ARTICLE = <<~WIKI
+      '''Article de test''' est un [[Français|article français]].
+      == Section ==
+      [[Fichier:Image.jpg|vignette|Description]]
+      [[Catégorie:Test]]
+      #REDIRECTION [[Autre page]]
+    WIKI
+
+    CHINESE_ARTICLE = <<~WIKI
+      '''测试文章'''是一个[[中文]]文章。
+      == 章节 ==
+      [[文件:图片.jpg|缩略图|说明]]
+      [[分类:测试]]
+      #重定向 [[其他页面]]
+    WIKI
+
+    RUSSIAN_ARTICLE = <<~WIKI
+      '''Тестовая статья''' — это [[Русский язык|русская]] статья.
+      == Раздел ==
+      [[Файл:Изображение.jpg|мини|Описание]]
+      [[Категория:Тест]]
+      #ПЕРЕНАПРАВЛЕНИЕ [[Другая страница]]
+    WIKI
+
+    KOREAN_ARTICLE = <<~WIKI
+      '''테스트 문서'''는 [[한국어]] 문서입니다.
+      == 섹션 ==
+      [[파일:Example.jpg|섬네일|설명]]
+      [[분류:테스트]]
+      #넘겨주기 [[다른 문서]]
+    WIKI
+
+    ARABIC_ARTICLE = <<~WIKI
+      '''مقالة اختبار''' هي [[اللغة العربية|مقالة عربية]].
+      == قسم ==
+      [[ملف:صورة.jpg|تصغير|وصف]]
+      [[تصنيف:اختبار]]
+      #تحويل [[صفحة أخرى]]
+    WIKI
+
+    # Edge cases
+    EMOJI_CONTENT = "Text with emoji &#x1F600; and &#128512; symbols"
+    DEEPLY_NESTED = "{{a|{{b|{{c|{{d|text}}}}}}}}"
+    MALFORMED_MARKUP = "[[Unclosed link\n{{Unclosed template"
+
+    # Complex nested structure
+    NESTED_TEMPLATES = <<~WIKI
+      {{Infobox person
+      |name = Test Person
+      |birth_date = {{Birth date|1990|1|15}}
+      |occupation = [[Scientist]]
+      }}
+    WIKI
+
+    # Table content
+    TABLE_CONTENT = <<~WIKI
+      {| class="wikitable"
+      |-
+      ! Header 1 !! Header 2
+      |-
+      | Cell 1 || Cell 2
+      |}
+    WIKI
+
+    # Reference content
+    REFERENCE_CONTENT = <<~WIKI
+      This is text with a reference.<ref>Citation here</ref>
+      Another reference.<ref name="test">Named citation</ref>
+    WIKI
+
+    # Multi-line link
+    MULTILINE_LINK = <<~WIKI
+      [[File:Example.jpg
+      |thumb
+      |200px
+      |A very long caption that spans
+      multiple lines]]
+    WIKI
+
+    # === Additional Edge Cases for v2.0.0 ===
+
+    # Special characters in titles
+    SPECIAL_TITLE_ARTICLE = <<~WIKI
+      '''C++ (programming language)''' is a [[programming language]].
+      '''O'Brien''' was an [[Irish people|Irish]] person.
+      '''Rock & Roll''' is a music genre.
+      [[Category:Programming languages]]
+    WIKI
+
+    # Very deeply nested templates (10 levels)
+    VERY_DEEPLY_NESTED = "{{a|{{b|{{c|{{d|{{e|{{f|{{g|{{h|{{i|{{j|content}}}}}}}}}}}}}}}}}}}}"
+
+    # Mixed multilingual content with emoji
+    MIXED_CONTENT = <<~WIKI
+      '''Test''' こんにちは 你好 مرحبا Привет 😀
+      == Section セクション ==
+      Text with emoji: &#x1F600; &#x1F4BB; &#x2764;
+      [[Category:Test]][[カテゴリ:テスト]][[分类:测试]]
+    WIKI
+
+    # Complex wikilinks with pipes and brackets
+    COMPLEX_LINKS = <<~WIKI
+      [[File:Photo.jpg|thumb|200px|alt=Alt text|Caption with [[nested link]]]]
+      [[Article|Display text with '''bold''' and ''italic'']]
+      [[Category:Test|Sort key]]
+    WIKI
+
+    # Multiple consecutive templates
+    CONSECUTIVE_TEMPLATES = <<~WIKI
+      {{Stub}}{{Cleanup}}{{Unreferenced}}
+      This article needs work.
+      {{Infobox|title=Test}}
+    WIKI
+
+    # HTML entities mixed with character references
+    HTML_ENTITIES_MIXED = <<~WIKI
+      &nbsp;&lt;tag&gt;&amp;&quot;
+      &#60;literal&#62;
+      &#x3C;hex&#x3E;
+      Japanese: &#x65E5;&#x672C;&#x8A9E;
+    WIKI
+
+    # Horizontal rules (various lengths)
+    HORIZONTAL_RULES = <<~WIKI
+      Text before
+      ----
+      Text between
+      --------
+      Text after
+      --
+      Not a rule
+      ---
+      Also not a rule
+    WIKI
+
+    # Headings with various formatting
+    COMPLEX_HEADINGS = <<~WIKI
+      == Simple Heading ==
+      === Heading with [[link]] ===
+      ==== Heading with '''bold''' ====
+      ===== Heading with trailing space =====
+      == 日本語見出し ==
+    WIKI
+
+    # Redirect variations
+    REDIRECT_VARIATIONS = <<~WIKI
+      #REDIRECT [[Target]]
+      #redirect [[lowercase]]
+      #REDIRECT[[no space]]
+      #REDIRECT  [[extra spaces]]
+    WIKI
+  end
+end