wp2txt 2.1.2 → 2.3.0

data/lib/wp2txt/multistream.rb

@@ -233,8 +233,9 @@ module Wp2txt
     def initialize(multistream_path, index_or_path, use_cache: true, cache_dir: nil)
       @multistream_path = multistream_path
 
-      # Accept either an existing index or a path to create one
-      if index_or_path.is_a?(MultistreamIndex)
+      # Accept an existing index (or any object with the same lookup interface,
+      # e.g. a lazy SQLite-backed one) or a path to create one
+      if index_or_path.respond_to?(:find_by_title)
         @index = index_or_path
       else
         @index = MultistreamIndex.new(index_or_path, use_cache: use_cache, cache_dir: cache_dir)
@@ -365,10 +366,13 @@ module Wp2txt
     end
 
     def find_next_offset(current_offset)
-      idx = @index.stream_offsets.index(current_offset)
-      return nil unless idx
+      offsets = @index.stream_offsets
+      idx = offsets.index(current_offset)
+      # A missing offset means a corrupt or empty stream index; returning nil
+      # here would silently read gigabytes to EOF, so fail fast instead
+      raise "Stream offset #{current_offset} not found in index (#{offsets.size} streams known)" unless idx
 
-      @index.stream_offsets[idx + 1]
+      offsets[idx + 1]
     end
 
     def decompress_bz2(data)
@@ -822,6 +826,35 @@ module Wp2txt
       File.join(@cache_dir, "#{@lang}wiki-#{latest_dump_date}-multistream.xml.bz2")
     end
 
+    # URL of the langlinks dump for an explicit dump date
+    def langlinks_url(date)
+      wiki = "#{@lang}wiki"
+      "#{DUMP_BASE_URL}/#{wiki}/#{date}/#{wiki}-#{date}-langlinks.sql.gz"
+    end
+
+    # Cache path of the langlinks dump for an explicit dump date
+    def cached_langlinks_path(date)
+      File.join(@cache_dir, "#{@lang}wiki-#{date}-langlinks.sql.gz")
+    end
+
+    # Download the langlinks dump for an explicit dump date. The date MUST
+    # come from the built metadata index (not latest_dump_date) so the
+    # imported links stay pinned to the indexed dump version.
+    def download_langlinks(date:, force: false)
+      path = cached_langlinks_path(date)
+      if File.exist?(path) && !force
+        puts "Langlinks already cached: #{File.basename(path)}"
+        $stdout.flush
+        return path
+      end
+
+      url = langlinks_url(date)
+      puts "Downloading langlinks: #{url}"
+      $stdout.flush
+      download_file(url, path)
+      path
+    end
+
     # Check if cache is fresh (within configured days)
     def cache_fresh?(days = nil)
       days ||= @dump_expiry_days

data/lib/wp2txt/output_path.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Wp2txt
+  # Server-side output path confinement shared by the file-writing tools
+  # (extract_corpus / start_extract_job / query_sql with output_path).
+  # Paths must resolve under the server's output directory — an agent mixing
+  # up paths must not be able to clobber arbitrary user files — and existing
+  # files are not replaced unless overwrite is set.
+  module OutputPath
+    module_function
+
+    # @return [String] the confined, absolute output path
+    # @raise [ArgumentError] when the path escapes base_dir or the file exists
+    def confine(output_path, base_dir, overwrite: false)
+      path = File.expand_path(output_path, base_dir)
+      base = File.expand_path(base_dir)
+      unless path == base || path.start_with?(base + File::SEPARATOR)
+        raise ArgumentError, "output_path must stay within the server output directory (#{base})"
+      end
+      if File.exist?(path) && !overwrite
+        raise ArgumentError, "output file already exists: #{path} (pass overwrite: true to replace it)"
+      end
+
+      path
+    end
+  end
+end

data/lib/wp2txt/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Wp2txt
-  VERSION = "2.1.2"
+  VERSION = "2.3.0"
 end

data/spec/auto_download_spec.rb

@@ -33,6 +33,26 @@ RSpec.describe "Wp2txt Auto Download" do
       end
     end
 
+    describe "langlinks paths" do
+      it "builds URL and cache path for an explicit dump date (no network)" do
+        manager = Wp2txt::DumpManager.new(:ja, cache_dir: cache_dir)
+        expect(manager.langlinks_url("20260101")).to eq(
+          "https://dumps.wikimedia.org/jawiki/20260101/jawiki-20260101-langlinks.sql.gz"
+        )
+        expect(manager.cached_langlinks_path("20260101")).to eq(
+          File.join(cache_dir, "jawiki-20260101-langlinks.sql.gz")
+        )
+      end
+
+      it "reuses a cached langlinks file without downloading" do
+        manager = Wp2txt::DumpManager.new(:ja, cache_dir: cache_dir)
+        path = manager.cached_langlinks_path("20260101")
+        File.write(path, "cached")
+        expect(manager.download_langlinks(date: "20260101")).to eq(path)
+        expect(File.read(path)).to eq("cached")
+      end
+    end
+
     describe "#cache_status" do
       it "returns status hash with expected keys" do
         manager = Wp2txt::DumpManager.new(:ja, cache_dir: cache_dir)
@@ -285,6 +305,63 @@ RSpec.describe "Wp2txt Auto Download" do
           end
         end
       end
+
+      context "--import-langlinks option" do
+        it "accepts --import-langlinks with --lang" do
+          opts = Wp2txt::CLI.parse_options(["--lang=ja", "--import-langlinks", "--cache-dir=#{cache_dir}"])
+          expect(opts[:import_langlinks]).to be true
+        end
+
+        it "requires --lang" do
+          suppress_stderr do
+            expect { Wp2txt::CLI.parse_options(["--import-langlinks", "--cache-dir=#{cache_dir}"]) }.to raise_error(SystemExit)
+          end
+        end
+
+        it "cannot be combined with --build-index" do
+          suppress_stderr do
+            expect do
+              Wp2txt::CLI.parse_options(["--lang=ja", "--import-langlinks", "--build-index", "--cache-dir=#{cache_dir}"])
+            end.to raise_error(SystemExit)
+          end
+        end
+
+        it "accepts --langlinks-file with an existing file" do
+          file = File.join(cache_dir, "jawiki-20260101-langlinks.sql")
+          File.write(file, "")
+          opts = Wp2txt::CLI.parse_options(["--lang=ja", "--import-langlinks", "--langlinks-file=#{file}", "--cache-dir=#{cache_dir}"])
+          expect(opts[:langlinks_file]).to eq(file)
+        end
+
+        it "rejects --langlinks-file without --import-langlinks" do
+          suppress_stderr do
+            expect do
+              Wp2txt::CLI.parse_options(["--lang=ja", "--langlinks-file=x.sql", "--cache-dir=#{cache_dir}"])
+            end.to raise_error(SystemExit)
+          end
+        end
+
+        it "rejects a missing --langlinks-file" do
+          suppress_stderr do
+            expect do
+              Wp2txt::CLI.parse_options(["--lang=ja", "--import-langlinks", "--langlinks-file=#{cache_dir}/nope.sql", "--cache-dir=#{cache_dir}"])
+            end.to raise_error(SystemExit)
+          end
+        end
+
+        it "accepts --langlinks-langs as a comma-separated list" do
+          opts = Wp2txt::CLI.parse_options(["--lang=ja", "--import-langlinks", "--langlinks-langs=en,de,fr", "--cache-dir=#{cache_dir}"])
+          expect(opts[:langlinks_langs]).to eq("en,de,fr")
+        end
+
+        it "rejects invalid language codes in --langlinks-langs" do
+          suppress_stderr do
+            expect do
+              Wp2txt::CLI.parse_options(["--lang=ja", "--import-langlinks", "--langlinks-langs=en,../x", "--cache-dir=#{cache_dir}"])
+            end.to raise_error(SystemExit)
+          end
+        end
+      end
     end
   end