arxiv-dl 0.2.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 36af277147beae3a55221ba50443204db5b9e4bdae1613a3d9e923a465d4bc31
-  data.tar.gz: 04d405743856249d2f78d4f9e711e73b716ddb27f1b6633ad92a96e6385677be
+  metadata.gz: 14d23483457b1dbb20c8aa9fa1384e52caf7144cab5ed357450f235dbcd29b80
+  data.tar.gz: 4d7255130c208f959bb0bb7249a33b3a076a362da7894dc370e480f2511109b7
 SHA512:
-  metadata.gz: 92527994ad88aec89a64f3f678f60ff8a9d286d3189d06a6eafa4a9ba4f5a02fa815e9d23c0e429c0ea44ad03d3d58ae870c79a001c0705a682fd22818210412
-  data.tar.gz: 5058f511ec0e2707275605bee72d29d44c190e563e9b83da65e8b8619199186345c78231b35aa685fe183d6c0dd6f6996115059fd72c991c8d7babb39dfe7be8
+  metadata.gz: '0572297cdb2a74bb5cb46cdcd8449e2e41611a9649493e8ba5bc22b2fcb84d264e5982f114d2cff640cfd76e1761de592c8a472815ee6a8116b70f35807a11a0'
+  data.tar.gz: 8fd1844b84d8fc2044b8d92078ad2dae136713c93768269b64fe9a83b169a9da37466661dbaf61bb12f81d2ba827276f9c56e00291b978e8bdf7524de4a858ad

data/CHANGELOG.md

@@ -1,3 +1,16 @@
+## [0.3.1]
+
+- Built on [dl-core](https://github.com/xoengineering/dl-core) 0.1: the HTTP client, errors, `Author`, `Slug`, sidecar writers, and CLI now come from it instead of copies. No change in behavior or output. `Arxiv::Downloader::Client`, `HTTPError`, `Author`, `Slug`, and `Metadata::YAML`/`JSON` still work; they are now `dl-core`'s classes. `Arxiv::Downloader::Error` now subclasses `DL::Core::Error`.
+- Direct dependencies on `http`, `stringex`, and `ostruct` replaced by `dl-core`.
+
+## [0.3.0]
+
+Less folder nesting for the common case: most papers only ever have one version.
+
+### Breaking
+
+- Single-version papers are archived flat again: a paper whose only archived version is v1 keeps its files directly in the paper folder, without a `v1/` level. `v<N>/` folders are used only when a paper has more than one version. Archiving a second version of a flat paper moves the existing files into `v<N>/` first, rewriting `html/` links into `_shared/` for the new depth. A paper whose latest version is v2 or later starts out in `v<N>/` folders. Existing 0.2.0 archives keep their `v1/` folders.
+
 ## [0.2.0]
 
 Versioned archives, author affiliations, and a round of robustness fixes. Two breaking changes to the output layout and metadata; see below.

data/README.md

@@ -108,27 +108,40 @@ $ARXIV_DOWNLOAD_PATH/                   # default: $HOME/Downloads/ArXiv_Papers
     arxiv.org/static/...
     cdn.jsdelivr.net/...
   YYYY/MM/DD/<primary_category>/<arxiv-id>-<slug>/
-    v<N>/                               # one folder per archived version
-      <arxiv-id>v<N>.pdf
-      <arxiv-id>v<N>-abstract.html
-      metadata.md                       # YAML frontmatter + Markdown body
-      metadata.yaml
-      metadata.json
-      metadata.bib                      # upstream BibTeX, falls back to synthesized
-      html/                             # absent when the paper has no HTML version
-        <arxiv-id>v<N>.html             # path-rewritten to local assets
-        x1.png, x2.png, ...             # paper-specific images
-      src/                              # absent for PDF-only submissions
-        *.tex, *.bbl, ...               # extracted from /src/<id>v<N>
+    <arxiv-id>v<N>.pdf
+    <arxiv-id>v<N>-abstract.html
+    metadata.md                         # YAML frontmatter + Markdown body
+    metadata.yaml
+    metadata.json
+    metadata.bib                        # upstream BibTeX, falls back to synthesized
+    html/                               # absent when the paper has no HTML version
+      <arxiv-id>v<N>.html               # path-rewritten to local assets
+      x1.png, x2.png, ...               # paper-specific images
+    src/                                # absent for PDF-only submissions
+      *.tex, *.bbl, ...                 # extracted from /src/<id>v<N>
 ```
 
-An unversioned ID (`2508.16190`) archives the latest version. A versioned ID (`2508.16190v1`) archives that version. Different versions of the same paper sit side by side under the same paper folder.
+An unversioned ID (`2508.16190`) archives the latest version. A versioned ID (`2508.16190v1`) archives that version.
 
-Each version downloads into `v<N>.partial/` and is renamed to `v<N>/` only when every file succeeded. Re-running skips versions whose `v<N>/` already exists and retries interrupted ones from scratch.
+A paper with a single archived version v1 is kept flat, as above. When a paper has more than one version, each version gets its own `v<N>/` folder with the same contents:
+
+```txt
+  YYYY/MM/DD/<primary_category>/<arxiv-id>-<slug>/
+    v1/
+      <arxiv-id>v1.pdf
+      ...
+    v2/
+      <arxiv-id>v2.pdf
+      ...
+```
+
+Archiving a second version of a flat paper first moves the existing files into `v<N>/` (rewriting `html/` links into `_shared/` for the new depth). A paper whose latest version is v2 or later starts out in `v<N>/` folders.
+
+Each version downloads into a sibling `.partial` folder and is renamed into place only when every file succeeded. Re-running skips versions already archived and retries interrupted ones from scratch.
 
 `YYYY/MM/DD` is the original submission date. `<primary_category>` is from the paper's metadata (`cs.CL`, `math.NT`, etc). `<slug>` is derived from the paper title (Unicode → ASCII, hyphenated, truncated to 80 chars at a word boundary).
 
-For legacy IDs containing `/` (e.g. `cs/0002001`), the slash is replaced with `-` in the directory name and file names (`cs-0002001-.../v1/cs-0002001v1.pdf`).
+For legacy IDs containing `/` (e.g. `cs/0002001`), the slash is replaced with `-` in the directory name and file names (`cs-0002001-.../cs-0002001v1.pdf`).
 
 ## Library usage
 
@@ -138,7 +151,7 @@ require 'arxiv/downloader'
 identifier = Arxiv::Downloader::Identifier.new '2508.16190'
 client     = Arxiv::Downloader::Client.new                # 3-second rate limit by default
 path       = Arxiv::Downloader::Archive.new(identifier, root: '/tmp/papers', client: client).run
-# => "/tmp/papers/2025/08/22/cs.CL/2508.16190-comicscene154-a-scene-dataset-for-comic-analysis/v1"
+# => "/tmp/papers/2025/08/22/cs.CL/2508.16190-comicscene154-a-scene-dataset-for-comic-analysis"
 ```
 
 ## Development

data/lib/arxiv/downloader/archive.rb

@@ -9,12 +9,14 @@ module Arxiv
         @client     = client
       end
 
-      # Downloads into v<N>.partial/ and
-      # renames it to v<N>/ only once everything succeeded.
-      # An existing v<N>/ is always complete and is skipped.
+      # Downloads into <destination>.partial/ and
+      # renames it into place only once everything succeeded.
+      # An archived version is always complete and is skipped.
+      # PaperFolder decides flat (single version) vs v<N>/ (several versions).
       def run
-        return paper_dir if Dir.exist? paper_dir
+        return paper_folder.location_of(metadata.version) if paper_folder.archived? metadata.version
 
+        paper_folder.unflatten!
         FileUtils.rm_rf staging_dir
         FileUtils.mkdir_p staging_dir
 
@@ -24,8 +26,8 @@ module Arxiv
         download_source_archive
         write_sidecars
 
-        File.rename staging_dir, paper_dir
-        paper_dir
+        File.rename staging_dir, destination
+        destination
       end
 
       private
@@ -44,13 +46,18 @@ module Arxiv
         @archived ||= Identifier.new "#{metadata.arxiv_id}v#{metadata.version}"
       end
 
-      def paper_dir
-        @paper_dir ||= File.join @root, Path.new(metadata).to_s, "v#{metadata.version}"
+      def paper_folder
+        @paper_folder ||= PaperFolder.new File.join(@root, Path.new(metadata).to_s)
       end
 
-      # a sibling of paper_dir, so relative ../_shared/ refs survive the rename
+      # evaluated after unflatten!, which can turn a flat folder into v<N>/ folders
+      def destination
+        @destination ||= paper_folder.destination_for metadata.version
+      end
+
+      # a sibling of destination, so relative ../_shared/ refs survive the rename
       def staging_dir
-        "#{paper_dir}.partial"
+        "#{destination}.partial"
       end
 
       def download_pdf

data/lib/arxiv/downloader/author.rb

@@ -1,11 +1,5 @@
 module Arxiv
   module Downloader
-    Author = Data.define :name, :affiliations do
-      def initialize name:, affiliations: []
-        super
-      end
-
-      def to_s = name
-    end
+    Author = DL::Core::Author
   end
 end

data/lib/arxiv/downloader/cli.rb

@@ -1,106 +1,24 @@
-require 'optparse'
-
 module Arxiv
   module Downloader
-    class CLI
-      DEFAULT_DOWNLOAD_PATH = File.join Dir.home, 'Downloads', 'ArXiv_Papers'
-      USAGE                 = 'Usage: arxiv-dl [options] <ARXIV_ID_OR_URL> [<ARXIV_ID_OR_URL>...]'.freeze
-
-      def initialize argv, stderr: $stderr, stdin: $stdin, stdout: $stdout
-        @argv   = argv
-        @stderr = stderr
-        @stdin  = stdin
-        @stdout = stdout
-      end
-
-      def run
-        options = parse
-        return options.fetch(:exit_status) if options.key? :exit_status
-
-        return error_with USAGE       if options[:targets].empty?
-        return error_with conflict if options[:verbose] && options[:quiet]
-
-        failures = download_each options
-        failures.zero? ? 0 : 1
-      end
-
-      private
-
-      def conflict
-        '-v and -q are mutually exclusive'
-      end
-
-      def parse
-        options = { targets: [], verbose: false, quiet: false }
-        parser  = build_parser options
-
-        begin
-          parser.parse! @argv
-          options[:targets] = @argv + input_targets(options[:input])
-        rescue OptionParser::ParseError, SystemCallError => e
-          @stderr.puts e.message
-          return { exit_status: 1 }
-        end
-
-        options[:path]       ||= ENV['ARXIV_DOWNLOAD_PATH'] || DEFAULT_DOWNLOAD_PATH
-        options[:rate_limit] ||= (ENV['ARXIV_RATE_LIMIT'] || Client::DEFAULT_RATE_LIMIT).to_i
-        options
-      end
+    class CLI < DL::Core::CLI
+      def program = 'arxiv-dl'
 
-      def build_parser options
-        OptionParser.new do |parser|
-          parser.banner = USAGE
-          parser.on('-i FILE', '--input FILE')       { |value| options[:input] = value }
-          parser.on('-p PATH', '--path PATH')        { |value| options[:path] = value }
-          parser.on('--rate-limit SECONDS', Integer) { |value| options[:rate_limit] = value }
-          parser.on('-v', '--verbose')               { options[:verbose] = true }
-          parser.on('-q', '--quiet')                 { options[:quiet]   = true }
-          parser.on('--version') do
-            @stdout.puts VERSION
-            options[:exit_status] = 0
-          end
-          parser.on('-h', '--help') do
-            @stdout.puts parser.help
-            options[:exit_status] = 0
-          end
-        end
-      end
+      def target_name = 'ARXIV_ID_OR_URL'
 
-      # one target per line from FILE, or stdin for "-"; blank lines and # comments skipped
-      def input_targets input
-        return [] if input.nil?
+      # ARXIV_DOWNLOAD_PATH, ARXIV_RATE_LIMIT
+      def env_prefix = 'ARXIV'
 
-        text = input == '-' ? @stdin.read : File.read(input)
-        text.lines.map(&:strip).reject { it.empty? || it.start_with?('#') }
-      end
+      def default_path = File.join(Dir.home, 'Downloads', 'ArXiv_Papers')
 
-      def error_with message
-        @stderr.puts message
-        1
-      end
+      def version = VERSION
 
-      def download_each options
-        client = Client.new rate_limit: options[:rate_limit], log: (options[:verbose] ? @stdout : nil)
+      def user_agent = Client::USER_AGENT
 
-        failures = 0
-        options[:targets].each do |target|
-          failures += 1 unless download_one(target, client:, options:)
-        end
-        failures
-      end
+      def client_for(rate_limit:, log:) = Client.new(rate_limit:, log:)
 
-      # true on success; reports the failure and returns false otherwise
-      def download_one target, client:, options:
-        identifier = Identifier.new target
-        @stdout.puts "==> Downloading #{identifier.id}" if options[:verbose]
+      def identifier_for(target) = Identifier.new(target)
 
-        path = Archive.new(identifier, root: options[:path], client: client).run
-        @stdout.puts path unless options[:quiet]
-        true
-      rescue Error, HTTP::Error => e
-        @stderr.puts "#{target}: #{e.message}"
-        false
-      end
+      def archive_for(identifier, root:, client:) = Archive.new(identifier, root:, client:)
     end
   end
 end

data/lib/arxiv/downloader/client.rb

@@ -1,85 +1,12 @@
-require 'http'
-
 module Arxiv
   module Downloader
-    class Client
-      SOURCE_URL         = 'https://github.com/xoengineering/arxiv-dl'.freeze
-      DEFAULT_RATE_LIMIT = 3
-      TIMEOUTS           = { connect: 10, read: 60, write: 10 }.freeze # seconds, per operation
-      MAX_RETRIES        = 3
-      RETRY_BACKOFF      = 10 # seconds before the first retry. doubles on each retry.
-      RETRYABLE_STATUSES = [429, 503].freeze
-
-      attr_reader :rate_limit
-
-      def initialize rate_limit: DEFAULT_RATE_LIMIT, log: nil
-        @rate_limit = rate_limit
-        @log        = log
-      end
-
-      def user_agent
-        "arxiv-dl/#{VERSION} (+#{SOURCE_URL})"
-      end
-
-      def get url
-        retries = 0
-
-        loop do
-          response = request url
-          return response if response.status.success?
-          raise http_error(url, response) unless retryable? response, retries
-
-          retries += 1
-          wait_before_retry response, retries
-        end
-      end
-
-      private
-
-      def request url
-        throttle
-        response = HTTP.timeout(TIMEOUTS).headers('User-Agent' => user_agent).follow.get(url)
-        @last_request_at = Time.now
-        log_request url, response
-        response
-      end
-
-      def http_error url, response
-        HTTPError.new status: response.status.code, url: url, reason: response.status.reason
-      end
-
-      def retryable? response, retries
-        RETRYABLE_STATUSES.include?(response.status.code) && retries < MAX_RETRIES
-      end
-
-      def wait_before_retry response, retries
-        seconds = retry_after(response) || (RETRY_BACKOFF * (2**(retries - 1)))
-        @log&.puts "==> #{response.status}. Retrying in #{seconds}s"
-        sleep seconds
-      end
-
-      # Retry-After in delay-seconds form. The HTTP-date form falls back to backoff.
-      def retry_after response
-        value = response.headers['Retry-After']
-        return if value.nil?
-
-        Integer(value, exception: false)
-      end
-
-      def log_request url, response
-        return if @log.nil?
-
-        @log.puts "==> GET #{url} (#{response.body.to_s.bytesize} bytes)"
-      end
-
-      def throttle
-        return if @rate_limit.zero?
-        return if @last_request_at.nil?
-
-        elapsed = Time.now - @last_request_at
-        return if elapsed >= @rate_limit
+    # DL::Core::Client with arxiv-dl's User-Agent filled in
+    class Client < DL::Core::Client
+      SOURCE_URL = 'https://github.com/xoengineering/arxiv-dl'.freeze
+      USER_AGENT = "arxiv-dl/#{VERSION} (+#{SOURCE_URL})".freeze
 
-        sleep(@rate_limit - elapsed)
+      def initialize log: nil, rate_limit: DEFAULT_RATE_LIMIT
+        super(user_agent: USER_AGENT, log:, rate_limit:)
       end
     end
   end

data/lib/arxiv/downloader/error.rb

@@ -1,5 +1,5 @@
 module Arxiv
   module Downloader
-    class Error < StandardError; end
+    class Error < DL::Core::Error; end
   end
 end

data/lib/arxiv/downloader/http_error.rb

@@ -1,14 +1,5 @@
 module Arxiv
   module Downloader
-    class HTTPError < Error
-      attr_reader :status, :url
-
-      def initialize status:, url:, reason: nil
-        @status = status
-        @url    = url
-
-        super("GET #{url} failed: #{[status, reason].compact.join ' '}")
-      end
-    end
+    HTTPError = DL::Core::HTTPError
   end
 end

data/lib/arxiv/downloader/metadata/json.rb

@@ -1,33 +1,7 @@
-require 'fileutils'
-require 'json'
-
 module Arxiv
   module Downloader
     class Metadata
-      class JSON
-        FILENAME = 'metadata.json'.freeze
-
-        def initialize metadata
-          @metadata = metadata
-        end
-
-        def write to:
-          FileUtils.mkdir_p to
-          File.write File.join(to, FILENAME), "#{::JSON.pretty_generate(serialize(@metadata.to_h))}\n"
-        end
-
-        private
-
-        def serialize object
-          case object
-          when Hash       then object.to_h { |key, value| [key.to_s, serialize(value)] }
-          when Array      then object.map { |item| serialize item }
-          when Data       then serialize object.to_h
-          when Date, Time then object.iso8601
-          else object
-          end
-        end
-      end
+      JSON = DL::Core::Sidecar::JSON
     end
   end
 end

data/lib/arxiv/downloader/metadata/markdown.rb

@@ -1,44 +1,22 @@
-require 'fileutils'
-require 'yaml'
-
 module Arxiv
   module Downloader
     class Metadata
+      # metadata.md: dl-core writes the frontmatter; the body is arxiv-specific
       class Markdown
-        FILENAME = 'metadata.md'.freeze
-
         def initialize metadata
           @metadata = metadata
         end
 
         def write to:
-          FileUtils.mkdir_p to
-          File.write File.join(to, FILENAME), "#{frontmatter}\n#{body}"
+          DL::Core::Sidecar::Markdown.new(@metadata, body:, extras: { bibtex_key: }).write to:
         end
 
         private
 
-        def frontmatter
-          "---\n#{::YAML.dump(stringify(frontmatter_hash)).delete_prefix("---\n")}---"
-        end
-
-        def frontmatter_hash
-          @metadata.to_h.merge bibtex_key: bibtex_key
-        end
-
         def bibtex_key
           Downloader::Bibtex.new(@metadata).key
         end
 
-        def stringify object
-          case object
-          when Hash  then object.to_h { |key, value| [key.to_s, stringify(value)] }
-          when Array then object.map { |item| stringify item }
-          when Data  then stringify object.to_h
-          else object
-          end
-        end
-
         def body
           <<~MARKDOWN
 

data/lib/arxiv/downloader/metadata/yaml.rb

@@ -1,32 +1,7 @@
-require 'fileutils'
-require 'yaml'
-
 module Arxiv
   module Downloader
     class Metadata
-      class YAML
-        FILENAME = 'metadata.yaml'.freeze
-
-        def initialize metadata
-          @metadata = metadata
-        end
-
-        def write to:
-          FileUtils.mkdir_p to
-          File.write File.join(to, FILENAME), ::YAML.dump(stringify(@metadata.to_h))
-        end
-
-        private
-
-        def stringify object
-          case object
-          when Hash  then object.to_h { |key, value| [key.to_s, stringify(value)] }
-          when Array then object.map { |item| stringify item }
-          when Data  then stringify object.to_h
-          else object
-          end
-        end
-      end
+      YAML = DL::Core::Sidecar::YAML
     end
   end
 end

data/lib/arxiv/downloader/paper_folder.rb

@@ -0,0 +1,85 @@
+require 'fileutils'
+require 'nokogiri'
+require 'yaml'
+
+module Arxiv
+  module Downloader
+    # One paper's folder. A paper with a single archived version is kept flat
+    # (files directly in the folder); once a second version arrives, each
+    # version gets its own v<N>/ folder.
+    class PaperFolder
+      VERSION_FOLDER = /\Av\d+(\.partial)?\z/
+
+      attr_reader :path
+
+      def initialize path
+        @path = path
+      end
+
+      def archived? version
+        Dir.exist? location_of(version)
+      end
+
+      def location_of version
+        return path if flat_version == version
+
+        version_path version
+      end
+
+      # where a new download of `version` goes; call unflatten! first if the folder is flat
+      def destination_for version
+        return path if version == 1 && !versioned?
+
+        version_path version
+      end
+
+      # moves the flat version into v<N>/ so another version can sit beside it
+      def unflatten!
+        return if flat_version.nil?
+
+        destination = version_path flat_version
+        FileUtils.mkdir_p destination
+
+        # metadata.yaml marks the folder as flat, so it moves last
+        children = Dir.children(path) - [File.basename(destination), metadata_filename]
+        children.each { FileUtils.mv File.join(path, it), destination }
+        deepen_shared_links File.join(destination, 'html')
+        FileUtils.mv File.join(path, metadata_filename), destination
+      end
+
+      private
+
+      def flat_version
+        metadata_path = File.join path, metadata_filename
+        return unless File.exist? metadata_path
+
+        ::YAML.safe_load_file(metadata_path, permitted_classes: [Date]).fetch 'version'
+      end
+
+      def versioned?
+        Dir.exist?(path) && Dir.children(path).any? { VERSION_FOLDER.match? it }
+      end
+
+      def version_path version
+        File.join path, "v#{version}"
+      end
+
+      def metadata_filename
+        Metadata::YAML::FILENAME
+      end
+
+      # html/ moved one level deeper, so relative links up into _shared/ need one more ../
+      def deepen_shared_links html_dir
+        Dir.glob(File.join(html_dir, '*.html')).each do |html_path|
+          document = Nokogiri::HTML File.read(html_path)
+          HTMLArchive::ASSET_SELECTORS.each do |selector, attribute|
+            document.css(selector).each do |node|
+              node[attribute] = "../#{node[attribute]}" if node[attribute].start_with? '../'
+            end
+          end
+          File.write html_path, document.to_html
+        end
+      end
+    end
+  end
+end

data/lib/arxiv/downloader/slug.rb

@@ -1,44 +1,5 @@
-require 'stringex'
-
 module Arxiv
   module Downloader
-    class Slug
-      MAX_LENGTH = 80
-
-      TEX_INLINE_MATH  = /\$[^$]*\$/                # $...$
-      TEX_DISPLAY_MATH = /\\\(.*?\\\)|\\\[.*?\\\]/m # \(...\) or \[...\]
-      TEX_COMMAND      = /\\[a-zA-Z]+\*?/           # \emph, \alpha, etc.
-
-      def initialize title
-        @title = title
-      end
-
-      def to_s
-        truncate strip_tex(@title).to_url
-      end
-
-      private
-
-      def strip_tex string
-        string
-          .gsub(TEX_INLINE_MATH, ' ')
-          .gsub(TEX_DISPLAY_MATH, ' ')
-          .gsub(TEX_COMMAND, ' ')
-      end
-
-      def truncate slug
-        return slug if slug.length <= MAX_LENGTH
-
-        words = slug.split '-'
-        result = +''
-        words.each do |word|
-          break if result.length + 1 + word.length > MAX_LENGTH
-
-          result << '-' unless result.empty?
-          result << word
-        end
-        result
-      end
-    end
+    Slug = DL::Core::Slug
   end
 end

data/lib/arxiv/downloader/version.rb

@@ -1,5 +1,5 @@
 module Arxiv
   module Downloader
-    VERSION = '0.2.0'.freeze
+    VERSION = '0.3.1'.freeze
   end
 end

data/lib/arxiv/downloader.rb

@@ -1,3 +1,7 @@
+require 'dl/core'
+
+require_relative 'downloader/version'           # before client: Client::USER_AGENT uses VERSION
+
 require_relative 'downloader/abstract_page'
 require_relative 'downloader/archive'
 require_relative 'downloader/assets_cache'
@@ -5,23 +9,23 @@ require_relative 'downloader/author'
 require_relative 'downloader/bibtex'
 require_relative 'downloader/categories'
 require_relative 'downloader/cli'
-require_relative 'downloader/client'
+require_relative 'downloader/client'            # after version
 require_relative 'downloader/error'             # before errors below that subclass Error
 require_relative 'downloader/feed_parser'
 require_relative 'downloader/html_archive'
-require_relative 'downloader/http_error'        # after error
+require_relative 'downloader/http_error'
 require_relative 'downloader/identifier'        # after error
 require_relative 'downloader/metadata'          # before metadata/*: they reopen class Metadata
 require_relative 'downloader/metadata/bibtex'   # after metadata
 require_relative 'downloader/metadata/json'     # after metadata
 require_relative 'downloader/metadata/markdown' # after metadata
 require_relative 'downloader/metadata/yaml'     # after metadata
+require_relative 'downloader/paper_folder'
 require_relative 'downloader/paper_not_found'   # after error
 require_relative 'downloader/path'
 require_relative 'downloader/pdf'
 require_relative 'downloader/slug'
 require_relative 'downloader/source_archive'
-require_relative 'downloader/version'
 
 module Arxiv
   module Downloader

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: arxiv-dl
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Shane Becker
@@ -10,33 +10,33 @@ cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: feedjira
+  name: dl-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '0.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '0.1'
 - !ruby/object:Gem::Dependency
-  name: http
+  name: feedjira
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '4.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -51,34 +51,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.19'
-- !ruby/object:Gem::Dependency
-  name: ostruct
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.6'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.6'
-- !ruby/object:Gem::Dependency
-  name: stringex
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.8'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.8'
 description: Command line tool and Ruby library for archiving arxiv.org papers as
   PDFs with sidecar metadata.
 email:
@@ -116,6 +88,7 @@ files:
 - lib/arxiv/downloader/metadata/json.rb
 - lib/arxiv/downloader/metadata/markdown.rb
 - lib/arxiv/downloader/metadata/yaml.rb
+- lib/arxiv/downloader/paper_folder.rb
 - lib/arxiv/downloader/paper_not_found.rb
 - lib/arxiv/downloader/path.rb
 - lib/arxiv/downloader/pdf.rb