arxiv-dl 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c919d2e16ae72397375aa3aa5f71bd08fe4f3d0687d83358bf652081ee994596
-  data.tar.gz: 35a55cb7152642e19816a76b4a00fe52b06f74d5a52cb587d2e2b6897e23ac61
+  metadata.gz: 14d23483457b1dbb20c8aa9fa1384e52caf7144cab5ed357450f235dbcd29b80
+  data.tar.gz: 4d7255130c208f959bb0bb7249a33b3a076a362da7894dc370e480f2511109b7
 SHA512:
-  metadata.gz: 67f1ea3177544e5c4e5472d2b1e81d894a42346313ae4666f53f9de89571b36a55d50ee247dae0b8af2dce869cec03ea72890009817ee613b7cab37b93bddbfe
-  data.tar.gz: 5bcc8fc0ab51a75032ed0a3d8d2e20dad830f9b6bb92f75841be0fa23e05f60d66e76cb7e23c5a8b17fffba0634213d3b4299fde0c86314858f396135d2a9a84
+  metadata.gz: '0572297cdb2a74bb5cb46cdcd8449e2e41611a9649493e8ba5bc22b2fcb84d264e5982f114d2cff640cfd76e1761de592c8a472815ee6a8116b70f35807a11a0'
+  data.tar.gz: 8fd1844b84d8fc2044b8d92078ad2dae136713c93768269b64fe9a83b169a9da37466661dbaf61bb12f81d2ba827276f9c56e00291b978e8bdf7524de4a858ad

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## [0.3.1]
+
+- Built on [dl-core](https://github.com/xoengineering/dl-core) 0.1: the HTTP client, errors, `Author`, `Slug`, sidecar writers, and CLI now come from it instead of copies. No change in behavior or output. `Arxiv::Downloader::Client`, `HTTPError`, `Author`, `Slug`, and `Metadata::YAML`/`JSON` still work; they are now `dl-core`'s classes. `Arxiv::Downloader::Error` now subclasses `DL::Core::Error`.
+- Direct dependencies on `http`, `stringex`, and `ostruct` replaced by `dl-core`.
+
 ## [0.3.0]
 
 Less folder nesting for the common case: most papers only ever have one version.

data/lib/arxiv/downloader/author.rb

@@ -1,11 +1,5 @@
 module Arxiv
   module Downloader
-    Author = Data.define :name, :affiliations do
-      def initialize name:, affiliations: []
-        super
-      end
-
-      def to_s = name
-    end
+    Author = DL::Core::Author
   end
 end

data/lib/arxiv/downloader/cli.rb

@@ -1,106 +1,24 @@
-require 'optparse'
-
 module Arxiv
   module Downloader
-    class CLI
-      DEFAULT_DOWNLOAD_PATH = File.join Dir.home, 'Downloads', 'ArXiv_Papers'
-      USAGE                 = 'Usage: arxiv-dl [options] <ARXIV_ID_OR_URL> [<ARXIV_ID_OR_URL>...]'.freeze
-
-      def initialize argv, stderr: $stderr, stdin: $stdin, stdout: $stdout
-        @argv   = argv
-        @stderr = stderr
-        @stdin  = stdin
-        @stdout = stdout
-      end
-
-      def run
-        options = parse
-        return options.fetch(:exit_status) if options.key? :exit_status
-
-        return error_with USAGE       if options[:targets].empty?
-        return error_with conflict if options[:verbose] && options[:quiet]
-
-        failures = download_each options
-        failures.zero? ? 0 : 1
-      end
-
-      private
-
-      def conflict
-        '-v and -q are mutually exclusive'
-      end
-
-      def parse
-        options = { targets: [], verbose: false, quiet: false }
-        parser  = build_parser options
-
-        begin
-          parser.parse! @argv
-          options[:targets] = @argv + input_targets(options[:input])
-        rescue OptionParser::ParseError, SystemCallError => e
-          @stderr.puts e.message
-          return { exit_status: 1 }
-        end
-
-        options[:path]       ||= ENV['ARXIV_DOWNLOAD_PATH'] || DEFAULT_DOWNLOAD_PATH
-        options[:rate_limit] ||= (ENV['ARXIV_RATE_LIMIT'] || Client::DEFAULT_RATE_LIMIT).to_i
-        options
-      end
+    class CLI < DL::Core::CLI
+      def program = 'arxiv-dl'
 
-      def build_parser options
-        OptionParser.new do |parser|
-          parser.banner = USAGE
-          parser.on('-i FILE', '--input FILE')       { |value| options[:input] = value }
-          parser.on('-p PATH', '--path PATH')        { |value| options[:path] = value }
-          parser.on('--rate-limit SECONDS', Integer) { |value| options[:rate_limit] = value }
-          parser.on('-v', '--verbose')               { options[:verbose] = true }
-          parser.on('-q', '--quiet')                 { options[:quiet]   = true }
-          parser.on('--version') do
-            @stdout.puts VERSION
-            options[:exit_status] = 0
-          end
-          parser.on('-h', '--help') do
-            @stdout.puts parser.help
-            options[:exit_status] = 0
-          end
-        end
-      end
+      def target_name = 'ARXIV_ID_OR_URL'
 
-      # one target per line from FILE, or stdin for "-"; blank lines and # comments skipped
-      def input_targets input
-        return [] if input.nil?
+      # ARXIV_DOWNLOAD_PATH, ARXIV_RATE_LIMIT
+      def env_prefix = 'ARXIV'
 
-        text = input == '-' ? @stdin.read : File.read(input)
-        text.lines.map(&:strip).reject { it.empty? || it.start_with?('#') }
-      end
+      def default_path = File.join(Dir.home, 'Downloads', 'ArXiv_Papers')
 
-      def error_with message
-        @stderr.puts message
-        1
-      end
+      def version = VERSION
 
-      def download_each options
-        client = Client.new rate_limit: options[:rate_limit], log: (options[:verbose] ? @stdout : nil)
+      def user_agent = Client::USER_AGENT
 
-        failures = 0
-        options[:targets].each do |target|
-          failures += 1 unless download_one(target, client:, options:)
-        end
-        failures
-      end
+      def client_for(rate_limit:, log:) = Client.new(rate_limit:, log:)
 
-      # true on success; reports the failure and returns false otherwise
-      def download_one target, client:, options:
-        identifier = Identifier.new target
-        @stdout.puts "==> Downloading #{identifier.id}" if options[:verbose]
+      def identifier_for(target) = Identifier.new(target)
 
-        path = Archive.new(identifier, root: options[:path], client: client).run
-        @stdout.puts path unless options[:quiet]
-        true
-      rescue Error, HTTP::Error => e
-        @stderr.puts "#{target}: #{e.message}"
-        false
-      end
+      def archive_for(identifier, root:, client:) = Archive.new(identifier, root:, client:)
     end
   end
 end

data/lib/arxiv/downloader/client.rb

@@ -1,85 +1,12 @@
-require 'http'
-
 module Arxiv
   module Downloader
-    class Client
-      SOURCE_URL         = 'https://github.com/xoengineering/arxiv-dl'.freeze
-      DEFAULT_RATE_LIMIT = 3
-      TIMEOUTS           = { connect: 10, read: 60, write: 10 }.freeze # seconds, per operation
-      MAX_RETRIES        = 3
-      RETRY_BACKOFF      = 10 # seconds before the first retry. doubles on each retry.
-      RETRYABLE_STATUSES = [429, 503].freeze
-
-      attr_reader :rate_limit
-
-      def initialize rate_limit: DEFAULT_RATE_LIMIT, log: nil
-        @rate_limit = rate_limit
-        @log        = log
-      end
-
-      def user_agent
-        "arxiv-dl/#{VERSION} (+#{SOURCE_URL})"
-      end
-
-      def get url
-        retries = 0
-
-        loop do
-          response = request url
-          return response if response.status.success?
-          raise http_error(url, response) unless retryable? response, retries
-
-          retries += 1
-          wait_before_retry response, retries
-        end
-      end
-
-      private
-
-      def request url
-        throttle
-        response = HTTP.timeout(TIMEOUTS).headers('User-Agent' => user_agent).follow.get(url)
-        @last_request_at = Time.now
-        log_request url, response
-        response
-      end
-
-      def http_error url, response
-        HTTPError.new status: response.status.code, url: url, reason: response.status.reason
-      end
-
-      def retryable? response, retries
-        RETRYABLE_STATUSES.include?(response.status.code) && retries < MAX_RETRIES
-      end
-
-      def wait_before_retry response, retries
-        seconds = retry_after(response) || (RETRY_BACKOFF * (2**(retries - 1)))
-        @log&.puts "==> #{response.status}. Retrying in #{seconds}s"
-        sleep seconds
-      end
-
-      # Retry-After in delay-seconds form. The HTTP-date form falls back to backoff.
-      def retry_after response
-        value = response.headers['Retry-After']
-        return if value.nil?
-
-        Integer(value, exception: false)
-      end
-
-      def log_request url, response
-        return if @log.nil?
-
-        @log.puts "==> GET #{url} (#{response.body.to_s.bytesize} bytes)"
-      end
-
-      def throttle
-        return if @rate_limit.zero?
-        return if @last_request_at.nil?
-
-        elapsed = Time.now - @last_request_at
-        return if elapsed >= @rate_limit
+    # DL::Core::Client with arxiv-dl's User-Agent filled in
+    class Client < DL::Core::Client
+      SOURCE_URL = 'https://github.com/xoengineering/arxiv-dl'.freeze
+      USER_AGENT = "arxiv-dl/#{VERSION} (+#{SOURCE_URL})".freeze
 
-        sleep(@rate_limit - elapsed)
+      def initialize log: nil, rate_limit: DEFAULT_RATE_LIMIT
+        super(user_agent: USER_AGENT, log:, rate_limit:)
       end
     end
   end

data/lib/arxiv/downloader/error.rb

@@ -1,5 +1,5 @@
 module Arxiv
   module Downloader
-    class Error < StandardError; end
+    class Error < DL::Core::Error; end
   end
 end

data/lib/arxiv/downloader/http_error.rb

@@ -1,14 +1,5 @@
 module Arxiv
   module Downloader
-    class HTTPError < Error
-      attr_reader :status, :url
-
-      def initialize status:, url:, reason: nil
-        @status = status
-        @url    = url
-
-        super("GET #{url} failed: #{[status, reason].compact.join ' '}")
-      end
-    end
+    HTTPError = DL::Core::HTTPError
   end
 end

data/lib/arxiv/downloader/metadata/json.rb

@@ -1,33 +1,7 @@
-require 'fileutils'
-require 'json'
-
 module Arxiv
   module Downloader
     class Metadata
-      class JSON
-        FILENAME = 'metadata.json'.freeze
-
-        def initialize metadata
-          @metadata = metadata
-        end
-
-        def write to:
-          FileUtils.mkdir_p to
-          File.write File.join(to, FILENAME), "#{::JSON.pretty_generate(serialize(@metadata.to_h))}\n"
-        end
-
-        private
-
-        def serialize object
-          case object
-          when Hash       then object.to_h { |key, value| [key.to_s, serialize(value)] }
-          when Array      then object.map { |item| serialize item }
-          when Data       then serialize object.to_h
-          when Date, Time then object.iso8601
-          else object
-          end
-        end
-      end
+      JSON = DL::Core::Sidecar::JSON
     end
   end
 end

data/lib/arxiv/downloader/metadata/markdown.rb

@@ -1,44 +1,22 @@
-require 'fileutils'
-require 'yaml'
-
 module Arxiv
   module Downloader
     class Metadata
+      # metadata.md: dl-core writes the frontmatter; the body is arxiv-specific
       class Markdown
-        FILENAME = 'metadata.md'.freeze
-
         def initialize metadata
           @metadata = metadata
         end
 
         def write to:
-          FileUtils.mkdir_p to
-          File.write File.join(to, FILENAME), "#{frontmatter}\n#{body}"
+          DL::Core::Sidecar::Markdown.new(@metadata, body:, extras: { bibtex_key: }).write to:
         end
 
         private
 
-        def frontmatter
-          "---\n#{::YAML.dump(stringify(frontmatter_hash)).delete_prefix("---\n")}---"
-        end
-
-        def frontmatter_hash
-          @metadata.to_h.merge bibtex_key: bibtex_key
-        end
-
         def bibtex_key
           Downloader::Bibtex.new(@metadata).key
         end
 
-        def stringify object
-          case object
-          when Hash  then object.to_h { |key, value| [key.to_s, stringify(value)] }
-          when Array then object.map { |item| stringify item }
-          when Data  then stringify object.to_h
-          else object
-          end
-        end
-
         def body
           <<~MARKDOWN
 

data/lib/arxiv/downloader/metadata/yaml.rb

@@ -1,32 +1,7 @@
-require 'fileutils'
-require 'yaml'
-
 module Arxiv
   module Downloader
     class Metadata
-      class YAML
-        FILENAME = 'metadata.yaml'.freeze
-
-        def initialize metadata
-          @metadata = metadata
-        end
-
-        def write to:
-          FileUtils.mkdir_p to
-          File.write File.join(to, FILENAME), ::YAML.dump(stringify(@metadata.to_h))
-        end
-
-        private
-
-        def stringify object
-          case object
-          when Hash  then object.to_h { |key, value| [key.to_s, stringify(value)] }
-          when Array then object.map { |item| stringify item }
-          when Data  then stringify object.to_h
-          else object
-          end
-        end
-      end
+      YAML = DL::Core::Sidecar::YAML
     end
   end
 end

data/lib/arxiv/downloader/slug.rb

@@ -1,44 +1,5 @@
-require 'stringex'
-
 module Arxiv
   module Downloader
-    class Slug
-      MAX_LENGTH = 80
-
-      TEX_INLINE_MATH  = /\$[^$]*\$/                # $...$
-      TEX_DISPLAY_MATH = /\\\(.*?\\\)|\\\[.*?\\\]/m # \(...\) or \[...\]
-      TEX_COMMAND      = /\\[a-zA-Z]+\*?/           # \emph, \alpha, etc.
-
-      def initialize title
-        @title = title
-      end
-
-      def to_s
-        truncate strip_tex(@title).to_url
-      end
-
-      private
-
-      def strip_tex string
-        string
-          .gsub(TEX_INLINE_MATH, ' ')
-          .gsub(TEX_DISPLAY_MATH, ' ')
-          .gsub(TEX_COMMAND, ' ')
-      end
-
-      def truncate slug
-        return slug if slug.length <= MAX_LENGTH
-
-        words = slug.split '-'
-        result = +''
-        words.each do |word|
-          break if result.length + 1 + word.length > MAX_LENGTH
-
-          result << '-' unless result.empty?
-          result << word
-        end
-        result
-      end
-    end
+    Slug = DL::Core::Slug
   end
 end

data/lib/arxiv/downloader/version.rb

@@ -1,5 +1,5 @@
 module Arxiv
   module Downloader
-    VERSION = '0.3.0'.freeze
+    VERSION = '0.3.1'.freeze
   end
 end

data/lib/arxiv/downloader.rb

@@ -1,3 +1,7 @@
+require 'dl/core'
+
+require_relative 'downloader/version'           # before client: Client::USER_AGENT uses VERSION
+
 require_relative 'downloader/abstract_page'
 require_relative 'downloader/archive'
 require_relative 'downloader/assets_cache'
@@ -5,11 +9,11 @@ require_relative 'downloader/author'
 require_relative 'downloader/bibtex'
 require_relative 'downloader/categories'
 require_relative 'downloader/cli'
-require_relative 'downloader/client'
+require_relative 'downloader/client'            # after version
 require_relative 'downloader/error'             # before errors below that subclass Error
 require_relative 'downloader/feed_parser'
 require_relative 'downloader/html_archive'
-require_relative 'downloader/http_error'        # after error
+require_relative 'downloader/http_error'
 require_relative 'downloader/identifier'        # after error
 require_relative 'downloader/metadata'          # before metadata/*: they reopen class Metadata
 require_relative 'downloader/metadata/bibtex'   # after metadata
@@ -22,7 +26,6 @@ require_relative 'downloader/path'
 require_relative 'downloader/pdf'
 require_relative 'downloader/slug'
 require_relative 'downloader/source_archive'
-require_relative 'downloader/version'
 
 module Arxiv
   module Downloader

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: arxiv-dl
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Shane Becker
@@ -10,33 +10,33 @@ cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: feedjira
+  name: dl-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '0.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '0.1'
 - !ruby/object:Gem::Dependency
-  name: http
+  name: feedjira
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '4.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -51,34 +51,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.19'
-- !ruby/object:Gem::Dependency
-  name: ostruct
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.6'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.6'
-- !ruby/object:Gem::Dependency
-  name: stringex
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.8'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.8'
 description: Command line tool and Ruby library for archiving arxiv.org papers as
   PDFs with sidecar metadata.
 email: