jstor-dl 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cd0503a0a9da1a8298c012fab0dd19f866e907a87a58537f6f1b26174b3bb1ab
-  data.tar.gz: 20779888e79ac989cc10b14ac96877b9200641f7e4cc3fe679f252b4602b1570
+  metadata.gz: 7711d17b180cc39c3938b9da490dab3f6a13d4ee692321d1ddda127e154c0453
+  data.tar.gz: f3374bc83297a607be90e3423b22602ec59ac9bf014fcaf27c85eff6553e7a14
 SHA512:
-  metadata.gz: 6839ae6451e76bfe92b282b4829a7dcea3b117c103bf68d345ad8e91d7a922d56d30bd27007a3be8c0920ac219c67c0b04ef98afa0e0206e0e9e7e8b3ebdf758
-  data.tar.gz: d0a458946ce663a24638a25b4fbc23d8579352c101105cb371e764b086524fef22723080e93314512cdfcdd0a8fd061cfb929edaac067f1c06087e24cef0d392
+  metadata.gz: 54e430f87d37a570476b5c3063f5b81f878721c384e23c21157a2ef96a9103a6fec9a04b17df18ba6cf29d389c200d4048df247753f7314bc48dc81878f71661
+  data.tar.gz: 9843468654206d7ded8033c86c943006ff042d97c036dc300020bd8007df27674fe18467b6436c1020036a9fef146156b2be41bdc6413e7ee348b3dd280fbe69

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## [0.1.1]
+
+- Built on [dl-core](https://github.com/xoengineering/dl-core) 0.1: the HTTP client, errors, `Author`, `Slug`, sidecar writers, and CLI now come from it instead of copies. No change in behavior or output. `Jstor::Downloader::Client`, `HTTPError`, `Author`, `Slug`, and `Metadata::YAML`/`JSON` still work; they are now `dl-core`'s classes. `Jstor::Downloader::Error` now subclasses `DL::Core::Error`.
+
 ## [0.1.0]
 
 First version. Per-article offline archive of JSTOR's public-domain Early Journal Content, fetched only from the Internet Archive's copy, never from jstor.org.

data/lib/jstor/downloader/author.rb

@@ -1,11 +1,5 @@
 module Jstor
   module Downloader
-    Author = Data.define :name, :affiliations do
-      def initialize name:, affiliations: []
-        super
-      end
-
-      def to_s = name
-    end
+    Author = DL::Core::Author
   end
 end

data/lib/jstor/downloader/cli.rb

@@ -1,106 +1,24 @@
-require 'optparse'
-
 module Jstor
   module Downloader
-    class CLI
-      DEFAULT_DOWNLOAD_PATH = File.join Dir.home, 'Downloads', 'JSTOR_Papers'
-      USAGE                 = 'Usage: jstor-dl [options] <JSTOR_ID_OR_URL> [<JSTOR_ID_OR_URL>...]'.freeze
-
-      def initialize argv, stderr: $stderr, stdin: $stdin, stdout: $stdout
-        @argv   = argv
-        @stderr = stderr
-        @stdin  = stdin
-        @stdout = stdout
-      end
-
-      def run
-        options = parse
-        return options.fetch(:exit_status) if options.key? :exit_status
-
-        return error_with USAGE       if options[:targets].empty?
-        return error_with conflict if options[:verbose] && options[:quiet]
-
-        failures = download_each options
-        failures.zero? ? 0 : 1
-      end
-
-      private
-
-      def conflict
-        '-v and -q are mutually exclusive'
-      end
-
-      def parse
-        options = { targets: [], verbose: false, quiet: false }
-        parser  = build_parser options
-
-        begin
-          parser.parse! @argv
-          options[:targets] = @argv + input_targets(options[:input])
-        rescue OptionParser::ParseError, SystemCallError => e
-          @stderr.puts e.message
-          return { exit_status: 1 }
-        end
-
-        options[:path]       ||= ENV['JSTOR_DOWNLOAD_PATH'] || DEFAULT_DOWNLOAD_PATH
-        options[:rate_limit] ||= (ENV['JSTOR_RATE_LIMIT'] || Client::DEFAULT_RATE_LIMIT).to_i
-        options
-      end
+    class CLI < DL::Core::CLI
+      def program = 'jstor-dl'
 
-      def build_parser options
-        OptionParser.new do |parser|
-          parser.banner = USAGE
-          parser.on('-i FILE', '--input FILE')       { |value| options[:input] = value }
-          parser.on('-p PATH', '--path PATH')        { |value| options[:path] = value }
-          parser.on('--rate-limit SECONDS', Integer) { |value| options[:rate_limit] = value }
-          parser.on('-v', '--verbose')               { options[:verbose] = true }
-          parser.on('-q', '--quiet')                 { options[:quiet]   = true }
-          parser.on('--version') do
-            @stdout.puts VERSION
-            options[:exit_status] = 0
-          end
-          parser.on('-h', '--help') do
-            @stdout.puts parser.help
-            options[:exit_status] = 0
-          end
-        end
-      end
+      def target_name = 'JSTOR_ID_OR_URL'
 
-      # one target per line from FILE, or stdin for "-"; blank lines and # comments skipped
-      def input_targets input
-        return [] if input.nil?
+      # JSTOR_DOWNLOAD_PATH, JSTOR_RATE_LIMIT
+      def env_prefix = 'JSTOR'
 
-        text = input == '-' ? @stdin.read : File.read(input)
-        text.lines.map(&:strip).reject { it.empty? || it.start_with?('#') }
-      end
+      def default_path = File.join(Dir.home, 'Downloads', 'JSTOR_Papers')
 
-      def error_with message
-        @stderr.puts message
-        1
-      end
+      def version = VERSION
 
-      def download_each options
-        client = Client.new rate_limit: options[:rate_limit], log: (options[:verbose] ? @stdout : nil)
+      def user_agent = Client::USER_AGENT
 
-        failures = 0
-        options[:targets].each do |target|
-          failures += 1 unless download_one(target, client:, options:)
-        end
-        failures
-      end
+      def client_for(rate_limit:, log:) = Client.new(rate_limit:, log:)
 
-      # true on success; reports the failure and returns false otherwise
-      def download_one target, client:, options:
-        identifier = Identifier.new target
-        @stdout.puts "==> Downloading #{identifier.id}" if options[:verbose]
+      def identifier_for(target) = Identifier.new(target)
 
-        path = Archive.new(identifier, root: options[:path], client: client).run
-        @stdout.puts path unless options[:quiet]
-        true
-      rescue Error, HTTP::Error => e
-        @stderr.puts "#{target}: #{e.message}"
-        false
-      end
+      def archive_for(identifier, root:, client:) = Archive.new(identifier, root:, client:)
     end
   end
 end

data/lib/jstor/downloader/client.rb

@@ -1,85 +1,12 @@
-require 'http'
-
 module Jstor
   module Downloader
-    class Client
-      SOURCE_URL         = 'https://github.com/xoengineering/jstor-dl'.freeze
-      DEFAULT_RATE_LIMIT = 3
-      TIMEOUTS           = { connect: 10, read: 60, write: 10 }.freeze # seconds, per operation
-      MAX_RETRIES        = 3
-      RETRY_BACKOFF      = 10 # seconds before the first retry. doubles on each retry.
-      RETRYABLE_STATUSES = [429, 503].freeze
-
-      attr_reader :rate_limit
-
-      def initialize rate_limit: DEFAULT_RATE_LIMIT, log: nil
-        @rate_limit = rate_limit
-        @log        = log
-      end
-
-      def user_agent
-        "jstor-dl/#{VERSION} (+#{SOURCE_URL})"
-      end
-
-      def get url
-        retries = 0
-
-        loop do
-          response = request url
-          return response if response.status.success?
-          raise http_error(url, response) unless retryable? response, retries
-
-          retries += 1
-          wait_before_retry response, retries
-        end
-      end
-
-      private
-
-      def request url
-        throttle
-        response = HTTP.timeout(TIMEOUTS).headers('User-Agent' => user_agent).follow.get(url)
-        @last_request_at = Time.now
-        log_request url, response
-        response
-      end
-
-      def http_error url, response
-        HTTPError.new status: response.status.code, url: url, reason: response.status.reason
-      end
-
-      def retryable? response, retries
-        RETRYABLE_STATUSES.include?(response.status.code) && retries < MAX_RETRIES
-      end
-
-      def wait_before_retry response, retries
-        seconds = retry_after(response) || (RETRY_BACKOFF * (2**(retries - 1)))
-        @log&.puts "==> #{response.status}. Retrying in #{seconds}s"
-        sleep seconds
-      end
-
-      # Retry-After in delay-seconds form. The HTTP-date form falls back to backoff.
-      def retry_after response
-        value = response.headers['Retry-After']
-        return if value.nil?
-
-        Integer(value, exception: false)
-      end
-
-      def log_request url, response
-        return if @log.nil?
-
-        @log.puts "==> GET #{url} (#{response.body.to_s.bytesize} bytes)"
-      end
-
-      def throttle
-        return if @rate_limit.zero?
-        return if @last_request_at.nil?
-
-        elapsed = Time.now - @last_request_at
-        return if elapsed >= @rate_limit
+    # DL::Core::Client with jstor-dl's User-Agent filled in
+    class Client < DL::Core::Client
+      SOURCE_URL = 'https://github.com/xoengineering/jstor-dl'.freeze
+      USER_AGENT = "jstor-dl/#{VERSION} (+#{SOURCE_URL})".freeze
 
-        sleep(@rate_limit - elapsed)
+      def initialize log: nil, rate_limit: DEFAULT_RATE_LIMIT
+        super(user_agent: USER_AGENT, log:, rate_limit:)
       end
     end
   end

data/lib/jstor/downloader/error.rb

@@ -1,5 +1,5 @@
 module Jstor
   module Downloader
-    class Error < StandardError; end
+    class Error < DL::Core::Error; end
   end
 end

data/lib/jstor/downloader/http_error.rb

@@ -1,14 +1,5 @@
 module Jstor
   module Downloader
-    class HTTPError < Error
-      attr_reader :status, :url
-
-      def initialize status:, url:, reason: nil
-        @status = status
-        @url    = url
-
-        super("GET #{url} failed: #{[status, reason].compact.join ' '}")
-      end
-    end
+    HTTPError = DL::Core::HTTPError
   end
 end

data/lib/jstor/downloader/metadata/json.rb

@@ -1,33 +1,7 @@
-require 'fileutils'
-require 'json'
-
 module Jstor
   module Downloader
     class Metadata
-      class JSON
-        FILENAME = 'metadata.json'.freeze
-
-        def initialize metadata
-          @metadata = metadata
-        end
-
-        def write to:
-          FileUtils.mkdir_p to
-          File.write File.join(to, FILENAME), "#{::JSON.pretty_generate(serialize(@metadata.to_h))}\n"
-        end
-
-        private
-
-        def serialize object
-          case object
-          when Hash       then object.to_h { |key, value| [key.to_s, serialize(value)] }
-          when Array      then object.map { |item| serialize item }
-          when Data       then serialize object.to_h
-          when Date, Time then object.iso8601
-          else object
-          end
-        end
-      end
+      JSON = DL::Core::Sidecar::JSON
     end
   end
 end

data/lib/jstor/downloader/metadata/markdown.rb

@@ -1,44 +1,22 @@
-require 'fileutils'
-require 'yaml'
-
 module Jstor
   module Downloader
     class Metadata
+      # metadata.md: dl-core writes the frontmatter; the body is JSTOR-specific
       class Markdown
-        FILENAME = 'metadata.md'.freeze
-
         def initialize metadata
           @metadata = metadata
         end
 
         def write to:
-          FileUtils.mkdir_p to
-          File.write File.join(to, FILENAME), "#{frontmatter}\n#{body}"
+          DL::Core::Sidecar::Markdown.new(@metadata, body:, extras: { bibtex_key: }).write to:
         end
 
         private
 
-        def frontmatter
-          "---\n#{::YAML.dump(stringify(frontmatter_hash)).delete_prefix("---\n")}---"
-        end
-
-        def frontmatter_hash
-          @metadata.to_h.merge bibtex_key: bibtex_key
-        end
-
         def bibtex_key
           Downloader::Bibtex.new(@metadata).key
         end
 
-        def stringify object
-          case object
-          when Hash  then object.to_h { |key, value| [key.to_s, stringify(value)] }
-          when Array then object.map { |item| stringify item }
-          when Data  then stringify object.to_h
-          else object
-          end
-        end
-
         # JSTOR asks for acknowledgement as the source of Early Journal Content
         def body
           <<~MARKDOWN

data/lib/jstor/downloader/metadata/yaml.rb

@@ -1,32 +1,7 @@
-require 'fileutils'
-require 'yaml'
-
 module Jstor
   module Downloader
     class Metadata
-      class YAML
-        FILENAME = 'metadata.yaml'.freeze
-
-        def initialize metadata
-          @metadata = metadata
-        end
-
-        def write to:
-          FileUtils.mkdir_p to
-          File.write File.join(to, FILENAME), ::YAML.dump(stringify(@metadata.to_h))
-        end
-
-        private
-
-        def stringify object
-          case object
-          when Hash  then object.to_h { |key, value| [key.to_s, stringify(value)] }
-          when Array then object.map { |item| stringify item }
-          when Data  then stringify object.to_h
-          else object
-          end
-        end
-      end
+      YAML = DL::Core::Sidecar::YAML
     end
   end
 end

data/lib/jstor/downloader/slug.rb

@@ -1,44 +1,5 @@
-require 'stringex'
-
 module Jstor
   module Downloader
-    class Slug
-      MAX_LENGTH = 80
-
-      TEX_INLINE_MATH  = /\$[^$]*\$/                # $...$
-      TEX_DISPLAY_MATH = /\\\(.*?\\\)|\\\[.*?\\\]/m # \(...\) or \[...\]
-      TEX_COMMAND      = /\\[a-zA-Z]+\*?/           # \emph, \alpha, etc.
-
-      def initialize title
-        @title = title
-      end
-
-      def to_s
-        truncate strip_tex(@title).to_url
-      end
-
-      private
-
-      def strip_tex string
-        string
-          .gsub(TEX_INLINE_MATH, ' ')
-          .gsub(TEX_DISPLAY_MATH, ' ')
-          .gsub(TEX_COMMAND, ' ')
-      end
-
-      def truncate slug
-        return slug if slug.length <= MAX_LENGTH
-
-        words = slug.split '-'
-        result = +''
-        words.each do |word|
-          break if result.length + 1 + word.length > MAX_LENGTH
-
-          result << '-' unless result.empty?
-          result << word
-        end
-        result
-      end
-    end
+    Slug = DL::Core::Slug
   end
 end

data/lib/jstor/downloader/version.rb

@@ -1,5 +1,5 @@
 module Jstor
   module Downloader
-    VERSION = '0.1.0'.freeze
+    VERSION = '0.1.1'.freeze
   end
 end

data/lib/jstor/downloader.rb

@@ -1,10 +1,14 @@
+require 'dl/core'
+
+require_relative 'downloader/version'           # before client: Client::USER_AGENT uses VERSION
+
 require_relative 'downloader/archive'
 require_relative 'downloader/author'
 require_relative 'downloader/bibtex'
 require_relative 'downloader/cli'
-require_relative 'downloader/client'
+require_relative 'downloader/client'            # after version
 require_relative 'downloader/error'             # before errors below that subclass Error
-require_relative 'downloader/http_error'        # after error
+require_relative 'downloader/http_error'
 require_relative 'downloader/identifier'        # after error
 require_relative 'downloader/item_file'
 require_relative 'downloader/item_not_found'    # after error
@@ -16,7 +20,6 @@ require_relative 'downloader/metadata/yaml'     # after metadata
 require_relative 'downloader/metadata_parser'
 require_relative 'downloader/path'
 require_relative 'downloader/slug'
-require_relative 'downloader/version'
 
 module Jstor
   module Downloader

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: jstor-dl
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Shane Becker
@@ -10,47 +10,19 @@ cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: http
+  name: dl-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '0.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '6.0'
-- !ruby/object:Gem::Dependency
-  name: ostruct
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.6'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.6'
-- !ruby/object:Gem::Dependency
-  name: stringex
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.8'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.8'
+        version: '0.1'
 description: |
   Command line tool and Ruby library for archiving JSTOR Early Journal Content articles
   as PDFs and OCR plaintext with sidecar metadata. Fetches only from the Internet Archive's copy,