arxiv-dl 0.1.0

data/lib/arxiv/downloader/cli.rb

@@ -0,0 +1,85 @@
+require 'optparse'
+
+module Arxiv
+  module Downloader
+    class CLI
+      DEFAULT_DOWNLOAD_PATH = File.join Dir.home, 'Downloads', 'ArXiv_Papers'
+      USAGE                 = 'Usage: arxiv-dl [options] <ARXIV_ID_OR_URL> [<ARXIV_ID_OR_URL>...]'.freeze
+
+      def initialize argv, stdout: $stdout, stderr: $stderr
+        @argv   = argv
+        @stdout = stdout
+        @stderr = stderr
+      end
+
+      def run
+        options = parse
+        return options.fetch(:exit_status) if options.key? :exit_status
+
+        return error_with USAGE       if options[:targets].empty?
+        return error_with conflict if options[:verbose] && options[:quiet]
+
+        download_each options
+        0
+      end
+
+      private
+
+      def conflict
+        '-v and -q are mutually exclusive'
+      end
+
+      def parse
+        options = { targets: [], verbose: false, quiet: false }
+        parser  = build_parser options
+
+        begin
+          parser.parse! @argv
+        rescue OptionParser::ParseError => e
+          @stderr.puts e.message
+          return { exit_status: 1 }
+        end
+
+        options[:targets] = @argv
+        options[:path]       ||= ENV['ARXIV_DOWNLOAD_PATH'] || DEFAULT_DOWNLOAD_PATH
+        options[:rate_limit] ||= (ENV['ARXIV_RATE_LIMIT'] || Client::DEFAULT_RATE_LIMIT).to_i
+        options
+      end
+
+      def build_parser options
+        OptionParser.new do |parser|
+          parser.banner = USAGE
+          parser.on('-p PATH', '--path PATH')        { |value| options[:path] = value }
+          parser.on('--rate-limit SECONDS', Integer) { |value| options[:rate_limit] = value }
+          parser.on('-v', '--verbose')               { options[:verbose] = true }
+          parser.on('-q', '--quiet')                 { options[:quiet]   = true }
+          parser.on('--version') do
+            @stdout.puts VERSION
+            options[:exit_status] = 0
+          end
+          parser.on('-h', '--help') do
+            @stdout.puts parser.help
+            options[:exit_status] = 0
+          end
+        end
+      end
+
+      def error_with message
+        @stderr.puts message
+        1
+      end
+
+      def download_each options
+        client = Client.new rate_limit: options[:rate_limit], log: (options[:verbose] ? @stdout : nil)
+
+        options[:targets].each do |target|
+          identifier = Identifier.new target
+          @stdout.puts "==> Downloading #{identifier.id}" if options[:verbose]
+
+          path = Archive.new(identifier, root: options[:path], client: client).run
+          @stdout.puts path unless options[:quiet]
+        end
+      end
+    end
+  end
+end

data/lib/arxiv/downloader/client.rb

@@ -0,0 +1,47 @@
+require 'http'
+
+module Arxiv
+  module Downloader
+    class Client
+      SOURCE_URL         = 'https://github.com/xoengineering/arxiv-dl'.freeze
+      DEFAULT_RATE_LIMIT = 3
+
+      attr_reader :rate_limit
+
+      def initialize rate_limit: DEFAULT_RATE_LIMIT, log: nil
+        @rate_limit = rate_limit
+        @log        = log
+      end
+
+      def user_agent
+        "arxiv-dl/#{VERSION} (+#{SOURCE_URL})"
+      end
+
+      def get url
+        throttle
+        response = HTTP.headers('User-Agent' => user_agent).follow.get(url)
+        @last_request_at = Time.now
+        log_request url, response
+        response
+      end
+
+      private
+
+      def log_request url, response
+        return if @log.nil?
+
+        @log.puts "==> GET #{url} (#{response.body.to_s.bytesize} bytes)"
+      end
+
+      def throttle
+        return if @rate_limit.zero?
+        return if @last_request_at.nil?
+
+        elapsed = Time.now - @last_request_at
+        return if elapsed >= @rate_limit
+
+        sleep(@rate_limit - elapsed)
+      end
+    end
+  end
+end

data/lib/arxiv/downloader/error.rb

@@ -0,0 +1,5 @@
+module Arxiv
+  module Downloader
+    class Error < StandardError; end
+  end
+end

data/lib/arxiv/downloader/feed_parser.rb

@@ -0,0 +1,61 @@
+require 'feedjira'
+
+module Arxiv
+  module Downloader
+    class FeedParser
+      class Author
+        include SAXMachine
+
+        element :name
+      end
+
+      class AtomEntry < Feedjira::Parser::AtomEntry
+        elements :author, as: :authors, class: Author
+
+        element 'arxiv:primary_category', as: :primary_category_id, value: :term
+        element 'arxiv:comment',          as: :comment
+        element 'arxiv:doi',              as: :doi
+        element 'arxiv:journal_ref',      as: :journal_ref
+      end
+
+      class AtomFeed
+        include SAXMachine
+        include Feedjira::FeedUtilities
+
+        elements :entry, as: :entries, class: AtomEntry
+
+        def self.able_to_parse? xml
+          xml.include? 'http://www.w3.org/2005/Atom'
+        end
+      end
+
+      Feedjira.configure { |config| config.parsers = [AtomFeed] + config.parsers }
+
+      def initialize xml
+        @feed       = Feedjira.parse xml
+        @categories = Categories.new
+      end
+
+      def metadata
+        entry    = @feed.entries.first
+        arxiv_id = Identifier.new(entry.entry_id).id
+
+        Metadata.new(
+          arxiv_id:         arxiv_id,
+          arxiv_url:        "https://arxiv.org/abs/#{arxiv_id}",
+          pdf_url:          "https://arxiv.org/pdf/#{arxiv_id}.pdf",
+          title:            entry.title,
+          authors:          entry.authors.map(&:name),
+          abstract:         entry.summary.strip,
+          published:        entry.published.to_date,
+          updated:          entry.updated.to_date,
+          primary_category: @categories.lookup(entry.primary_category_id),
+          categories:       entry.categories.map { |id| @categories.lookup id },
+          comment:          entry.comment,
+          doi:              entry.doi,
+          journal_ref:      entry.journal_ref
+        )
+      end
+    end
+  end
+end

data/lib/arxiv/downloader/html_archive.rb

@@ -0,0 +1,69 @@
+require 'fileutils'
+require 'nokogiri'
+require 'uri'
+
+module Arxiv
+  module Downloader
+    class HTMLArchive
+      ASSET_SELECTORS = {
+        'img[src]'               => 'src',
+        'script[src]'            => 'src',
+        'link[rel="stylesheet"]' => 'href'
+      }.freeze
+
+      def initialize identifier, client:, assets_cache:
+        @identifier   = identifier
+        @client       = client
+        @assets_cache = assets_cache
+      end
+
+      def download to:
+        FileUtils.mkdir_p to
+
+        document = Nokogiri::HTML @client.get(html_url).to_s
+        ASSET_SELECTORS.each do |selector, attribute|
+          document.css(selector).each { |node| process node, attribute, to }
+        end
+
+        File.write File.join(to, "#{@identifier.id}.html"), document.to_html
+      end
+
+      private
+
+      def html_url
+        "https://arxiv.org/html/#{@identifier.id}"
+      end
+
+      def process node, attribute, html_dir
+        reference = node[attribute]
+        return if reference.nil? || reference.empty?
+
+        if relative? reference
+          download_relative reference, html_dir
+        else
+          cache_remote node, attribute, reference, html_dir
+        end
+      end
+
+      def relative? reference
+        URI.parse(reference).scheme.nil?
+      rescue URI::InvalidURIError
+        false
+      end
+
+      def download_relative reference, html_dir
+        File.binwrite File.join(html_dir, reference), @client.get(absolute_for(reference)).to_s
+      end
+
+      def absolute_for reference
+        "https://arxiv.org/html/#{@identifier.id}/#{reference}"
+      end
+
+      def cache_remote node, attribute, reference, html_dir
+        cached_path  = @assets_cache.fetch reference
+        relative_ref = Pathname.new(cached_path).relative_path_from(Pathname.new(html_dir)).to_s
+        node[attribute] = relative_ref
+      end
+    end
+  end
+end

data/lib/arxiv/downloader/identifier.rb

@@ -0,0 +1,135 @@
+module Arxiv
+  module Downloader
+    class Identifier
+      class Invalid < Error; end
+
+      attr_reader :id, :version, :input
+
+      def initialize input
+        @input = input
+        @cleaned_input = input.dup
+
+        validate_input
+        set_id_and_version
+      end
+
+      private
+
+      # validations
+
+      def validate_input
+        invalidate_nil_input
+        invalidate_blank_input
+        invalidate_wrong_domain_input
+        invalidate_no_dot_no_slash_input
+        invalidate_pathless_arxiv_url_input
+        invalidate_apex_arxiv_url_input
+        invalidate_incomplete_arxiv_url_input
+      end
+
+      def invalidate_nil_input
+        raise Invalid, 'blank input is invalid' if @input.nil?
+      end
+
+      def invalidate_blank_input
+        raise Invalid, 'blank input is invalid' if @input.empty?
+      end
+
+      def invalidate_wrong_domain_input
+        invalid = true if @cleaned_input.include?('http') && !@cleaned_input.include?('arxiv.org')
+
+        raise Invalid, 'domains other than arxiv.org are invalid' if invalid
+      end
+
+      def invalidate_no_dot_no_slash_input
+        invalid = true if !@cleaned_input.include?('.') && !@cleaned_input.include?('/')
+
+        raise Invalid, "not a recognizable arXiv identifier: #{input}" if invalid
+      end
+
+      def invalidate_pathless_arxiv_url_input
+        invalid = true if @cleaned_input.split('arxiv.org/').last.nil?
+
+        raise Invalid, "not a recognizable arXiv identifier URL: #{input}" if invalid
+      end
+
+      def invalidate_apex_arxiv_url_input
+        invalid = true if @cleaned_input.split('arxiv.org').last.nil?
+
+        raise Invalid, "not a recognizable arXiv identifier URL: #{input}" if invalid
+      end
+
+      def invalidate_incomplete_arxiv_url_input
+        path = @cleaned_input.split('arxiv.org/').last
+        invalid = true if !path.include?('.') && !path.include?('/')
+
+        raise Invalid, "not a recognizable arXiv identifier URL: #{input}" if invalid
+      end
+
+      # setter
+
+      def set_id_and_version
+        normalize_input
+        @id = @cleaned_input
+      end
+
+      # mutaters
+
+      def normalize_input
+        delete_spaces!
+        strip_slashes!
+        delete_protocols!
+        delete_domain!
+        delete_format_namespaces!
+        delete_arxiv_namespace!
+        delete_file_extensions!
+        extract_version!
+      end
+
+      def delete_spaces!
+        @cleaned_input.chomp!
+        @cleaned_input.strip!
+        @cleaned_input.squeeze! ' '
+      end
+
+      def strip_slashes!
+        loop do
+          @cleaned_input.delete_suffix! '/'
+          break unless @cleaned_input.end_with? '/'
+        end
+      end
+
+      def delete_protocols!
+        @cleaned_input.delete_prefix! 'http://'
+        @cleaned_input.delete_prefix! 'https://'
+      end
+
+      def delete_domain!
+        @cleaned_input.sub! 'arxiv.org', ''
+      end
+
+      def delete_format_namespaces!
+        %w[/abs/ /html/ /pdf/].each do |format_namespace|
+          @cleaned_input.sub! format_namespace, ''
+        end
+      end
+
+      def delete_arxiv_namespace!
+        @cleaned_input.sub!(/(arxiv:)/i, '')
+      end
+
+      def delete_file_extensions!
+        @cleaned_input.delete_suffix! '.pdf'
+      end
+
+      def extract_version!
+        dot_parts   = @cleaned_input.split('.')
+        slash_parts = dot_parts.last.split('/')
+        v_parts     = slash_parts.last.split('v')
+
+        @version    = Integer(v_parts.last) if v_parts.length > 1
+        @cleaned_input.delete_suffix! "v#{@version}" unless @version.nil?
+      end
+    end
+  end
+end

data/lib/arxiv/downloader/metadata/bibtex.rb

@@ -0,0 +1,21 @@
+require 'fileutils'
+
+module Arxiv
+  module Downloader
+    class Metadata
+      class Bibtex
+        FILENAME = 'metadata.bib'.freeze
+
+        def initialize metadata, client: nil
+          @metadata = metadata
+          @client   = client
+        end
+
+        def write to:
+          FileUtils.mkdir_p to
+          File.write File.join(to, FILENAME), Downloader::Bibtex.new(@metadata, client: @client).to_s
+        end
+      end
+    end
+  end
+end

data/lib/arxiv/downloader/metadata/json.rb

@@ -0,0 +1,32 @@
+require 'fileutils'
+require 'json'
+
+module Arxiv
+  module Downloader
+    class Metadata
+      class JSON
+        FILENAME = 'metadata.json'.freeze
+
+        def initialize metadata
+          @metadata = metadata
+        end
+
+        def write to:
+          FileUtils.mkdir_p to
+          File.write File.join(to, FILENAME), "#{::JSON.pretty_generate(serialize(@metadata.to_h))}\n"
+        end
+
+        private
+
+        def serialize object
+          case object
+          when Hash       then object.to_h { |key, value| [key.to_s, serialize(value)] }
+          when Array      then object.map { |item| serialize item }
+          when Date, Time then object.iso8601
+          else object
+          end
+        end
+      end
+    end
+  end
+end

data/lib/arxiv/downloader/metadata/markdown.rb

@@ -0,0 +1,65 @@
+require 'fileutils'
+require 'yaml'
+
+module Arxiv
+  module Downloader
+    class Metadata
+      class Markdown
+        FILENAME = 'metadata.md'.freeze
+
+        def initialize metadata
+          @metadata = metadata
+        end
+
+        def write to:
+          FileUtils.mkdir_p to
+          File.write File.join(to, FILENAME), "#{frontmatter}\n#{body}"
+        end
+
+        private
+
+        def frontmatter
+          "---\n#{::YAML.dump(stringify(frontmatter_hash)).delete_prefix("---\n")}---"
+        end
+
+        def frontmatter_hash
+          @metadata.to_h.merge bibtex_key: bibtex_key
+        end
+
+        def bibtex_key
+          Downloader::Bibtex.new(@metadata).key
+        end
+
+        def stringify object
+          case object
+          when Hash  then object.to_h { |key, value| [key.to_s, stringify(value)] }
+          when Array then object.map { |item| stringify item }
+          else object
+          end
+        end
+
+        def body
+          <<~MARKDOWN
+
+            # #{@metadata.title}
+
+            #{authors_list}
+
+            - Published: #{@metadata.published.iso8601}
+            - Primary category: #{@metadata.primary_category[:id]} — #{@metadata.primary_category[:name]} (#{@metadata.primary_category[:group]})
+            - arXiv: [#{@metadata.arxiv_id}](#{@metadata.arxiv_url})
+            - [PDF](#{@metadata.pdf_url})
+
+            ## Abstract
+
+            #{@metadata.abstract}
+          MARKDOWN
+        end
+
+        def authors_list
+          @metadata.authors.map { |author| "- #{author}" }.join "\n"
+        end
+      end
+    end
+  end
+end

data/lib/arxiv/downloader/metadata/yaml.rb

@@ -0,0 +1,31 @@
+require 'fileutils'
+require 'yaml'
+
+module Arxiv
+  module Downloader
+    class Metadata
+      class YAML
+        FILENAME = 'metadata.yaml'.freeze
+
+        def initialize metadata
+          @metadata = metadata
+        end
+
+        def write to:
+          FileUtils.mkdir_p to
+          File.write File.join(to, FILENAME), ::YAML.dump(stringify(@metadata.to_h))
+        end
+
+        private
+
+        def stringify object
+          case object
+          when Hash  then object.to_h { |key, value| [key.to_s, stringify(value)] }
+          when Array then object.map { |item| stringify item }
+          else object
+          end
+        end
+      end
+    end
+  end
+end

data/lib/arxiv/downloader/metadata.rb

@@ -0,0 +1,19 @@
+module Arxiv
+  module Downloader
+    Metadata = Data.define(
+      :arxiv_id,
+      :arxiv_url,
+      :pdf_url,
+      :title,
+      :authors,
+      :abstract,
+      :published,
+      :updated,
+      :primary_category,
+      :categories,
+      :comment,
+      :doi,
+      :journal_ref
+    )
+  end
+end

data/lib/arxiv/downloader/path.rb

@@ -0,0 +1,31 @@
+module Arxiv
+  module Downloader
+    class Path
+      def initialize metadata
+        @metadata = metadata
+      end
+
+      def to_s
+        [date_dir, category, "#{filesystem_safe_id}-#{slug}"].join '/'
+      end
+
+      private
+
+      def date_dir
+        @metadata.published.strftime '%Y/%m/%d'
+      end
+
+      def category
+        @metadata.primary_category[:id]
+      end
+
+      def filesystem_safe_id
+        @metadata.arxiv_id.tr '/', '-'
+      end
+
+      def slug
+        Slug.new(@metadata.title).to_s
+      end
+    end
+  end
+end

data/lib/arxiv/downloader/pdf.rb

@@ -0,0 +1,20 @@
+module Arxiv
+  module Downloader
+    class PDF
+      def initialize identifier, client:
+        @identifier = identifier
+        @client     = client
+      end
+
+      def download to:
+        File.binwrite to, @client.get(url).to_s
+      end
+
+      private
+
+      def url
+        "https://arxiv.org/pdf/#{@identifier.id}.pdf"
+      end
+    end
+  end
+end

data/lib/arxiv/downloader/slug.rb

@@ -0,0 +1,44 @@
+require 'stringex'
+
+module Arxiv
+  module Downloader
+    class Slug
+      MAX_LENGTH = 80
+
+      TEX_INLINE_MATH  = /\$[^$]*\$/                # $...$
+      TEX_DISPLAY_MATH = /\\\(.*?\\\)|\\\[.*?\\\]/m # \(...\) or \[...\]
+      TEX_COMMAND      = /\\[a-zA-Z]+\*?/           # \emph, \alpha, etc.
+
+      def initialize title
+        @title = title
+      end
+
+      def to_s
+        truncate strip_tex(@title).to_url
+      end
+
+      private
+
+      def strip_tex string
+        string
+          .gsub(TEX_INLINE_MATH, ' ')
+          .gsub(TEX_DISPLAY_MATH, ' ')
+          .gsub(TEX_COMMAND, ' ')
+      end
+
+      def truncate slug
+        return slug if slug.length <= MAX_LENGTH
+
+        words = slug.split '-'
+        result = +''
+        words.each do |word|
+          break if result.length + 1 + word.length > MAX_LENGTH
+
+          result << '-' unless result.empty?
+          result << word
+        end
+        result
+      end
+    end
+  end
+end