arxiv-dl 0.1.0

data/lib/arxiv/downloader/source_archive.rb

@@ -0,0 +1,43 @@
+require 'fileutils'
+require 'rubygems/package'
+require 'stringio'
+require 'zlib'
+
+module Arxiv
+  module Downloader
+    class SourceArchive
+      def initialize identifier, client:
+        @identifier = identifier
+        @client     = client
+      end
+
+      def download to:
+        FileUtils.mkdir_p to
+
+        body = @client.get(url).to_s
+        Zlib::GzipReader.wrap StringIO.new(body) do |gz|
+          Gem::Package::TarReader.new(gz) do |tar|
+            tar.each { |entry| extract entry, to }
+          end
+        end
+      end
+
+      private
+
+      def url
+        "https://arxiv.org/src/#{@identifier.id}"
+      end
+
+      def extract entry, root
+        path = File.join root, entry.full_name
+
+        if entry.directory?
+          FileUtils.mkdir_p path
+        elsif entry.file?
+          FileUtils.mkdir_p File.dirname(path)
+          File.binwrite path, entry.read
+        end
+      end
+    end
+  end
+end

data/lib/arxiv/downloader/version.rb

@@ -0,0 +1,5 @@
+module Arxiv
+  module Downloader
+    VERSION = '0.1.0'.freeze
+  end
+end

data/lib/arxiv/downloader.rb

@@ -0,0 +1,26 @@
+require_relative 'downloader/version'
+require_relative 'downloader/error'
+require_relative 'downloader/identifier'
+require_relative 'downloader/slug'
+require_relative 'downloader/categories'
+require_relative 'downloader/client'
+require_relative 'downloader/metadata'
+require_relative 'downloader/feed_parser'
+require_relative 'downloader/path'
+require_relative 'downloader/pdf'
+require_relative 'downloader/bibtex'
+require_relative 'downloader/abstract_page'
+require_relative 'downloader/source_archive'
+require_relative 'downloader/assets_cache'
+require_relative 'downloader/html_archive'
+require_relative 'downloader/metadata/yaml'
+require_relative 'downloader/metadata/json'
+require_relative 'downloader/metadata/bibtex'
+require_relative 'downloader/metadata/markdown'
+require_relative 'downloader/archive'
+require_relative 'downloader/cli'
+
+module Arxiv
+  module Downloader
+  end
+end

data/sig/arxiv/downloader.rbs

@@ -0,0 +1,5 @@
+module Arxiv
+  module Downloader
+    VERSION: String
+  end
+end

metadata

@@ -0,0 +1,146 @@
+--- !ruby/object:Gem::Specification
+name: arxiv-dl
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Shane Becker
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: feedjira
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: http
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.19'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.19'
+- !ruby/object:Gem::Dependency
+  name: ostruct
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: stringex
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.8'
+description: Command line tool and Ruby library for archiving arxiv.org papers as
+  PDFs with sidecar metadata.
+email:
+- veganstraightedge@gmail.com
+executables:
+- arxiv-dl
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CODE_OF_CONDUCT.md
+- LICENSE.md
+- README.md
+- exe/arxiv-dl
+- lib/arxiv/downloader.rb
+- lib/arxiv/downloader/abstract_page.rb
+- lib/arxiv/downloader/archive.rb
+- lib/arxiv/downloader/assets_cache.rb
+- lib/arxiv/downloader/bibtex.rb
+- lib/arxiv/downloader/categories.rb
+- lib/arxiv/downloader/categories.yaml
+- lib/arxiv/downloader/cli.rb
+- lib/arxiv/downloader/client.rb
+- lib/arxiv/downloader/error.rb
+- lib/arxiv/downloader/feed_parser.rb
+- lib/arxiv/downloader/html_archive.rb
+- lib/arxiv/downloader/identifier.rb
+- lib/arxiv/downloader/metadata.rb
+- lib/arxiv/downloader/metadata/bibtex.rb
+- lib/arxiv/downloader/metadata/json.rb
+- lib/arxiv/downloader/metadata/markdown.rb
+- lib/arxiv/downloader/metadata/yaml.rb
+- lib/arxiv/downloader/path.rb
+- lib/arxiv/downloader/pdf.rb
+- lib/arxiv/downloader/slug.rb
+- lib/arxiv/downloader/source_archive.rb
+- lib/arxiv/downloader/version.rb
+- sig/arxiv/downloader.rbs
+homepage: https://github.com/xoengineering/arxiv-dl
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/xoengineering/arxiv-dl
+  source_code_uri: https://github.com/xoengineering/arxiv-dl
+  changelog_uri: https://github.com/xoengineering/arxiv-dl/blob/main/CHANGELOG.md
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 4.0.6
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.16
+specification_version: 4
+summary: Download papers and metadata from arxiv.org for offline archives.
+test_files: []