arxiv-dl 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5b80906292c193199cba698da4d7e85ff927e41af088c2452b1b3c8ee3234997
-  data.tar.gz: bd732114a37495687cd23d62bbdc67cf645db682c8b3302cc1eb283c64ebd356
+  metadata.gz: 34f7046b2c2b52988b860c9257481f9434ca427feb018346dd66fe87c73ec13d
+  data.tar.gz: 21afcc2a72d40d1a8c98690066d39a05c182aa6739d0cf6cb21cfc54a79ece67
 SHA512:
-  metadata.gz: 76153d238bfc2dddf1d9343e130e10beac24f508193c7429c8bbbe42a603a4451996bfb88dc1b592b2d7e34e31055e313d69c2f09235f28c714f03fa25288189
-  data.tar.gz: 7c508e83063cbf8d6832d34b9d55a89c7fa8ba55c60f97cbfb1aaab08a5c0d4489a2af359bf55d0a483416ec390ed8402ee7f42114cf15365c9ed647bb53028f
+  metadata.gz: ff13eecbd1489e0c865044f69703fa50ccec59a953fe23a227d9c2b4daa2aa2769bb79e538cb1e8bfdbed2dfe4cb8097ec758e761b958de5bf02ffbd9f0483ef
+  data.tar.gz: 55a30abd80c2252150fbe8736fff4a52fbbe4a143737610cbb578e686f771fb2f30bd7238ea69a0bd3fb105426cf136600d3089c545188909d6510517133ae7e

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## [0.1.1]
+
+Bug fix: the HTML archive step crashed with `NoMethodError` on papers whose HTML embeds `data:` URI images (e.g. arxiv's feedback-overlay mascot), and mis-fetched root-relative `/static/...` asset references from a wrong page-relative URL.
+
+- Asset references are now routed by type: page-relative refs download as siblings, absolute `http(s)` refs cache under `_shared/`, root-relative refs are absolutized against `arxiv.org` and cache under `_shared/`, and unfetchable refs (`data:`, `javascript:`, `mailto:`, malformed URIs) are left in the HTML untouched.
+
 ## [0.1.0]
 
 First release. Per-paper offline archive of arxiv.org papers, with PDF, abstract HTML, full HTML version (with relative-path images and a deduplicated cross-paper shared assets cache), TeX source, and four metadata sidecar files.

data/lib/arxiv/downloader/html_archive.rb

@@ -38,17 +38,25 @@ module Arxiv
         reference = node[attribute]
         return if reference.nil? || reference.empty?
 
-        if relative? reference
-          download_relative reference, html_dir
-        else
-          cache_remote node, attribute, reference, html_dir
+        case reference_type(reference)
+        when :page_relative then download_relative reference, html_dir
+        when :remote        then cache_remote node, attribute, reference, html_dir
+        when :root_relative then cache_remote node, attribute, "https://arxiv.org#{reference}", html_dir
         end
       end
 
-      def relative? reference
-        URI.parse(reference).scheme.nil?
+      # :skip covers refs that can't or shouldn't be fetched: data:/javascript:/
+      # mailto: URIs, protocol-relative refs, and malformed URIs. They are left
+      # in the HTML untouched.
+      def reference_type reference
+        uri = URI.parse reference
+        return :remote if %w[http https].include? uri.scheme
+        return :skip unless uri.scheme.nil? && uri.host.nil?
+        return :root_relative if reference.start_with? '/'
+
+        :page_relative
       rescue URI::InvalidURIError
-        false
+        :skip
       end
 
       def download_relative reference, html_dir

data/lib/arxiv/downloader/version.rb

@@ -1,5 +1,5 @@
 module Arxiv
   module Downloader
-    VERSION = '0.1.0'.freeze
+    VERSION = '0.1.1'.freeze
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: arxiv-dl
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Shane Becker