arxiv-dl 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 36af277147beae3a55221ba50443204db5b9e4bdae1613a3d9e923a465d4bc31
-  data.tar.gz: 04d405743856249d2f78d4f9e711e73b716ddb27f1b6633ad92a96e6385677be
+  metadata.gz: c919d2e16ae72397375aa3aa5f71bd08fe4f3d0687d83358bf652081ee994596
+  data.tar.gz: 35a55cb7152642e19816a76b4a00fe52b06f74d5a52cb587d2e2b6897e23ac61
 SHA512:
-  metadata.gz: 92527994ad88aec89a64f3f678f60ff8a9d286d3189d06a6eafa4a9ba4f5a02fa815e9d23c0e429c0ea44ad03d3d58ae870c79a001c0705a682fd22818210412
-  data.tar.gz: 5058f511ec0e2707275605bee72d29d44c190e563e9b83da65e8b8619199186345c78231b35aa685fe183d6c0dd6f6996115059fd72c991c8d7babb39dfe7be8
+  metadata.gz: 67f1ea3177544e5c4e5472d2b1e81d894a42346313ae4666f53f9de89571b36a55d50ee247dae0b8af2dce869cec03ea72890009817ee613b7cab37b93bddbfe
+  data.tar.gz: 5bcc8fc0ab51a75032ed0a3d8d2e20dad830f9b6bb92f75841be0fa23e05f60d66e76cb7e23c5a8b17fffba0634213d3b4299fde0c86314858f396135d2a9a84

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## [0.3.0]
+
+Less folder nesting for the common case: most papers only ever have one version.
+
+### Breaking
+
+- Single-version papers are archived flat again: a paper whose only archived version is v1 keeps its files directly in the paper folder, without a `v1/` level. `v<N>/` folders are used only when a paper has more than one version. Archiving a second version of a flat paper moves the existing files into `v<N>/` first, rewriting `html/` links into `_shared/` for the new depth. A paper whose latest version is v2 or later starts out in `v<N>/` folders. Existing 0.2.0 archives keep their `v1/` folders.
+
 ## [0.2.0]
 
 Versioned archives, author affiliations, and a round of robustness fixes. Two breaking changes to the output layout and metadata; see below.

data/README.md

@@ -108,27 +108,40 @@ $ARXIV_DOWNLOAD_PATH/                   # default: $HOME/Downloads/ArXiv_Papers
     arxiv.org/static/...
     cdn.jsdelivr.net/...
   YYYY/MM/DD/<primary_category>/<arxiv-id>-<slug>/
-    v<N>/                               # one folder per archived version
-      <arxiv-id>v<N>.pdf
-      <arxiv-id>v<N>-abstract.html
-      metadata.md                       # YAML frontmatter + Markdown body
-      metadata.yaml
-      metadata.json
-      metadata.bib                      # upstream BibTeX, falls back to synthesized
-      html/                             # absent when the paper has no HTML version
-        <arxiv-id>v<N>.html             # path-rewritten to local assets
-        x1.png, x2.png, ...             # paper-specific images
-      src/                              # absent for PDF-only submissions
-        *.tex, *.bbl, ...               # extracted from /src/<id>v<N>
+    <arxiv-id>v<N>.pdf
+    <arxiv-id>v<N>-abstract.html
+    metadata.md                         # YAML frontmatter + Markdown body
+    metadata.yaml
+    metadata.json
+    metadata.bib                        # upstream BibTeX, falls back to synthesized
+    html/                               # absent when the paper has no HTML version
+      <arxiv-id>v<N>.html               # path-rewritten to local assets
+      x1.png, x2.png, ...               # paper-specific images
+    src/                                # absent for PDF-only submissions
+      *.tex, *.bbl, ...                 # extracted from /src/<id>v<N>
 ```
 
-An unversioned ID (`2508.16190`) archives the latest version. A versioned ID (`2508.16190v1`) archives that version. Different versions of the same paper sit side by side under the same paper folder.
+An unversioned ID (`2508.16190`) archives the latest version. A versioned ID (`2508.16190v1`) archives that version.
 
-Each version downloads into `v<N>.partial/` and is renamed to `v<N>/` only when every file succeeded. Re-running skips versions whose `v<N>/` already exists and retries interrupted ones from scratch.
+A paper with a single archived version v1 is kept flat, as above. When a paper has more than one version, each version gets its own `v<N>/` folder with the same contents:
+
+```txt
+  YYYY/MM/DD/<primary_category>/<arxiv-id>-<slug>/
+    v1/
+      <arxiv-id>v1.pdf
+      ...
+    v2/
+      <arxiv-id>v2.pdf
+      ...
+```
+
+Archiving a second version of a flat paper first moves the existing files into `v<N>/` (rewriting `html/` links into `_shared/` for the new depth). A paper whose latest version is v2 or later starts out in `v<N>/` folders.
+
+Each version downloads into a sibling `.partial` folder and is renamed into place only when every file succeeded. Re-running skips versions already archived and retries interrupted ones from scratch.
 
 `YYYY/MM/DD` is the original submission date. `<primary_category>` is from the paper's metadata (`cs.CL`, `math.NT`, etc). `<slug>` is derived from the paper title (Unicode → ASCII, hyphenated, truncated to 80 chars at a word boundary).
 
-For legacy IDs containing `/` (e.g. `cs/0002001`), the slash is replaced with `-` in the directory name and file names (`cs-0002001-.../v1/cs-0002001v1.pdf`).
+For legacy IDs containing `/` (e.g. `cs/0002001`), the slash is replaced with `-` in the directory name and file names (`cs-0002001-.../cs-0002001v1.pdf`).
 
 ## Library usage
 
@@ -138,7 +151,7 @@ require 'arxiv/downloader'
 identifier = Arxiv::Downloader::Identifier.new '2508.16190'
 client     = Arxiv::Downloader::Client.new                # 3-second rate limit by default
 path       = Arxiv::Downloader::Archive.new(identifier, root: '/tmp/papers', client: client).run
-# => "/tmp/papers/2025/08/22/cs.CL/2508.16190-comicscene154-a-scene-dataset-for-comic-analysis/v1"
+# => "/tmp/papers/2025/08/22/cs.CL/2508.16190-comicscene154-a-scene-dataset-for-comic-analysis"
 ```
 
 ## Development

data/lib/arxiv/downloader/archive.rb

@@ -9,12 +9,14 @@ module Arxiv
         @client     = client
       end
 
-      # Downloads into v<N>.partial/ and
-      # renames it to v<N>/ only once everything succeeded.
-      # An existing v<N>/ is always complete and is skipped.
+      # Downloads into <destination>.partial/ and
+      # renames it into place only once everything succeeded.
+      # An archived version is always complete and is skipped.
+      # PaperFolder decides flat (single version) vs v<N>/ (several versions).
       def run
-        return paper_dir if Dir.exist? paper_dir
+        return paper_folder.location_of(metadata.version) if paper_folder.archived? metadata.version
 
+        paper_folder.unflatten!
         FileUtils.rm_rf staging_dir
         FileUtils.mkdir_p staging_dir
 
@@ -24,8 +26,8 @@ module Arxiv
         download_source_archive
         write_sidecars
 
-        File.rename staging_dir, paper_dir
-        paper_dir
+        File.rename staging_dir, destination
+        destination
       end
 
       private
@@ -44,13 +46,18 @@ module Arxiv
         @archived ||= Identifier.new "#{metadata.arxiv_id}v#{metadata.version}"
       end
 
-      def paper_dir
-        @paper_dir ||= File.join @root, Path.new(metadata).to_s, "v#{metadata.version}"
+      def paper_folder
+        @paper_folder ||= PaperFolder.new File.join(@root, Path.new(metadata).to_s)
       end
 
-      # a sibling of paper_dir, so relative ../_shared/ refs survive the rename
+      # evaluated after unflatten!, which can turn a flat folder into v<N>/ folders
+      def destination
+        @destination ||= paper_folder.destination_for metadata.version
+      end
+
+      # a sibling of destination, so relative ../_shared/ refs survive the rename
       def staging_dir
-        "#{paper_dir}.partial"
+        "#{destination}.partial"
       end
 
       def download_pdf

data/lib/arxiv/downloader/paper_folder.rb

@@ -0,0 +1,85 @@
+require 'fileutils'
+require 'nokogiri'
+require 'yaml'
+
+module Arxiv
+  module Downloader
+    # One paper's folder. A paper with a single archived version is kept flat
+    # (files directly in the folder); once a second version arrives, each
+    # version gets its own v<N>/ folder.
+    class PaperFolder
+      VERSION_FOLDER = /\Av\d+(\.partial)?\z/
+
+      attr_reader :path
+
+      def initialize path
+        @path = path
+      end
+
+      def archived? version
+        Dir.exist? location_of(version)
+      end
+
+      def location_of version
+        return path if flat_version == version
+
+        version_path version
+      end
+
+      # where a new download of `version` goes; call unflatten! first if the folder is flat
+      def destination_for version
+        return path if version == 1 && !versioned?
+
+        version_path version
+      end
+
+      # moves the flat version into v<N>/ so another version can sit beside it
+      def unflatten!
+        return if flat_version.nil?
+
+        destination = version_path flat_version
+        FileUtils.mkdir_p destination
+
+        # metadata.yaml marks the folder as flat, so it moves last
+        children = Dir.children(path) - [File.basename(destination), metadata_filename]
+        children.each { FileUtils.mv File.join(path, it), destination }
+        deepen_shared_links File.join(destination, 'html')
+        FileUtils.mv File.join(path, metadata_filename), destination
+      end
+
+      private
+
+      def flat_version
+        metadata_path = File.join path, metadata_filename
+        return unless File.exist? metadata_path
+
+        ::YAML.safe_load_file(metadata_path, permitted_classes: [Date]).fetch 'version'
+      end
+
+      def versioned?
+        Dir.exist?(path) && Dir.children(path).any? { VERSION_FOLDER.match? it }
+      end
+
+      def version_path version
+        File.join path, "v#{version}"
+      end
+
+      def metadata_filename
+        Metadata::YAML::FILENAME
+      end
+
+      # html/ moved one level deeper, so relative links up into _shared/ need one more ../
+      def deepen_shared_links html_dir
+        Dir.glob(File.join(html_dir, '*.html')).each do |html_path|
+          document = Nokogiri::HTML File.read(html_path)
+          HTMLArchive::ASSET_SELECTORS.each do |selector, attribute|
+            document.css(selector).each do |node|
+              node[attribute] = "../#{node[attribute]}" if node[attribute].start_with? '../'
+            end
+          end
+          File.write html_path, document.to_html
+        end
+      end
+    end
+  end
+end

data/lib/arxiv/downloader/version.rb

@@ -1,5 +1,5 @@
 module Arxiv
   module Downloader
-    VERSION = '0.2.0'.freeze
+    VERSION = '0.3.0'.freeze
   end
 end

data/lib/arxiv/downloader.rb

@@ -16,6 +16,7 @@ require_relative 'downloader/metadata/bibtex'   # after metadata
 require_relative 'downloader/metadata/json'     # after metadata
 require_relative 'downloader/metadata/markdown' # after metadata
 require_relative 'downloader/metadata/yaml'     # after metadata
+require_relative 'downloader/paper_folder'
 require_relative 'downloader/paper_not_found'   # after error
 require_relative 'downloader/path'
 require_relative 'downloader/pdf'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: arxiv-dl
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Shane Becker
@@ -116,6 +116,7 @@ files:
 - lib/arxiv/downloader/metadata/json.rb
 - lib/arxiv/downloader/metadata/markdown.rb
 - lib/arxiv/downloader/metadata/yaml.rb
+- lib/arxiv/downloader/paper_folder.rb
 - lib/arxiv/downloader/paper_not_found.rb
 - lib/arxiv/downloader/path.rb
 - lib/arxiv/downloader/pdf.rb