sfdown 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 44df4c3a6fc531cc1ef2f8454abf8d6224980ee4d896d27508741df69f3aab11
-  data.tar.gz: 562025b5cda6afd162a623cfec735badf2f8fc5aad69b4588392d8b24d8a6299
+  metadata.gz: 5ac26367a5dfb5104d0765390e182530c231d080825c2af68ac3cd55746eefb7
+  data.tar.gz: 1f6198ec690293e27ee67165c1ae6e131a08564398d529ae5107c65cfce66eae
 SHA512:
-  metadata.gz: 9df2bf33aca1160bfd233dfcc9cc8c4cd91bf8341b2ddffbec7ced700b972bdc8bd9c8350b5e602ce0116df989f694a2bf3570fa7551f4a0f64e7d19a2b3d3ce
-  data.tar.gz: 6ecd5025e1afc37fd265c91e444c47773d6b8d3d648946d7e3ba454f1f9dab897fc8d6b86f0aeef789c86a8b8fd05f549ba6d1aad09a558f828542516b9c4920
+  metadata.gz: b9cc50d15933e4d0c3bf55447c8a10ab9caf651e0fb4ef52540cbfcfda5fe8d02b39f102ff43a500de06471d5865e22a0a82508c706f45ba93dbbb2230f80057
+  data.tar.gz: a3ce4092601da155a1e9311aa1d1bc88ef2b7a05435381f07d07499e8b470bec1c629467b881c305dfa091bb8ce3a4ce8c4bef52405ce551056224b96347ae0d

data/CHANGELOG.md

@@ -2,6 +2,14 @@
 
 All notable changes to this project are documented here.
 
+## [0.4.1] - 2026-07-25
+
+Resumable downloads.
+
+- Downloader now skips files already on disk whose content verifies against their MD5/SHA1 hash, so an interrupted download can be resumed by rerunning the same command (works with `-i` too).
+- Missing, corrupt, or checksum-less files are re-downloaded.
+- Final summary reports how many files were resumed and how many bytes that skipped.
+
 ## [0.4.0] - 2026-07-24
 
 Added file deduplication.

data/README.md

@@ -1,8 +1,11 @@
 # sfdown
 
+[![Gem Version](https://img.shields.io/gem/v/sfdown.svg)](https://rubygems.org/gems/sfdown)
+[![Gem](https://img.shields.io/gem/dt/sfdown.svg)](https://rubygems.org/gems/sfdown)
+
 SourceForge project downloader. Will clone the project's directory tree and fetch all files.
 
-Supports [concurrency](#concurrency-and-throttling), [integrity verification](#integrity-verification) and [file deduplication](#file-deduplication).
+Supports [concurrency](#concurrency-and-throttling), [integrity verification](#integrity-verification), [file deduplication](#file-deduplication) and [resuming](#resuming-an-interrupted-download) interrupted downloads.
 
 ## Installation
 
@@ -58,4 +61,10 @@ Multiple workers can be used in parallel for both stages via the `-c` flag.
 
 To prevent rate-limits, an optional sleep can be added between fetches via the `-s` flag.
 
-**Note**: `-s` is applied per worker, so with `-c` the effective request rate scales roughly linearly with the number of workers.
+**Note**: `-s` is applied per worker, so with `-c` the effective request rate scales roughly linearly with the number of workers.
+
+### Resuming interrupted downloads
+
+The download process (stage 2) can be resumed by simply running the same command again, whether the process had been bootstrapped with `-i` or not.
+
+Before downloading anything, sfdown checks each file's local path and verifies any existing copy against its published **MD5** / **SHA1** hash. Files that match are kept and skipped; anything missing, corrupt, or unverifiable (no checksum) is re-downloaded.

data/lib/sfdown/version.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
 module Sfdown
-  VERSION = "0.4.0"
-  DATE = "2026/07/24"
+  VERSION = "0.4.1"
+  DATE = "2026/07/25"
 end

data/lib/sfdown.rb

@@ -420,8 +420,10 @@ module Sfdown
       @total_files = 0
       @total_bytes = 0
       @duped_bytes = 0
-      @files_done = 0
-      @bytes_done = 0
+      @resumed_files = 0
+      @resumed_bytes = 0
+      @done_files = 0    # completed this session (fetched + cloned)
+      @done_bytes = 0    # streamed this session
       @duped_files = 0
       @failures = 0
       @mismatches = 0
@@ -429,16 +431,22 @@ module Sfdown
       @current = ""
     end
 
-    def files_done = @mutex.synchronize { @files_done }
-    def bytes_done = @mutex.synchronize { @bytes_done }
+    # Progress counters credit files already on disk from a previous run, so a
+    # resumed download shows e.g. 500/700 MB rather than restarting at 0.
+    def files_done = @mutex.synchronize { @resumed_files + @done_files }
+    def bytes_done = @mutex.synchronize { @resumed_bytes + @done_bytes }
+    def downloaded_bytes = @mutex.synchronize { @done_bytes } # this session only; drives speed
+    def resumed_files = @mutex.synchronize { @resumed_files }
+    def resumed_bytes = @mutex.synchronize { @resumed_bytes }
     def duped_files = @mutex.synchronize { @duped_files }
     def failures = @mutex.synchronize { @failures }
     def mismatches = @mutex.synchronize { @mismatches }
     def current = @mutex.synchronize { @current }
 
-    # Create every directory up front, then split the files into those to fetch
-    # and the duplicates that will be cloned from them. total_bytes counts only
-    # what will actually cross the network, so the progress bar stays honest.
+    # Create every directory up front, then split the files into those already
+    # on disk (resume), those to fetch, and the duplicates cloned from them.
+    # total_bytes covers the resumed + fetched bytes, so a resumed run's bar
+    # starts partway to 100% instead of at zero.
     def prepare(root)
       FileUtils.mkdir_p(@dest)
       files = []
@@ -448,8 +456,9 @@ module Sfdown
       end
       @total_files = files.size
       plan(files)
-      @total_bytes = @files.sum(&:size)
+      @total_bytes = @resumed_bytes + @files.sum(&:size)
       @duped_bytes = @dupes.sum { |node, _| node.size }
+      log_resume if @resumed_files.positive?
     end
 
     # The work list is fixed here (unlike stage 1), so the queue is filled and
@@ -482,14 +491,19 @@ module Sfdown
       Pool.run(queue, [@concurrent, files.size].min) { |node| download_one(node) }
     end
 
-    # The first node carrying a given checksum is downloaded; any later node with
-    # the same one is a duplicate of it. Files SF gives no checksum for can't be
-    # deduped, so they're always fetched.
+    # Classify every file. A copy already on disk that verifies against its
+    # checksum is kept as-is (resume). Otherwise the first node carrying a given
+    # checksum is downloaded and any later node with the same one is a duplicate
+    # of it. Files SF gives no checksum for can't be verified or deduped, so they
+    # are always fetched (their on-disk state can't be trusted).
     def plan(files)
       sources = {}
       files.each do |node|
         key = dedupe_key(node)
-        if (source = key && sources[key])
+        if present?(node)
+          resume(node)
+          sources[key] ||= node if key # a verified local copy can source dupes
+        elsif key && (source = sources[key])
           @dupes << [node, source]
         else
           sources[key] = node if key
@@ -503,19 +517,44 @@ module Sfdown
       algo && "#{algo}:#{hash.downcase}"
     end
 
+    # Count an already-present file as done and register it so dupes can clone it.
+    def resume(node)
+      @resumed_files += 1
+      @resumed_bytes += node.size
+      @ok << node.path
+    end
+
+    # True if the local file exists and its content matches the node's checksum.
+    # Without a checksum we can't confirm integrity, so we never trust the copy.
+    def present?(node)
+      algo, expected = expected_hash(node)
+      return false unless algo
+
+      path = local_path(node)
+      File.file?(path) && digest_file(path, algo).casecmp?(expected)
+    rescue StandardError
+      false
+    end
+
+    def digest_file(path, algo)
+      digest = Digest.const_get(algo).new
+      File.open(path, "rb") { |f| digest.update(f.read(1 << 16)) while !f.eof? }
+      digest.hexdigest
+    end
+
     def download_one(node)
       @mutex.synchronize { @current = node.path }
       dest = local_path(node)
       algo, expected = expected_hash(node)
       digest = algo && Digest.const_get(algo).new
       @http.download(Sf.file_url(@project, node.path), dest) do |chunk|
-        @mutex.synchronize { @bytes_done += chunk.bytesize }
+        @mutex.synchronize { @done_bytes += chunk.bytesize }
         digest&.update(chunk)
       end
       File.utime(node.timestamp, node.timestamp, dest) if node.timestamp
       verify(node, digest.hexdigest, expected) if digest
       @mutex.synchronize do
-        @files_done += 1
+        @done_files += 1
         @ok << node.path
       end
     rescue StandardError => e
@@ -541,7 +580,7 @@ module Sfdown
       # original's timestamps; only a real copy can carry its own.
       File.utime(node.timestamp, node.timestamp, dest) if node.timestamp && !linked
       @mutex.synchronize do
-        @files_done += 1
+        @done_files += 1
         @duped_files += 1
         @ok << node.path
       end
@@ -585,6 +624,10 @@ module Sfdown
       @log.call("WARN: checksum mismatch for #{node.path} (expected #{expected}, got #{actual})")
     end
 
+    def log_resume
+      @log.call("Resuming: #{@resumed_files} verified file(s) already present (#{Fmt.size(@resumed_bytes)}), skipping them")
+    end
+
     def each_node(node, &blk)
       blk.call(node)
       node.children.each { |c| each_node(c, &blk) }
@@ -782,7 +825,9 @@ module Sfdown
     def render_stage2(bar, dl, global_start, stage2_start)
       now = Time.now
       s2 = now - stage2_start
-      speed = s2.positive? ? dl.bytes_done / s2 : 0
+      # Speed reflects only bytes moved this session; resumed bytes weren't
+      # transferred now, so counting them would inflate speed and skew the ETA.
+      speed = s2.positive? ? dl.downloaded_bytes / s2 : 0
       remaining = [dl.total_bytes - dl.bytes_done, 0].max
       eta = speed.positive? ? remaining / speed : 0
       bar.update("Fetching #{dl.current}", stage2_line(dl, now - global_start, speed, (now - global_start) + eta))
@@ -807,10 +852,13 @@ module Sfdown
       downloader.apply_folder_times(root)
 
       s2 = Time.now - stage2_start
-      avg = s2.positive? ? downloader.bytes_done / s2 : 0
-      summary = format("Stage 2: %d/%d files, %s in %s (avg %s/s)",
+      avg = s2.positive? ? downloader.downloaded_bytes / s2 : 0
+      summary = format("Stage 2: %d/%d files, %s downloaded in %s (avg %s/s)",
                        downloader.files_done, downloader.total_files,
-                       Fmt.size(downloader.bytes_done), Fmt.duration(s2), Fmt.size(avg))
+                       Fmt.size(downloader.downloaded_bytes), Fmt.duration(s2), Fmt.size(avg))
+      if downloader.resumed_files.positive?
+        summary += format(" (%d resumed, %s)", downloader.resumed_files, Fmt.size(downloader.resumed_bytes))
+      end
       if downloader.duped_files.positive?
         summary += format(" (%d duplicates, %s saved)", downloader.duped_files, Fmt.size(downloader.duped_bytes))
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sfdown
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - edelkas
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-07-24 00:00:00.000000000 Z
+date: 2026-07-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -27,7 +27,8 @@ dependencies:
 description: Clones a SourceForge project's directory tree and downloads its files
   by scraping the Files pages (SourceForge has no official API). Runs in two stages
   - map the tree, then download - with a live status bar and optional JSON metadata.
-  Supports concurrency and integrity verification (MD5 / SHA1).
+  Supports concurrency, file integrity verification (MD5 / SHA1), file deduplication,
+  and resuming interrupted downloads.
 email:
 - edlucasma@gmail.com
 executables: