sfdown 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: fa096fb3592683630bc0c865b1d5d0108e9a513f9a56585d429f7cc3e6d0d7b9
+  data.tar.gz: 9fc819fc96022335efc1c6f8f25a139c1cbe7a02ef9b4c20be5927b1952768fa
+SHA512:
+  metadata.gz: fb087cc0e7247b3cb34d4382e5a01b65cc1f4d1aea1c1ecbedf7f80ff40055c047f35060b3e452e61fa30d223357bf50acf029148198e484af30262caadf7403
+  data.tar.gz: b57deab485055a92164f7d974239154022b09fc77a49d9d0a86991bd45948a50b57559a06750617295366db7c5e508476d4ce12ebb9d53f3892474d8bfc0dabf

data/CHANGELOG.md

@@ -0,0 +1,15 @@
+# Changelog
+
+All notable changes to this project are documented here.
+
+## [1.0.0] - 2026-07-22
+
+Initial release.
+
+- Two-stage download: map the directory tree, then fetch files.
+- Scrapes the SourceForge "Files" pages with Nokogiri (HTML table + the
+  `net.sf.files` JS metadata object).
+- Preserves file and folder timestamps.
+- Live two-line status bar with per-stage analytics.
+- Options: `-c` concurrency, `-m` metadata, `-n` structure-only, `-o` output,
+  `-t` timeout, `-s` sleep (first two still unimplemented).

data/DOC.md

@@ -0,0 +1,51 @@
+# SourceForge project format
+
+This document contains essential information for automatically parsing and downloading a SourceForge project.
+
+The info is current as of 2026-07-21.
+
+## URL templates
+
+The first two templates allow to browse and map out the directory tree. The third one can be used to fetch each found file.
+
+- Project tree root: `https://sourceforge.net/projects/[PROJECT_NAME]/files/`
+- Directory page: `https://sourceforge.net/projects/[PROJECT_NAME]/files/[FULL_PATH]/`
+- Direct file download: `https://downloads.sourceforge.net/project/[PROJECT_NAME]/[FULL_PATH]`
+
+The direct file download link might actually redirect a few times (via HTTP 3xx codes). In those cases the `location` header link should be followed until the final link can be resolved.
+
+## HTML structure
+
+The current relevant structure of each directory's HTML page:
+
+- Contents stored in a **<table>** with id `files_list`.
+- It has the following 4 **<col>** elements (not within a **<colgroup>**) with self-explanatory classes: `name-column`, `date-column`, `size-column` and `downloads-column`.
+- Each **<tr>** in its **<tbody>** represents an entry of the directory. The class is either `folder` or `file`, and the title is the folder / file name.
+- Each **<th>** / **<td>** represents an attribute of the corresponding entry:
+    - The first one has a headers attribute of `files_name_h`, and contains a **<span>** element with class `name` whose content should match the title of the row, i.e., the folder / file name. It also contains an **<a>** link relative to SourceForge's root:
+        - For folders this link can be used to navigate the directory tree.
+        - For files this link should be ignored, as it directs to the user-facing download page, instead of the direct link found in [URL templates](#url-templates).
+    - The second one has a headers attribute of `files_date_h`, and contains the timestamp inside an **<abbr>** element. The full one follows the format `YYYY-mm-dd HH:MM:SS UTC`, the short one `YYYY-mm-dd`.
+    - The third one has a headers attribute of `files_size_h`. It is empty for folders, and is in human-readable form for files (e.g. `3.4 MB`).
+    - The fourth one has a headers attribute of `files_downloads_h`, and contains a **<span>** element with class `count` whose content is an integer representing the number of weekly downloads. For folders these are the counts of its contents aggregated recursively.
+
+## Additional metadata
+
+Additional file information can be found inside a **<script>** element that defines the object `net.sf.files` with metadata for each folder / file:
+
+    - The **key** is the folder / file name, i.e. the title attribute of the corresponding table row.
+    - The **value** is another object with the corresponding properties, potentially including the following ones:
+        - **name**: String, the folder / file name, should probably coincide with all other mentioned instances of this name.
+        - **path**: String, file base path relative to project's root.
+        - **download_url**: String, absolute link. For files, links to user-facing download page, same as **<a>** attribute in the **<td>** element. For folders, redirects to project root.
+        - **url**: String, relative link. For folders, links to directory page. For files, redirects to user-facing download page.
+        - **full_path**: String, file path relative to project's root (base path + name).
+        - **type**: String, `d` for directories and `f` for files.
+        - **downloads**: Integer, total download count, aggregated recursively for folders.
+        - **sha1**: String, SHA1 hash as lowercase hexdump for files, empty for folders.
+        - **md5**: String, MD5 hash as lowercase hexdump for files, empty for folders.
+        - **default**: String, comma-separated list of supported OS's for files, empty for folders.
+        - **downloadable**: Boolean, generally `true` for files and `false` for folders.
+        - **files_url**: String, relative link to project tree root (see [URL templates](#url-templates)).
+
+Note that in the above descriptions a "paths" are w.r.t. the project tree itself, whereas "urls" are w.r.t SourceForge's server, either absolute or relative.

data/LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 edelkas
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,30 @@
+# sfdown
+
+SourceForge project downloader. Will clone the project's directory tree and fetch all files.
+
+## Installation
+
+```
+gem install sfdown
+```
+
+## Usage
+
+Run with:
+
+```
+sfdown project_name [-mn] [-c concurrent] [-o output] [-t timeout] [-s sleep]
+```
+
+Arguments:
+
+| Arguments | Default | Description |
+|--|--|--|
+| `-c` or `--concurrent` | 1 | Number of parallel downloads |
+|`-m` or `--metadata`|False|Save metadata to disk at project's root|
+|`-n` or `--no`|False|Only fetch directory tree structure and file metadata, not files|
+|`-o` or `--output`|Current dir|Path to store project's root|
+|`-t` or `--timeout`|5|Timeout for each GET request|
+|`-s` or `--sleep`|0|Wait in-between requests|
+
+**Note**: `-c` and `-m` are still unimplemented.

data/exe/sfdown

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "sfdown"
+
+Sfdown::CLI.main(ARGV)

data/lib/sfdown/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Sfdown
+  VERSION = "0.1.0"
+end

data/lib/sfdown.rb

@@ -0,0 +1,601 @@
+# frozen_string_literal: true
+
+# sfdown — SourceForge project downloader.
+# Clones a project's directory tree and downloads its files by scraping the
+# SF "Files" pages.
+
+require "optparse"
+require "net/http"
+require "uri"
+require "json"
+require "time"
+require "io/console"
+require "fileutils"
+require "nokogiri"
+
+require_relative "sfdown/version"
+
+module Sfdown
+  # Parsed CLI configuration.
+  Config = Struct.new(
+    :project, :concurrent, :metadata, :no, :output, :timeout, :sleep,
+    keyword_init: true
+  )
+
+  module Options
+    DEFAULTS = {
+      concurrent: 1,
+      metadata: false,
+      no: false,
+      output: ".",
+      timeout: 5,
+      sleep: 0.0
+    }.freeze
+
+    # Build the OptionParser and the options hash it fills.
+    def self.parser(opts)
+      OptionParser.new do |o|
+        o.banner = "Usage: sfdown project_name [-mn] " \
+                   "[-c concurrent] [-o output] [-t timeout] [-s sleep]"
+
+        o.on("-c", "--concurrent N", Integer, "Number of parallel downloads (default 1) (unimplemented)") do |v|
+          raise OptionParser::InvalidArgument, "#{v} (must be >= 1)" if v < 1
+
+          opts[:concurrent] = v
+        end
+        o.on("-m", "--metadata", "Save metadata to disk at project's root (unimplemented)") { opts[:metadata] = true }
+        o.on("-n", "--no", "Only fetch directory tree structure and file metadata, not files") { opts[:no] = true }
+        o.on("-o", "--output PATH", String, "Path to store project's root (default current dir)") { |v| opts[:output] = v }
+        o.on("-t", "--timeout SECS", Integer, "Timeout for each GET request (default 5)") do |v|
+          raise OptionParser::InvalidArgument, "#{v} (must be > 0)" if v <= 0
+
+          opts[:timeout] = v
+        end
+        o.on("-s", "--sleep SECS", Float, "Wait in-between requests (default 0)") do |v|
+          raise OptionParser::InvalidArgument, "#{v} (must be >= 0)" if v.negative?
+
+          opts[:sleep] = v
+        end
+      end
+    end
+
+    # Parse argv into a Config, or print the banner and exit non-zero on misuse.
+    def self.parse(argv)
+      opts = DEFAULTS.dup
+      parser = parser(opts)
+      rest = parser.parse(argv)
+
+      raise OptionParser::ParseError, "missing project_name" if rest.empty?
+      raise OptionParser::ParseError, "unexpected arguments: #{rest[1..].join(' ')}" if rest.size > 1
+
+      Config.new(project: rest.first, **opts)
+    rescue OptionParser::ParseError => e
+      warn "Error: #{e.message}\n\n"
+      warn parser
+      exit 1
+    end
+  end
+
+  # SourceForge URL templates (see DOC.md).
+  module Sf
+    BASE = "https://sourceforge.net"
+    DOWNLOADS = "https://downloads.sourceforge.net"
+
+    module_function
+
+    # Percent-encode a single path segment, byte-wise (keeps UTF-8 safe; space -> %20).
+    def encode_segment(name)
+      name.b.gsub(%r{[^A-Za-z0-9\-_.~]}n) { |c| format("%%%02X", c.ord) }
+    end
+
+    def encode_path(path)
+      path.split("/").map { |s| encode_segment(s) }.join("/")
+    end
+
+    # Directory page URL; path is the full path relative to the project root ("" = root).
+    def dir_url(project, path = "")
+      suffix = path.empty? ? "" : "#{encode_path(path)}/"
+      "#{BASE}/projects/#{encode_segment(project)}/files/#{suffix}"
+    end
+
+    # Direct file download URL (redirects to a mirror).
+    def file_url(project, full_path)
+      "#{DOWNLOADS}/project/#{encode_segment(project)}/#{encode_path(full_path)}"
+    end
+  end
+
+  # One tree entry (folder or file).
+  Node = Struct.new(:name, :type, :path, :timestamp, :size, :downloads, :children, keyword_init: true) do
+    def dir? = type == :d
+    def file? = type == :f
+  end
+
+  # HTTP GET/download with timeout, redirect-following and per-request sleep.
+  class Http
+    Error = Class.new(StandardError)
+    MAX_REDIRECTS = 10
+
+    def initialize(timeout: 5, sleep: 0)
+      @timeout = timeout
+      @sleep = sleep
+    end
+
+    # GET url (following redirects), returning the response body.
+    def get(url)
+      with_sleep { with_response(url) { |res| res.body } }
+    end
+
+    # Stream url to dest in chunks (never buffered whole); yields each chunk's bytesize.
+    def download(url, dest)
+      with_sleep do
+        with_response(url) do |res|
+          File.open(dest, "wb") do |f|
+            res.read_body do |chunk|
+              f.write(chunk)
+              yield chunk.bytesize if block_given?
+            end
+          end
+        end
+      end
+    end
+
+    private
+
+    # Sleep once per logical fetch (not per redirect hop).
+    def with_sleep
+      yield
+    ensure
+      sleep(@sleep) if @sleep.positive?
+    end
+
+    def with_response(url, limit = MAX_REDIRECTS, &block)
+      uri = url.is_a?(URI) ? url : URI(url)
+      Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https",
+                      open_timeout: @timeout, read_timeout: @timeout) do |http|
+        http.request(Net::HTTP::Get.new(uri)) do |res|
+          case res
+          when Net::HTTPRedirection
+            raise Error, "too many redirects for #{url}" if limit <= 0
+
+            return with_response(URI.join(uri.to_s, res["location"]), limit - 1, &block)
+          when Net::HTTPSuccess
+            return block.call(res)
+          else
+            raise Error, "HTTP #{res.code} #{res.message} for #{uri}"
+          end
+        end
+      end
+    end
+  end
+
+  # Turns a directory page's HTML into child Nodes, merging table rows with
+  # the net.sf.files JS object.
+  class Parser
+    UNITS = { "b" => 1, "kb" => 1024, "mb" => 1024**2, "gb" => 1024**3, "tb" => 1024**4 }.freeze
+
+    def initialize(project)
+      @project = project
+    end
+
+    # Parse HTML into child Nodes. parent_path is the directory's full path ("" = root).
+    def parse(html, parent_path = "")
+      doc = Nokogiri::HTML(html)
+      meta = extract_metadata(html)
+      doc.css("table#files_list > tbody > tr").filter_map { |row| node_from_row(row, parent_path, meta) }
+    end
+
+    private
+
+    def node_from_row(row, parent_path, meta)
+      name = row["title"].to_s
+      return nil if name.empty?
+
+      type = row["class"].to_s.split.include?("folder") ? :d : :f
+      path = parent_path.empty? ? name : "#{parent_path}/#{name}"
+      info = meta[name] || {}
+
+      Node.new(
+        name: name,
+        type: type,
+        path: path,
+        timestamp: parse_time(row),
+        size: type == :f ? parse_size(cell_text(row, "files_size_h")) : 0,
+        downloads: downloads_for(row, info),
+        children: []
+      )
+    end
+
+    def cell(row, header)
+      row.at_css(%([headers="#{header}"]))
+    end
+
+    def cell_text(row, header)
+      cell(row, header)&.text.to_s.strip
+    end
+
+    def parse_time(row)
+      full = cell(row, "files_date_h")&.at_css("abbr")&.[]("title")
+      full && !full.empty? ? Time.parse(full).utc : nil
+    rescue ArgumentError
+      nil
+    end
+
+    # "1.7 MB" / "602.1 kB" -> bytes (approximate; SF exposes no exact byte count).
+    def parse_size(text)
+      m = text.to_s.match(/([\d.]+)\s*([kmgt]?b|bytes?)/i) or return 0
+
+      unit = m[2].downcase
+      unit = "b" if unit.start_with?("byte")
+      (m[1].to_f * (UNITS[unit] || 1)).round
+    end
+
+    # Prefer the JS object's total downloads; fall back to the HTML weekly count.
+    def downloads_for(row, info)
+      return info["downloads"] if info["downloads"].is_a?(Integer)
+
+      count = cell(row, "files_downloads_h")&.at_css("span.count")&.text
+      count ? count.delete(",").to_i : 0
+    end
+
+    # Extract and JSON-parse the `net.sf.files = { ... }` object literal.
+    def extract_metadata(html)
+      idx = html.index("net.sf.files") or return {}
+      start = html.index("{", idx) or return {}
+      finish = matching_brace(html, start) or return {}
+
+      JSON.parse(html[start..finish])
+    rescue JSON::ParserError
+      {}
+    end
+
+    # Index of the brace matching the one at `start`, honoring string literals.
+    def matching_brace(str, start)
+      depth = 0
+      in_str = false
+      esc = false
+      (start...str.length).each do |i|
+        c = str[i]
+        if in_str
+          if esc then esc = false
+          elsif c == "\\" then esc = true
+          elsif c == '"' then in_str = false
+          end
+        elsif c == '"' then in_str = true
+        elsif c == "{" then depth += 1
+        elsif c == "}"
+          depth -= 1
+          return i if depth.zero?
+        end
+      end
+      nil
+    end
+  end
+
+  # Stage 1: walk the directory tree from the root, building the in-memory Node
+  # tree. Network only — never touches the filesystem. Sequential for now
+  # (concurrency arrives in a later milestone).
+  class Mapper
+    attr_reader :folders, :files, :total_size, :failures
+
+    # log: callback for non-fatal diagnostics (defaults to stderr; the CLI routes
+    # it through StatusBar#log so the bar stays pinned at the bottom).
+    def initialize(project, http, parser, log: ->(m) { warn m })
+      @project = project
+      @http = http
+      @parser = parser
+      @log = log
+      @folders = 0
+      @files = 0
+      @total_size = 0
+      @failures = 0
+    end
+
+    # Build and return the synthetic root Node with the whole tree mapped.
+    # Yields each directory Node right after its page is parsed (progress hook).
+    def map(&on_page)
+      @on_page = on_page
+      root = Node.new(name: @project, type: :d, path: "", timestamp: nil, size: 0, downloads: 0, children: [])
+      walk(root)
+      aggregate(root)
+      # Root timestamp isn't provided by SF; use the newest child's.
+      root.timestamp = root.children.map(&:timestamp).compact.max
+      root
+    end
+
+    private
+
+    def walk(node)
+      begin
+        html = @http.get(Sf.dir_url(@project, node.path))
+        node.children = @parser.parse(html, node.path)
+      rescue Http::Error => e
+        @failures += 1
+        @log.call("WARN: failed to map #{node.path.empty? ? '(root)' : node.path}: #{e.message}")
+        node.children = []
+      end
+      @on_page&.call(node)
+
+      node.children.each do |child|
+        if child.dir?
+          @folders += 1
+          walk(child)
+        else
+          @files += 1
+          @total_size += child.size
+        end
+      end
+    end
+
+    # Post-order pass: recompute folder size/downloads from their leaves.
+    def aggregate(node)
+      return if node.file?
+
+      node.children.each { |c| aggregate(c) }
+      node.size = node.children.sum(&:size)
+      node.downloads = node.children.sum(&:downloads)
+    end
+  end
+
+  # Stage 2: create the local tree and download files. Counters are mutex-guarded
+  # so a ticker thread can snapshot progress while workers advance it. Sequential
+  # for now (concurrency arrives in a later milestone).
+  class Downloader
+    ILLEGAL = /[<>:"\\|?*\x00-\x1f]/.freeze # Windows-illegal chars (per path segment)
+
+    attr_reader :total_files, :total_bytes
+
+    # dest is the project root directory (<output>/<project>).
+    def initialize(project, http, dest, log: ->(m) { warn m })
+      @project = project
+      @http = http
+      @dest = dest
+      @log = log
+      @mutex = Mutex.new
+      @files = []
+      @total_files = 0
+      @total_bytes = 0
+      @files_done = 0
+      @bytes_done = 0
+      @failures = 0
+      @current = ""
+    end
+
+    def files_done = @mutex.synchronize { @files_done }
+    def bytes_done = @mutex.synchronize { @bytes_done }
+    def failures = @mutex.synchronize { @failures }
+    def current = @mutex.synchronize { @current }
+
+    # Create every directory up front and collect the flat file list + totals.
+    def prepare(root)
+      FileUtils.mkdir_p(@dest)
+      each_node(root) do |n|
+        FileUtils.mkdir_p(local_path(n)) if n.dir? && !n.path.empty?
+        @files << n if n.file?
+      end
+      @total_files = @files.size
+      @total_bytes = @files.sum(&:size)
+    end
+
+    def download_all
+      @files.each { |f| download_one(f) }
+    end
+
+    # Apply folder timestamps last, deepest-first — writing files into a directory
+    # bumps its mtime, so folders must be stamped after their contents are final.
+    def apply_folder_times(root)
+      dirs = []
+      each_node(root) { |n| dirs << n if n.dir? }
+      dirs.sort_by { |n| -n.path.count("/") }.each do |n|
+        path = local_path(n)
+        File.utime(n.timestamp, n.timestamp, path) if n.timestamp && File.directory?(path)
+      end
+    end
+
+    private
+
+    def download_one(node)
+      @mutex.synchronize { @current = node.path }
+      dest = local_path(node)
+      @http.download(Sf.file_url(@project, node.path), dest) do |n|
+        @mutex.synchronize { @bytes_done += n }
+      end
+      File.utime(node.timestamp, node.timestamp, dest) if node.timestamp
+      @mutex.synchronize { @files_done += 1 }
+    rescue Http::Error => e
+      @mutex.synchronize { @failures += 1 }
+      @log.call("WARN: failed to download #{node.path}: #{e.message}")
+    end
+
+    def each_node(node, &blk)
+      blk.call(node)
+      node.children.each { |c| each_node(c, &blk) }
+    end
+
+    # Local path for a node; sanitizes each segment for the filesystem.
+    def local_path(node)
+      return @dest if node.path.empty?
+
+      File.join(@dest, *node.path.split("/").map { |s| s.gsub(ILLEGAL, "_") })
+    end
+  end
+
+  # Human-readable formatting for sizes and durations.
+  module Fmt
+    module_function
+
+    UNITS = %w[B KB MB GB TB].freeze
+
+    def size(bytes)
+      v = bytes.to_f
+      i = 0
+      while v >= 1024 && i < UNITS.size - 1
+        v /= 1024
+        i += 1
+      end
+      i.zero? ? "#{bytes} B" : format("%.1f %s", v, UNITS[i])
+    end
+
+    def duration(secs)
+      return format("%.1fs", secs) if secs < 60
+
+      m, s = secs.round.divmod(60)
+      h, m = m.divmod(60)
+      h.zero? ? format("%dm%02ds", m, s) : format("%dh%02dm%02ds", h, m, s)
+    end
+  end
+
+  # Two-line status region pinned to the bottom of the terminal, redrawn in
+  # place with ANSI. Thread-safe: a mutex serializes updates, logs and teardown
+  # so the ticker and log() never interleave. Assumes a VT-capable terminal.
+  class StatusBar
+    def initialize(out: $stdout, width: nil)
+      @out = out
+      @width = width
+      @mutex = Mutex.new
+      @top = ""
+      @bottom = ""
+      @drawn = false
+      @out.sync = true
+    end
+
+    # Set both lines and redraw in place.
+    def update(top, bottom)
+      @mutex.synchronize do
+        @top = top
+        @bottom = bottom
+        erase
+        draw
+      end
+    end
+
+    # Print a message above the bar (scrolls into history), then redraw the bar.
+    def log(msg)
+      @mutex.synchronize do
+        erase
+        @out.puts(msg)
+        draw
+      end
+    end
+
+    # Remove the bar entirely (on completion).
+    def finish
+      @mutex.synchronize { erase }
+    end
+
+    private
+
+    # Park the cursor at the region's top-left and clear from there down.
+    def erase
+      return unless @drawn
+
+      @out.print("\r\e[1A\e[J")
+      @drawn = false
+    end
+
+    def draw
+      @out.print("#{truncate(@top)}\n#{truncate(@bottom)}")
+      @drawn = true
+    end
+
+    # Truncate to terminal width so a line never wraps (wrapping corrupts the
+    # up-count on the next redraw).
+    def truncate(line)
+      w = width
+      line.length > w ? line[0, w] : line
+    end
+
+    def width
+      @width || IO.console&.winsize&.last || 80
+    rescue StandardError
+      80
+    end
+  end
+
+  # Command-line interface: wires the stages together and drives the status bar.
+  module CLI
+    module_function
+
+    # Debug helper: indented tree listing.
+    def dump_tree(node, depth = 0)
+      puts "#{'  ' * depth}#{node.name}#{node.dir? ? '/' : ''}"
+      node.children.each { |c| dump_tree(c, depth + 1) }
+    end
+
+    # Stage-1 analytics line: starts with the stage number, ends with elapsed time.
+    def stage1_line(mapper, elapsed)
+      format("[1] folders: %d | files: %d | size: %s | elapsed: %s",
+             mapper.folders, mapper.files, Fmt.size(mapper.total_size), Fmt.duration(elapsed))
+    end
+
+    # Stage-2 analytics line: elapsed is the global (both-stage) time, complemented
+    # by an estimated total = elapsed + remaining_bytes / speed.
+    def stage2_line(dl, global_elapsed, speed, total_est)
+      format("[2] files: %d/%d | %s/%s | speed: %s/s | elapsed: %s / ~%s",
+             dl.files_done, dl.total_files, Fmt.size(dl.bytes_done), Fmt.size(dl.total_bytes),
+             Fmt.size(speed), Fmt.duration(global_elapsed), Fmt.duration(total_est))
+    end
+
+    def render_stage2(bar, dl, global_start, stage2_start)
+      now = Time.now
+      s2 = now - stage2_start
+      speed = s2.positive? ? dl.bytes_done / s2 : 0
+      remaining = [dl.total_bytes - dl.bytes_done, 0].max
+      eta = speed.positive? ? remaining / speed : 0
+      bar.update("Fetching #{dl.current}", stage2_line(dl, now - global_start, speed, (now - global_start) + eta))
+    end
+
+    # Drive stage 2: prepare the tree, download files while a ~1s ticker refreshes
+    # the bar, apply folder timestamps, then log the stage summary.
+    def run_stage2(downloader, root, bar, global_start)
+      downloader.prepare(root)
+      stage2_start = Time.now
+      ticker = Thread.new do
+        loop do
+          render_stage2(bar, downloader, global_start, stage2_start)
+          sleep 1
+        end
+      end
+
+      downloader.download_all
+      ticker.kill
+      ticker.join
+      render_stage2(bar, downloader, global_start, stage2_start) # final 100% frame
+      downloader.apply_folder_times(root)
+
+      s2 = Time.now - stage2_start
+      avg = s2.positive? ? downloader.bytes_done / s2 : 0
+      summary = format("Stage 2: %d/%d files, %s in %s (avg %s/s)",
+                       downloader.files_done, downloader.total_files,
+                       Fmt.size(downloader.bytes_done), Fmt.duration(s2), Fmt.size(avg))
+      summary += " (#{downloader.failures} failures)" if downloader.failures.positive?
+      bar.log(summary)
+    end
+
+    def main(argv)
+      config = Options.parse(argv)
+      http = Http.new(timeout: config.timeout, sleep: config.sleep)
+      parser = Parser.new(config.project)
+      bar = StatusBar.new
+      mapper = Mapper.new(config.project, http, parser, log: bar.method(:log))
+
+      start = Time.now
+      root = mapper.map do |node|
+        bar.update("Parsing #{node.path}/", stage1_line(mapper, Time.now - start))
+      end
+
+      summary = format("Stage 1: %d folders, %d files, %s total in %s",
+                       mapper.folders, mapper.files, Fmt.size(mapper.total_size),
+                       Fmt.duration(Time.now - start))
+      summary += " (#{mapper.failures} failures)" if mapper.failures.positive?
+      bar.log(summary)
+
+      unless config.no
+        dest = File.join(config.output, config.project)
+        downloader = Downloader.new(config.project, http, dest, log: bar.method(:log))
+        run_stage2(downloader, root, bar, start)
+      end
+
+      bar.finish
+    end
+  end
+end

metadata

@@ -0,0 +1,70 @@
+--- !ruby/object:Gem::Specification
+name: sfdown
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- edelkas
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2026-07-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+description: Clones a SourceForge project's directory tree and downloads its files
+  by scraping the Files pages (SourceForge has no official API). Runs in two stages
+  — map the tree, then download — with a live status bar and optional JSON metadata.
+email:
+- edlucasma@gmail.com
+executables:
+- sfdown
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- DOC.md
+- LICENSE.txt
+- README.md
+- exe/sfdown
+- lib/sfdown.rb
+- lib/sfdown/version.rb
+homepage: https://github.com/edelkas/sfdown
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/edelkas/sfdown
+  source_code_uri: https://github.com/edelkas/sfdown
+  changelog_uri: https://github.com/edelkas/sfdown/blob/master/CHANGELOG.md
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.19
+signing_key:
+specification_version: 4
+summary: SourceForge project downloader.
+test_files: []