durable_huggingface_hub 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 06a07c82312c48d5aa31f0a14bdc6b3c59b623dc5a557408ed8c18e83d43ffcf
-  data.tar.gz: 32398bb88e28325b3f5a61356b94513f159579380c13b6fbfd5cdb2cb491dabb
+  metadata.gz: 9693f9faf5186aeabfc3df8e1f2c615ec208cf9b13abc8f6cfcdcd6c0a563241
+  data.tar.gz: 6331e9e4eb9226f8d48ea90e61915d8de1f1beea77d8e78205cda7491127efd7
 SHA512:
-  metadata.gz: 2796f5bca481a6910f558435adb79c989252584cbaaf0a552774f29fe78952e715a8a667b5de420dd7670dcbb348982f8132aaf56f68490d0c05a38fdfae8645
-  data.tar.gz: 4d67ed42b115fc43ec4f5fc9c54d2a52afe5776c16dfe081a3d3eb9e355dec552acf7cb30bfda5590a3ac780477fdafc98d2da888b15d04b5659536ba39421f1
+  metadata.gz: e343052c327d7ddc433ef77087d2e3b0745e35ccdddc46cd06fd33cc7123d7f33d262ef7dd757fde32ce7ae3aacf4e2ea56049be68a0b4c5d5d852b485353f92
+  data.tar.gz: afeb7ed0f145916d722e5057d09325d0727e9e5bcf611bfbe413d7158d351427d02e1f6aec0f7c9728dc4cdb5eec36227855e83257ce9a053b6029db28a2abfa

data/CHANGELOG.md

@@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+### Fixed
+- `hf_hub_download` now follows 3xx redirects issued by HuggingFace Hub
+  resolve URLs by adding the `faraday-follow_redirects` middleware
+  (`Utils::HttpClient#build_connection`).
+- Streaming downloads no longer produce files prefixed with a redirect body.
+  The temp file is now opened once with `"wb"` (truncate) instead of being
+  appended to per-chunk with `"ab"`, which also reduces filesystem overhead.
+
 ## [0.2.0] - 2025-01-24
 
 ### Added

data/CONTRIBUTORS.md

@@ -0,0 +1,8 @@
+# Contributors
+
+Thank you to everyone who has helped improve this project.
+
+## Bug Reports & Research
+
+**Ori Pekelman ([@OriPekelman](https://github.com/OriPekelman))**
+... for Q&A and excellent research.

data/README.md

@@ -1,6 +1,6 @@
 # HuggingFace Hub Ruby
 
-A pure Ruby implementation of the HuggingFace Hub client library. This library provides a complete, production-ready interface to HuggingFace Hub for downloading models, datasets, and managing repositories - with zero Python dependencies.
+A pure Ruby implementation of the HuggingFace Hub client library. This library provides a interface to HuggingFace Hub for downloading models, datasets, and managing repositories - with zero Python dependencies.
 
 ## Features
 

data/huggingface_hub.gemspec

@@ -38,6 +38,7 @@ Gem::Specification.new do |spec|
 
   # Runtime dependencies
   spec.add_dependency "faraday", "~> 2.0"
+  spec.add_dependency "faraday-follow_redirects", "~> 0.3"
   spec.add_dependency "faraday-multipart", "~> 1.0"
   spec.add_dependency "faraday-retry", "~> 2.0"
   spec.add_dependency "dry-struct", "~> 1.6"

data/integration-tests/test_hf_hub_download.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# Integration test reproducing the exact scenario from issue #1:
+#   hf_hub_download fails on every dataset/model file: redirects not followed
+#   and streaming concatenates redirect body.
+#
+# Requires network access to huggingface.co.
+# Run from the repo root:  bash integration-tests/test_hf_hub_download.sh
+
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+
+echo "=== hf_hub_download integration test (issue #1) ==="
+
+ruby -I "$REPO_ROOT/lib" - <<'RUBY'
+require "huggingface_hub"
+
+path = DurableHuggingfaceHub::FileDownload.hf_hub_download(
+  repo_id:   "Trelis/tiny-shakespeare",
+  filename:  "input.txt",
+  repo_type: "dataset",
+)
+
+abort "FAIL: returned path does not exist: #{path}" unless File.exist?(path)
+
+first_line = File.open(path, "rb", &:readline).chomp
+if first_line.start_with?("Temporary Redirect")
+  abort "FAIL: file is corrupted — starts with redirect body:\n  #{first_line}"
+end
+
+puts "PASS: downloaded to #{path}"
+puts "      first line: #{first_line[0, 80]}"
+RUBY

data/lib/durable_huggingface_hub/authentication.rb

@@ -54,15 +54,8 @@ module DurableHuggingfaceHub
        # Update configuration
        Configuration.instance.token = token
 
-        # Return masked token and user info
-        masked = Utils::Auth.mask_token(token)
-        username = user_info.name || "unknown"
-
-       puts "Login successful!"
-       puts "Token: #{masked}"
-       puts "User: #{username}"
-
-       masked
+      # Return user info for CLI
+       { name: user_info.name, type: user_info.type || "user" }
      end
 
      # Logs out of HuggingFace Hub by removing the stored token.
@@ -106,7 +99,7 @@ module DurableHuggingfaceHub
        token = Utils::Auth.get_token!(token: token)
 
        client = Utils::HttpClient.new(token: token)
-       response = client.get("/api/whoami")
+       response = client.get("/api/whoami-v2")
 
        Types::User.from_hash(response.body)
      end

data/lib/durable_huggingface_hub/cli.rb

@@ -0,0 +1,247 @@
+# frozen_string_literal: true
+
+require "optparse"
+require "json"
+
+module DurableHuggingfaceHub
+  # Command-line interface for the Hugging Face Hub
+  class CLI
+    def initialize(args)
+      @args = args
+      @command = nil
+      @options = {}
+    end
+
+    def run
+      parse_args
+      execute_command
+    rescue StandardError => e
+      abort "Error: #{e.message}"
+    end
+
+    private
+
+    def parse_args
+      if @args.empty?
+        print_usage
+        exit 0
+      end
+
+      @command = @args.shift
+
+      OptionParser.new do |opts|
+        case @command
+        when "login"
+          parse_login_options(opts)
+        when "logout"
+          parse_logout_options(opts)
+        when "whoami"
+          parse_whoami_options(opts)
+        when "download"
+          parse_download_options(opts)
+        when "info"
+          parse_info_options(opts)
+        when "list"
+          parse_list_options(opts)
+        when "scan-cache"
+          parse_scan_cache_options(opts)
+        when "help", "--help", "-h"
+          print_usage
+          exit 0
+        else
+          puts "Unknown command: #{@command}"
+          print_usage
+          exit 1
+        end
+      end.parse!(@args)
+    end
+
+    def parse_login_options(opts)
+      opts.banner = "Usage: dhf login [options]"
+      opts.on("-t", "--token TOKEN", "HuggingFace API token") { |v| @options[:token] = v }
+      opts.on("--add-to-git-credential", "Add token to git credential store") { @options[:add_to_git_credential] = true }
+    end
+
+    def parse_logout_options(opts)
+      opts.banner = "Usage: dhf logout"
+    end
+
+    def parse_whoami_options(opts)
+      opts.banner = "Usage: dhf whoami [options]"
+      opts.on("-t", "--token TOKEN", "HuggingFace API token") { |v| @options[:token] = v }
+    end
+
+    def parse_download_options(opts)
+      opts.banner = "Usage: dhf download REPO_ID [options]"
+      opts.on("-f", "--filename FILENAME", "Specific file to download") { |v| @options[:filename] = v }
+      opts.on("-r", "--revision REVISION", "Git revision (branch, tag, or commit)") { |v| @options[:revision] = v }
+      opts.on("--repo-type TYPE", "Repository type (model, dataset, space)") { |v| @options[:repo_type] = v }
+      opts.on("--cache-dir DIR", "Path to cache directory") { |v| @options[:cache_dir] = v }
+      opts.on("--snapshot", "Download entire repository snapshot") { @options[:snapshot] = true }
+    end
+
+    def parse_info_options(opts)
+      opts.banner = "Usage: dhf info REPO_ID [options]"
+      opts.on("--repo-type TYPE", "Repository type (model, dataset, space)") { |v| @options[:repo_type] = v }
+      opts.on("-r", "--revision REVISION", "Git revision (branch, tag, or commit)") { |v| @options[:revision] = v }
+    end
+
+    def parse_list_options(opts)
+      opts.banner = "Usage: dhf list TYPE [options]"
+      opts.on("--author AUTHOR", "Filter by author") { |v| @options[:author] = v }
+      opts.on("--search QUERY", "Search query") { |v| @options[:search] = v }
+      opts.on("--limit N", Integer, "Limit number of results") { |v| @options[:limit] = v }
+      opts.on("--full", "Return full information") { @options[:full] = true }
+    end
+
+    def parse_scan_cache_options(opts)
+      opts.banner = "Usage: dhf scan-cache [options]"
+      opts.on("--cache-dir DIR", "Path to cache directory") { |v| @options[:cache_dir] = v }
+    end
+
+    def execute_command
+      case @command
+      when "login"
+        cmd_login
+      when "logout"
+        cmd_logout
+      when "whoami"
+        cmd_whoami
+      when "download"
+        cmd_download
+      when "info"
+        cmd_info
+      when "list"
+        cmd_list
+      when "scan-cache"
+        cmd_scan_cache
+      end
+    end
+
+    def cmd_login
+      result = DurableHuggingfaceHub.login(
+        token: @options[:token],
+        add_to_git_credential: @options[:add_to_git_credential] || false
+      )
+      puts "Successfully logged in as: #{result[:name]} (#{result[:type]})"
+    end
+
+    def cmd_logout
+      DurableHuggingfaceHub.logout
+      puts "Successfully logged out"
+    end
+
+    def cmd_whoami
+      result = DurableHuggingfaceHub.whoami(token: @options[:token])
+      puts JSON.pretty_generate(result)
+    end
+
+    def cmd_download
+      repo_id = @args.shift
+      abort "Error: REPO_ID required" unless repo_id
+
+      if @options[:snapshot]
+        path = DurableHuggingfaceHub.snapshot_download(
+          repo_id: repo_id,
+          revision: @options[:revision],
+          repo_type: @options[:repo_type] || "model",
+          cache_dir: @options[:cache_dir]
+        )
+      else
+        filename = @options[:filename]
+        abort "Error: --filename required for single file download" unless filename
+
+        path = DurableHuggingfaceHub.hf_hub_download(
+          repo_id: repo_id,
+          filename: filename,
+          revision: @options[:revision],
+          repo_type: @options[:repo_type] || "model",
+          cache_dir: @options[:cache_dir]
+        )
+      end
+
+      puts "Downloaded to: #{path}"
+    end
+
+    def cmd_info
+      repo_id = @args.shift
+      abort "Error: REPO_ID required" unless repo_id
+
+      repo_type = @options[:repo_type] || "model"
+      info = DurableHuggingfaceHub.repo_info(
+        repo_id,
+        repo_type: repo_type,
+        revision: @options[:revision]
+      )
+
+      puts JSON.pretty_generate(info.to_h)
+    end
+
+    def cmd_list
+      type = @args.shift
+      abort "Error: TYPE required (models, datasets, or spaces)" unless type
+
+      results = case type
+                when "models"
+                  DurableHuggingfaceHub.list_models(
+                    author: @options[:author],
+                    search: @options[:search],
+                    limit: @options[:limit],
+                    full: @options[:full] || false
+                  )
+                when "datasets"
+                  DurableHuggingfaceHub.list_datasets(
+                    author: @options[:author],
+                    search: @options[:search],
+                    limit: @options[:limit],
+                    full: @options[:full] || false
+                  )
+                when "spaces"
+                  DurableHuggingfaceHub.list_spaces(
+                    author: @options[:author],
+                    search: @options[:search],
+                    limit: @options[:limit],
+                    full: @options[:full] || false
+                  )
+                else
+                  abort "Error: Unknown type '#{type}'. Must be models, datasets, or spaces"
+                end
+
+      results.each do |item|
+        puts JSON.pretty_generate(item.to_h)
+      end
+    end
+
+    def cmd_scan_cache
+      info = DurableHuggingfaceHub.scan_cache_dir(cache_dir: @options[:cache_dir])
+      puts JSON.pretty_generate(info.to_h)
+    end
+
+    def print_usage
+      puts <<~USAGE
+        Usage: dhf COMMAND [options]
+
+        Commands:
+          login                 Login to HuggingFace Hub
+          logout                Logout from HuggingFace Hub
+          whoami                Display current user information
+          download REPO_ID      Download a file or repository
+          info REPO_ID          Get information about a repository
+          list TYPE             List models, datasets, or spaces
+          scan-cache            Scan the local cache directory
+          help                  Show this help message
+
+        Examples:
+          dhf login --token hf_...
+          dhf whoami
+          dhf download bert-base-uncased --filename config.json
+          dhf download bert-base-uncased --snapshot
+          dhf info openai/whisper-large-v3
+          dhf list models --author openai --limit 10
+          dhf scan-cache
+
+        Use 'dhf COMMAND --help' for more information on a specific command.
+      USAGE
+    end
+  end
+end

data/lib/durable_huggingface_hub/file_download.rb

@@ -463,8 +463,11 @@ module DurableHuggingfaceHub
          end
        end
 
+      # Use the redirect-resolved URL so the streaming GET never sees a 3xx.
+      download_url = metadata[:resolved_url] || url_path
+
       # Download the file to blob storage
-      download_to_blob(client, url_path, blob_path, metadata, progress)
+      download_to_blob(client, download_url, blob_path, metadata, progress)
 
       # Create snapshot symlink
       ensure_snapshot_link(blob_path, snapshot_path)
@@ -483,12 +486,16 @@ module DurableHuggingfaceHub
     def self.get_file_metadata(client, url_path)
       response = client.head(url_path)
 
-      # Extract metadata from headers (response is now a Faraday::Response object)
       headers = response.headers
+      # After following redirects, env[:url] holds the final resolved URL.
+      # We store it so the subsequent streaming GET can target it directly,
+      # bypassing the redirect entirely (on_data fires below middleware).
+      resolved_url = response.env[:url].to_s
       {
         etag: extract_etag(headers["etag"] || headers["x-linked-etag"]),
         size: headers["x-linked-size"]&.to_i,
-        commit_hash: headers["x-repo-commit"]
+        commit_hash: headers["x-repo-commit"],
+        resolved_url: resolved_url
       }
     end
 
@@ -543,11 +550,12 @@ module DurableHuggingfaceHub
       end
 
       begin
-        # Download file
-        response = client.request(:get, url_path) do |req|
-          req.options.on_data = proc do |chunk, _overall_received_bytes, _env|
-            File.open(temp_path, "ab") { |f| f.write(chunk) }
-            progress_tracker.update(chunk.bytesize)
+        File.open(temp_path, "wb") do |f|
+          client.request(:get, url_path) do |req|
+            req.options.on_data = proc do |chunk, _overall_received_bytes, _env|
+              f.write(chunk)
+              progress_tracker.update(chunk.bytesize)
+            end
           end
         end
 

data/lib/durable_huggingface_hub/hf_api.rb

@@ -629,12 +629,24 @@ module DurableHuggingfaceHub
       DurableHuggingfaceHub::Utils::Validators.validate_repo_id(repo_id)
       repo_type = DurableHuggingfaceHub::Utils::Validators.validate_repo_type(repo_type)
 
-      path = "/api/#{repo_type}s"
+      # Extract organization from repo_id if not provided
+      repo_parts = repo_id.split("/")
+      if organization.nil? && repo_parts.length == 2
+        # Get current username to check if it's a personal repo
+        current_user = whoami
+        potential_org = repo_parts[0]
+        # Only set organization if it's not the current user
+        organization = potential_org unless potential_org == current_user.name
+      end
+      repo_name = repo_parts.last
+
+      path = "/api/repos/create"
       payload = {
-        name: repo_id.split("/").last,
-        private: private,
-        organization: organization
+        name: repo_name,
+        private: private
       }
+      payload[:type] = repo_type if repo_type != "model"
+      payload[:organization] = organization if organization
 
       response = http_client.post(path, body: payload, timeout: timeout)
       body = response.body.is_a?(String) ? JSON.parse(response.body) : response.body

data/lib/durable_huggingface_hub/types/dataset_info.rb

@@ -96,6 +96,24 @@ module DurableHuggingfaceHub
        #   @return [Integer, nil] Trending score
        attribute :trending_score, Types::OptionalInteger.default(nil)
 
+       # Transform API response to filter out unknown keys
+       def self.from_hash(data)
+         transformed = data.dup
+
+         # Filter out unknown keys to avoid dry-struct errors
+         known_keys = [:id, :sha, :last_modified, :tags, :siblings, :private, :gated,
+                       :disabled, :downloads, :likes, :author, :created_at, :card_data,
+                       :description, :citation, :downloads_all_time, :paperswithcode_id,
+                       :trending_score,
+                       "id", "sha", "last_modified", "tags", "siblings", "private", "gated",
+                       "disabled", "downloads", "likes", "author", "created_at", "card_data",
+                       "description", "citation", "downloads_all_time", "paperswithcode_id",
+                       "trending_score"]
+         transformed = transformed.select { |k, _| known_keys.include?(k) }
+
+         new(transformed)
+       end
+
        # Returns the list of file names in the repository.
       #
       # @return [Array<String>] File names

data/lib/durable_huggingface_hub/types/user.rb

@@ -25,6 +25,10 @@ module DurableHuggingfaceHub
       #   @return [String, nil] User type (e.g., "user")
       attribute :type, Types::OptionalString.default(nil)
 
+      # @!attribute [r] id
+      #   @return [String, nil] User ID
+      attribute :id, Types::OptionalString.default(nil)
+
       # @!attribute [r] name
       #   @return [String] Username
       attribute :name, Types::String
@@ -49,6 +53,10 @@ module DurableHuggingfaceHub
       #   @return [Array<Hash>, nil] Organizations the user belongs to
       attribute :orgs, Types::Array.of(Types::Hash).optional.default(nil)
 
+      # @!attribute [r] auth
+      #   @return [Hash, nil] Authentication information including token details
+      attribute :auth, Types::Hash.optional.default(nil)
+
       # Transform isPro from API to is_pro
       def self.from_hash(data)
         transformed = data.dup
@@ -64,6 +72,11 @@ module DurableHuggingfaceHub
           transformed[:avatar_url] = transformed.delete(:avatarUrl)
         end
 
+        # Filter out unknown keys to avoid dry-struct errors
+        known_keys = [:type, :name, :fullname, :email, :avatar_url, :is_pro, :orgs,
+                      "type", "name", "fullname", "email", "avatar_url", "is_pro", "orgs"]
+        transformed = transformed.select { |k, _| known_keys.include?(k) }
+
         new(transformed)
       end
 

data/lib/durable_huggingface_hub/utils/http.rb

@@ -2,6 +2,7 @@
 
 require "json"
 require "faraday"
+require "faraday/follow_redirects"
 require "faraday/retry"
 
 require_relative "../configuration"
@@ -173,6 +174,9 @@ module DurableHuggingfaceHub
             conn.response :logger, @logger, { headers: true, bodies: false }
           end
 
+          # Follow 3xx redirects (HuggingFace Hub issues 307s for file resolve URLs)
+          conn.response :follow_redirects, limit: 5
+
           # Retry middleware with exponential backoff
           conn.request :retry,
                        max: 3,

data/lib/durable_huggingface_hub/version.rb

@@ -4,5 +4,5 @@ require "rubygems/version"
 
 module DurableHuggingfaceHub
   # Current version of the HuggingFace Hub Ruby client library
-  VERSION = "0.2.0"
+  VERSION = "0.2.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: durable_huggingface_hub
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - David Berube
@@ -23,6 +23,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: faraday-follow_redirects
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
 - !ruby/object:Gem::Dependency
   name: faraday-multipart
   requirement: !ruby/object:Gem::Requirement
@@ -274,14 +288,17 @@ files:
 - ".editorconfig"
 - ".rubocop.yml"
 - CHANGELOG.md
+- CONTRIBUTORS.md
 - README.md
 - Rakefile
 - devenv.lock
 - devenv.nix
 - devenv.yaml
 - huggingface_hub.gemspec
+- integration-tests/test_hf_hub_download.sh
 - lib/durable_huggingface_hub/authentication.rb
 - lib/durable_huggingface_hub/cache.rb
+- lib/durable_huggingface_hub/cli.rb
 - lib/durable_huggingface_hub/configuration.rb
 - lib/durable_huggingface_hub/constants.rb
 - lib/durable_huggingface_hub/errors.rb
@@ -328,7 +345,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.9
+rubygems_version: 3.7.2
 specification_version: 4
 summary: Pure Ruby client for HuggingFace Hub
 test_files: []