durable_huggingface_hub 0.2.0

data/lib/durable_huggingface_hub/utils/http.rb

@@ -0,0 +1,329 @@
+# frozen_string_literal: true
+
+require "json"
+require "faraday"
+require "faraday/retry"
+
+require_relative "../configuration"
+require_relative "headers"
+
+module DurableHuggingfaceHub
+  module Utils
+    # HTTP client for making requests to the HuggingFace Hub API.
+    #
+    # This class provides a configured Faraday client with retry logic,
+    # connection pooling, timeout settings, and proper error handling.
+    #
+    # @example Basic GET request
+    #   client = HttpClient.new
+    #   response = client.get("https://huggingface.co/api/models/bert-base-uncased")
+    #
+    # @example With authentication
+    #   client = HttpClient.new(token: "hf_...")
+    #   response = client.get("/api/whoami")
+    class HttpClient
+      # @return [String, nil] Authentication token
+      attr_reader :token
+
+      # @return [String] Base URL for API requests
+      attr_reader :endpoint
+
+      # @return [Hash] Default headers for all requests
+      attr_reader :default_headers
+
+      # @return [Faraday::Connection] The underlying Faraday connection
+      attr_reader :connection
+
+      # Creates a new HTTP client.
+      #
+      # @param token [String, nil] Authentication token
+      # @param endpoint [String] Base endpoint URL
+      # @param headers [Hash, nil] Additional default headers
+      # @param timeout [Integer, nil] Request timeout in seconds
+      # @param open_timeout [Integer, nil] Connection timeout in seconds
+      # @param proxy [String, nil] Proxy URL
+      # @param logger [Logger, nil] Logger for request/response logging
+      def initialize(
+        token: nil,
+        endpoint: nil,
+        headers: nil,
+        timeout: nil,
+        open_timeout: nil,
+        proxy: nil,
+        logger: nil
+      )
+        @token = token || Configuration.instance.token
+        @endpoint = endpoint || Configuration.instance.endpoint
+        @default_headers = build_default_headers(headers)
+        @logger = logger
+        @connection = build_connection(
+          timeout: timeout,
+          open_timeout: open_timeout,
+          proxy: proxy
+        )
+      end
+
+      # Performs a GET request.
+      #
+      # @param path [String] URL path or full URL
+      # @param params [Hash, nil] Query parameters
+      # @param headers [Hash, nil] Additional headers
+      # @param timeout [Numeric, nil] Request timeout in seconds
+      # @return [Faraday::Response] Response object
+      # @raise [HfHubHTTPError] On HTTP errors
+      #
+      # @example
+      #   response = client.get("/api/models", params: { limit: 10 })
+      def get(path, params: nil, headers: nil, timeout: nil)
+        request(:get, path, params: params, headers: headers, timeout: timeout)
+      end
+
+      # Performs a POST request.
+      #
+      # @param path [String] URL path or full URL
+      # @param body [Hash, String, nil] Request body
+      # @param params [Hash, nil] Query parameters
+      # @param headers [Hash, nil] Additional headers
+      # @param timeout [Numeric, nil] Request timeout in seconds
+      # @return [Faraday::Response] Response object
+      # @raise [HfHubHTTPError] On HTTP errors
+      #
+      # @example
+      #   response = client.post("/api/repos", body: { name: "my-model" })
+      def post(path, body: nil, params: nil, headers: nil, timeout: nil)
+        request(:post, path, body: body, params: params, headers: headers, timeout: timeout)
+      end
+
+      # Performs a PUT request.
+      #
+      # @param path [String] URL path or full URL
+      # @param body [Hash, String, nil] Request body
+      # @param params [Hash, nil] Query parameters
+      # @param headers [Hash, nil] Additional headers
+      # @return [Faraday::Response] Response object
+      # @raise [HfHubHTTPError] On HTTP errors
+      def put(path, body: nil, params: nil, headers: nil)
+        request(:put, path, body: body, params: params, headers: headers)
+      end
+
+      # Performs a DELETE request.
+      #
+      # @param path [String] URL path or full URL
+      # @param params [Hash, nil] Query parameters
+      # @param headers [Hash, nil] Additional headers
+      # @return [Faraday::Response] Response object
+      # @raise [HfHubHTTPError] On HTTP errors
+      def delete(path, params: nil, headers: nil)
+        request(:delete, path, params: params, headers: headers)
+      end
+
+      # Performs a HEAD request.
+      #
+      # @param path [String] URL path or full URL
+      # @param params [Hash, nil] Query parameters
+      # @param headers [Hash, nil] Additional headers
+      # @param timeout [Numeric, nil] Request timeout in seconds
+      # @return [Faraday::Response] Response object
+      # @raise [HfHubHTTPError] On HTTP errors
+      def head(path, params: nil, headers: nil, timeout: nil)
+        request(:head, path, params: params, headers: headers, timeout: timeout)
+      end
+
+      # Performs an HTTP request with error handling.
+      #
+      # @param method [Symbol] HTTP method
+      # @param path [String] URL path
+      # @param body [Hash, String, nil] Request body
+      # @param params [Hash, nil] Query parameters
+      # @param headers [Hash, nil] Additional headers
+      # @param timeout [Numeric, nil] Request timeout in seconds (overrides default)
+      # @yield [req] Optional block for Faraday request configuration
+      # @return [Faraday::Response] Response object
+      # @raise [HfHubHTTPError] On HTTP errors
+      def request(method, path, body: nil, params: nil, headers: nil, timeout: nil, &block)
+        url = build_url(path)
+        merged_headers = @default_headers.merge(headers || {})
+
+        response = @connection.send(method) do |req|
+          req.url(url)
+          req.params.update(params) if params
+          req.headers.update(merged_headers)
+          req.body = prepare_body(body) if body && method != :get && method != :head
+          req.options.timeout = timeout if timeout
+          block&.call(req)
+        end
+
+        handle_response(response)
+      rescue Faraday::Error => e
+        handle_faraday_error(e)
+      end
+
+      private
+
+      # Builds the Faraday connection with middleware.
+      #
+      # @param timeout [Integer, nil] Request timeout
+      # @param open_timeout [Integer, nil] Connection timeout
+      # @param proxy [String, nil] Proxy URL
+      # @return [Faraday::Connection] Configured connection
+      def build_connection(timeout: nil, open_timeout: nil, proxy: nil)
+        Faraday.new(url: @endpoint) do |conn|
+          # Request/response logging (if logger provided)
+          if @logger
+            conn.response :logger, @logger, { headers: true, bodies: false }
+          end
+
+          # Retry middleware with exponential backoff
+          conn.request :retry,
+                       max: 3,
+                       interval: 1,
+                       interval_randomness: 0.5,
+                       backoff_factor: 2,
+                       retry_statuses: [408, 429, 500, 502, 503, 504],
+                       methods: %i[get head options delete],
+                       exceptions: [
+                         Faraday::TimeoutError,
+                         Faraday::ConnectionFailed
+                       ]
+
+          # JSON request/response handling
+          conn.request :json
+          conn.response :json, content_type: /\bjson$/
+
+          # Set timeouts
+          conn.options.timeout = timeout || Configuration.instance.request_timeout
+          conn.options.open_timeout = open_timeout || 10
+
+          # Set proxy if provided
+          conn.proxy = proxy if proxy
+
+          # Use default adapter
+          conn.adapter Faraday.default_adapter
+        end
+      end
+
+      # Builds default headers for requests.
+      #
+      # @param additional_headers [Hash, nil] Additional headers
+      # @return [Hash] Complete headers hash
+      def build_default_headers(additional_headers)
+        Headers.build_hf_headers(
+          token: @token,
+          headers: additional_headers
+        )
+      end
+
+       # Builds the full URL from a path.
+       #
+       # @param path [String] URL path or full URL
+       # @return [String] Full URL
+       def build_url(path)
+         return path if path.start_with?("http://", "https://")
+
+         # Ensure endpoint doesn't end with / and path doesn't start with /
+         endpoint = @endpoint.chomp("/")
+         path = path.start_with?("/") ? path : "/#{path}"
+         "#{endpoint}#{path}"
+       end
+
+       # Prepares request body for transmission.
+       #
+       # @param body [Hash, String] Request body
+       # @return [String] Prepared body
+       def prepare_body(body)
+         return body unless body.is_a?(Hash)
+
+         body.to_json
+       end
+
+      # Handles HTTP response and raises errors for non-success status.
+      #
+      # @param response [Faraday::Response] HTTP response
+      # @return [Faraday::Response] Response object if successful
+      # @raise [HfHubHTTPError] On error status codes
+      def handle_response(response)
+        return response if response.success?
+
+        raise_http_error(response)
+      end
+
+      # Raises appropriate HTTP error based on response.
+      #
+      # @param response [Faraday::Response] HTTP response
+      # @raise [HfHubHTTPError] Appropriate error subclass
+      def raise_http_error(response)
+        status = response.status
+        body = response.body.is_a?(String) ? response.body : (response.body ? response.body.to_json : nil)
+        request_id = Headers.extract_request_id(response.headers)
+
+        case status
+        when 400
+          raise BadRequestError.new("Bad request", response_body: body)
+        when 401
+          raise HfHubHTTPError.new("Unauthorized", status_code: 401, response_body: body, request_id: request_id)
+        when 403
+          # Try to determine if it's gated or disabled
+          if body&.include?("gated")
+            raise GatedRepoError.new("unknown", message: extract_error_message(body, "Access to this repository is gated"))
+          elsif body&.include?("disabled")
+            raise DisabledRepoError.new("unknown", message: extract_error_message(body, "Repository has been disabled"))
+          else
+            raise HfHubHTTPError.new("Forbidden", status_code: 403, response_body: body, request_id: request_id)
+          end
+        when 404
+          raise RepositoryNotFoundError.new("unknown", message: extract_error_message(body, "Repository not found"))
+        when 408
+          raise HfHubHTTPError.new("Request timeout", status_code: 408, response_body: body, request_id: request_id)
+        when 429
+          raise HfHubHTTPError.new("Too many requests", status_code: 429, response_body: body, request_id: request_id)
+        when 500..599
+          raise HfHubHTTPError.new("Server error", status_code: status, response_body: body, request_id: request_id)
+        else
+          raise HfHubHTTPError.new("HTTP error", status_code: status, response_body: body, request_id: request_id)
+        end
+      end
+
+      # Extracts error message from response body.
+      #
+      # @param body [String] Response body
+      # @param default [String] Default message
+      # @return [String] Error message
+      def extract_error_message(body, default)
+        return default unless body
+
+        begin
+          parsed = JSON.parse(body)
+          parsed["error"] || parsed["message"] || default
+        rescue JSON::ParserError
+          default
+        end
+      end
+
+      # Handles Faraday errors.
+      #
+      # @param error [Faraday::Error] Faraday error
+      # @raise [HfHubHTTPError] Converted error
+      def handle_faraday_error(error)
+        case error
+        when Faraday::RetriableResponse
+          # Retry middleware exhausted retries - extract response and handle as HTTP error
+          if error.response && error.response.is_a?(Hash) && error.response[:response]
+            raise_http_error(error.response[:response])
+          elsif error.response && error.response.respond_to?(:status)
+            raise_http_error(error.response)
+          else
+            raise HfHubHTTPError.new("Retryable error: #{error.message}")
+          end
+        when Faraday::TimeoutError
+          raise HfHubHTTPError.new("Request timed out: #{error.message}")
+        when Faraday::ConnectionFailed
+          raise HfHubHTTPError.new("Connection failed: #{error.message}")
+        when Faraday::SSLError
+          raise HfHubHTTPError.new("SSL error: #{error.message}")
+        else
+          raise HfHubHTTPError.new("HTTP error: #{error.message}")
+        end
+      end
+    end
+  end
+end

data/lib/durable_huggingface_hub/utils/paths.rb

@@ -0,0 +1,230 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+module DurableHuggingfaceHub
+  module Utils
+    # Path manipulation and filtering utilities.
+    #
+    # This module provides functions for working with file paths,
+    # including expansion, filtering, and pattern matching.
+    module Paths
+      # Expands a path, resolving home directory and environment variables.
+      #
+      # @param path [String, Pathname] Path to expand
+      # @return [Pathname] Expanded path
+      #
+      # @example
+      #   Paths.expand_path("~/models")  # => Pathname("/home/user/models")
+      #   Paths.expand_path("$HOME/data")  # => Pathname("/home/user/data")
+      def self.expand_path(path)
+        path_str = path.to_s
+
+        # Expand environment variables
+        path_str = path_str.gsub(/\$([A-Z_][A-Z0-9_]*)|\$\{([A-Z_][A-Z0-9_]*)\}/) do
+          key = Regexp.last_match(1) || Regexp.last_match(2)
+          ENV[key] || ""
+        end
+
+        # Expand home directory
+        Pathname.new(path_str).expand_path
+      end
+
+      # Filters a list of repository objects (files) based on allow and ignore patterns.
+      #
+      # This function implements the filtering logic used by HuggingFace Hub for
+      # selecting which files to include in operations like snapshot downloads.
+      #
+      # @param objects [Array<String>, Array<Hash>] List of file paths or file info hashes
+      # @param allow_patterns [Array<String>, String, nil] Patterns to allow (globs or regexes)
+      # @param ignore_patterns [Array<String>, String, nil] Patterns to ignore (globs or regexes)
+      # @param key [String, Symbol, nil] Key to extract path from hash objects
+      # @return [Array] Filtered list of objects
+      #
+      # @example Filter file list with glob patterns
+      #   files = ["config.json", "model.safetensors", "README.md", "data/train.csv"]
+      #   Paths.filter_repo_objects(files, allow_patterns: ["*.json", "*.safetensors"])
+      #   # => ["config.json", "model.safetensors"]
+      #
+      # @example Filter with ignore patterns
+      #   files = ["model.bin", "config.json", "training_log.txt"]
+      #   Paths.filter_repo_objects(files, ignore_patterns: ["*.txt"])
+      #   # => ["model.bin", "config.json"]
+      #
+      # @example Filter hash objects
+      #   files = [{ path: "config.json" }, { path: "model.bin" }]
+      #   Paths.filter_repo_objects(files, allow_patterns: "*.json", key: :path)
+      #   # => [{ path: "config.json" }]
+      def self.filter_repo_objects(objects, allow_patterns: nil, ignore_patterns: nil, key: nil)
+        return objects if objects.nil? || objects.empty?
+
+        # Normalize patterns to arrays
+        allow_patterns = normalize_patterns(allow_patterns)
+        ignore_patterns = normalize_patterns(ignore_patterns)
+
+        # If no patterns, return all objects
+        return objects if allow_patterns.nil? && ignore_patterns.nil?
+
+        objects.select do |obj|
+          path = extract_path(obj, key)
+          next false if path.nil?
+
+          should_include?(path, allow_patterns: allow_patterns, ignore_patterns: ignore_patterns)
+        end
+      end
+
+      # Checks if a path should be included based on allow and ignore patterns.
+      #
+      # @param path [String] File path to check
+      # @param allow_patterns [Array<String>, nil] Patterns to allow
+      # @param ignore_patterns [Array<String>, nil] Patterns to ignore
+      # @return [Boolean] True if path should be included
+      #
+      # @example
+      #   Paths.should_include?("config.json", allow_patterns: ["*.json"])  # => true
+      #   Paths.should_include?("data.txt", allow_patterns: ["*.json"])  # => false
+      #   Paths.should_include?("temp.log", ignore_patterns: ["*.log"])  # => false
+      def self.should_include?(path, allow_patterns: nil, ignore_patterns: nil)
+        # If ignore patterns specified and path matches, exclude it
+        if ignore_patterns && matches_any_pattern?(path, ignore_patterns)
+          return false
+        end
+
+        # If allow patterns specified, path must match at least one
+        if allow_patterns
+          return matches_any_pattern?(path, allow_patterns)
+        end
+
+        # If no allow patterns, include by default (unless already ignored above)
+        true
+      end
+
+      # Checks if a path matches any of the given patterns.
+      #
+      # @param path [String] File path to check
+      # @param patterns [Array<String>] Glob or regex patterns
+      # @return [Boolean] True if path matches any pattern
+      #
+      # @example
+      #   Paths.matches_any_pattern?("config.json", ["*.json", "*.yaml"])  # => true
+      #   Paths.matches_any_pattern?("data.txt", ["*.json", "*.yaml"])  # => false
+      def self.matches_any_pattern?(path, patterns)
+        return false if patterns.nil? || patterns.empty?
+
+        patterns.any? { |pattern| matches_pattern?(path, pattern) }
+      end
+
+      # Checks if a path matches a single pattern.
+      #
+      # Supports both glob patterns and regular expressions.
+      #
+      # @param path [String] File path to check
+      # @param pattern [String, Regexp] Glob pattern or regex
+      # @return [Boolean] True if path matches pattern
+      #
+      # @example Glob patterns
+      #   Paths.matches_pattern?("config.json", "*.json")  # => true
+      #   Paths.matches_pattern?("data/train.csv", "data/*.csv")  # => true
+      #   Paths.matches_pattern?("model.bin", "*.json")  # => false
+      #
+      # @example Regex patterns
+      #   Paths.matches_pattern?("config.json", /\.json$/)  # => true
+      def self.matches_pattern?(path, pattern)
+        case pattern
+        when Regexp
+          !pattern.match(path).nil?
+        when String
+          # Convert glob pattern to regex
+          File.fnmatch?(pattern, path, File::FNM_PATHNAME | File::FNM_EXTGLOB)
+        else
+          false
+        end
+      end
+
+      # Sanitizes a filename by removing or replacing unsafe characters.
+      #
+      # @param filename [String] Filename to sanitize
+      # @return [String] Sanitized filename
+      #
+      # @example
+      #   Paths.sanitize_filename("my file!.txt")  # => "my_file_.txt"
+      #   Paths.sanitize_filename("test/file.json")  # => "test_file.json"
+      def self.sanitize_filename(filename)
+        # Replace path separators with underscores
+        sanitized = filename.gsub(%r{[/\\]}, "_")
+
+        # Replace spaces with underscores
+        sanitized = sanitized.gsub(/\s/, "_")
+
+        # Replace other problematic characters
+        sanitized.gsub(/[<>:"|?*]/, "_")
+      end
+
+      # Joins path components safely, ensuring no path traversal.
+      #
+      # @param base [String, Pathname] Base path
+      # @param *parts [String] Path components to join
+      # @return [Pathname] Joined path
+      # @raise [ValidationError] If result would escape base path
+      #
+      # @example
+      #   Paths.safe_join("/cache", "models", "bert")
+      #   # => Pathname("/cache/models/bert")
+      def self.safe_join(base, *parts)
+        # Validate that no part is an absolute path
+        parts.each do |part|
+          if part.to_s.start_with?("/")
+            raise ValidationError.new(
+              "path",
+              "Path component cannot be absolute: #{part}"
+            )
+          end
+        end
+
+        base_path = Pathname.new(base).expand_path
+        joined_path = parts.reduce(base_path) { |path, part| path.join(part) }
+        final_path = joined_path.expand_path
+
+        # Ensure the final path is within the base path
+        unless final_path.to_s.start_with?(base_path.to_s)
+          raise ValidationError.new(
+            "path",
+            "Path traversal detected: result would escape base directory"
+          )
+        end
+
+        final_path
+      end
+
+      private
+
+      # Normalizes pattern input to an array.
+      #
+      # @param patterns [Array, String, nil] Patterns
+      # @return [Array<String>, nil] Normalized patterns
+      def self.normalize_patterns(patterns)
+        return nil if patterns.nil?
+        return [patterns] if patterns.is_a?(String) || patterns.is_a?(Regexp)
+        return patterns if patterns.is_a?(Array)
+
+        nil
+      end
+
+      # Extracts path from an object (string or hash).
+      #
+      # @param obj [String, Hash] Object
+      # @param key [String, Symbol, nil] Key for hash extraction
+      # @return [String, nil] Extracted path
+      def self.extract_path(obj, key)
+        case obj
+        when String
+          obj
+        when Hash
+          key ? (obj[key] || obj[key.to_s] || obj[key.to_sym]) : nil
+        else
+          nil
+        end
+      end
+    end
+  end
+end