fetch-api 0.6.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4c44fa15b798e01607f5c81e771ad55d24c56abea4aaa1829de0046aa2a761a2
-  data.tar.gz: d8f8c324b7685c23d613ea1f00f5621bc23ef6e682aa66c0c181bb57c035d094
+  metadata.gz: bda47bde351ccf1a6f948922ec3feacc448084da4c00314ed2917cbf5f3e3b90
+  data.tar.gz: b4dcd409c7ed44f98cf5d40c3dca16fc0daa0e4ed255f1b965694cfa42d0d4dd
 SHA512:
-  metadata.gz: 89fc2a039bac8f9dea78f86a6af9d195dff47f1e47fdfbf86026364a916be5dba60961daccd4d24a56f321f95823e24148c9e32df58b50736b7aba03e9ae24cf
-  data.tar.gz: d60ae6beb1b1736cb4a5e71b5049742771a8905565475121f02e5d23b0691905575cb6f1ab55afcc8f7a97f2ed38fee6d169aa6f1f4220a9513ab01c3b97adc3
+  metadata.gz: 3cea9f26869b527d4dc03afdb8a4a6dd57d72c5cae14d42399feef89c5f9ccc56bf9a2afcfea6875b0f3aead7360b7af77b449bba0839b5faa239f0a1073910d
+  data.tar.gz: 6a37f3138b20dd12228a29960959a834e0e2306230c21b8783831231569ae79cdd909a402055bdba920617c091f6660f9a521ebcbcf244ad0716301114e042a2

data/README.md

@@ -50,6 +50,12 @@ Options for `fetch` method:
 - `body`: Request body (default: `nil`)
 - `redirect`: Follow redirects (one of `:follow`, `:error`, `:manual`, default: `:follow`)
 
+When following redirects, `fetch` behaves like the Fetch Standard:
+
+- Gives up with `Fetch::RedirectError` after 20 redirects
+- Changes the method to `GET` and drops the body on 301 / 302 (for `POST`) and 303 (for anything but `GET` / `HEAD`)
+- Removes the `Authorization` header when redirected to a different origin
+
 Methods of `Fetch::Response` object:
 
 - `body`: Response body (String)
@@ -104,13 +110,13 @@ res = fetch('http://example.com', **{
 
 ### Connection pooling
 
-Fetch::API automatically pools and reuses connections to the same origin. Connections are closed after a certain amount of time has passed since the last use, and a new connection is used the next time. Different connections are returned for different threads (in other words, Fetch::API is thread-safe).
+Fetch::API automatically pools and reuses connections to the same origin. Connections are closed after a certain amount of time has passed since the last use, and a new connection is used the next time. The pool is shared by all threads and a connection is never used by more than one thread at a time (in other words, Fetch::API is thread-safe). A forked process does not reuse the connections of its parent. The number of connections is not limited: as many connections as concurrent requests are opened, and they stay in the pool until they expire.
 
 These values can be configured as follows (in seconds):
 
 ``` ruby
 Fetch.configure do |config|
-  config.connection_max_idle_time = 30 # default
+  config.connection_max_idle_time = 30 # default (Float::INFINITY to keep connections forever)
 end
 ```
 
@@ -145,7 +151,7 @@ end
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then push a `v*` tag for the version. GitHub Actions will push the `.gem` file to [rubygems.org](https://rubygems.org).
 
 ## Contributing
 

data/lib/fetch/client.rb

@@ -15,12 +15,70 @@ module Fetch
   class Client
     include Singleton
 
-    def fetch(resource, method:, headers:, body:, redirect:, redirected: false)
-      uri = URI.parse(resource)
+    MAX_REDIRECTS     = 20
+    REDIRECT_STATUSES = [301, 302, 303, 307, 308]
+    REDIRECT_MODES    = %w(follow error manual)
+
+    BODY_HEADERS = %w(
+      content-encoding
+      content-language
+      content-length
+      content-location
+      content-type
+    )
+
+    def initialize
+      @pool = ConnectionPool.new
+    end
+
+    def fetch(resource, method:, headers:, body:, redirect:)
+      raise ArgumentError, "invalid redirect option: #{redirect.inspect}" unless REDIRECT_MODES.include?(redirect.to_s)
+
+      uri     = URI.parse(resource)
+      method  = method.to_s.upcase
+      headers = Headers.new(headers)
+
+      raise ArgumentError, "not an HTTP URL: #{uri}" unless uri.is_a?(URI::HTTP)
+
+      MAX_REDIRECTS.succ.times do |redirect_count|
+        res      = request(uri, method, headers, body)
+        response = to_response(uri, res, redirect_count > 0)
+        location = res['Location']
+
+        return response if !REDIRECT_STATUSES.include?(response.status) || redirect.to_s == 'manual'
+
+        raise RedirectError, location ? "redirected to #{location}" : 'redirected without a location' if redirect.to_s == 'error'
+
+        return response unless location
+
+        redirected_uri = resolve(uri, location)
+
+        if rewrite_to_get?(response.status, method)
+          method = 'GET'
+          body   = nil
+
+          BODY_HEADERS.each do |name|
+            headers.delete name
+          end
+        end
+
+        headers.delete 'authorization' unless redirected_uri.origin == uri.origin
+
+        rewind body
+
+        uri = redirected_uri
+      end
+
+      raise RedirectError, "too many redirects (more than #{MAX_REDIRECTS})"
+    end
+
+    private
+
+    def request(uri, method, headers, body)
       req = Net::HTTP.const_get(method.capitalize).new(uri)
 
       headers.each do |k, v|
-        req[k.to_s] = v.to_s
+        req[k] = v
       end
 
       case body
@@ -41,42 +99,42 @@ module Fetch
         req.body = body
       end
 
-      # @type var uri: URI::HTTP
-      res = pool.with_connection(uri) { _1.request(req) }
-
-      case res
-      when Net::HTTPRedirection
-        case redirect.to_s
-        when 'follow'
-          # @type var location: String
-          location = res['Location']
-
-          fetch(location, method:, headers:, body:, redirect:, redirected: true)
-        when 'error'
-          raise RedirectError, "redirected to #{res['Location']}"
-        when 'manual'
-          to_response(resource, res, redirected)
-        else
-          raise ArgumentError, "invalid redirect option: #{redirect.inspect}"
-        end
+      @pool.with_connection(uri) { _1.request(req) }
+    end
+
+    # Like browsers, percent-encodes the characters that are not allowed in a URL instead of rejecting them.
+    def resolve(uri, location)
+      redirected_uri = URI.join(uri, location.b.gsub(/[^\x21-\x7e]/n) { format('%%%02X', _1.ord) })
+
+      raise RedirectError, "redirected to unsupported URL: #{location}" unless redirected_uri.is_a?(URI::HTTP) && !redirected_uri.host.to_s.empty?
+
+      redirected_uri
+    rescue URI::InvalidURIError
+      raise RedirectError, "redirected to invalid URL: #{location}"
+    end
+
+    def rewrite_to_get?(status, method)
+      case status
+      when 301, 302
+        method == 'POST'
+      when 303
+        !%w(GET HEAD).include?(method)
       else
-        to_response(resource, res, redirected)
+        false
       end
     end
 
-    private
+    def rewind(body)
+      return unless body.is_a?(FormData)
 
-    def pool
-      if pool = Thread.current.thread_variable_get(:fetch_connection_pool)
-        pool
-      else
-        Thread.current.thread_variable_set(:fetch_connection_pool, ConnectionPool.new)
+      body.each do |_, v|
+        v.rewind if v.is_a?(File)
       end
     end
 
-    def to_response(url, res, redirected)
+    def to_response(uri, res, redirected)
       Response.new(
-        url:        url.to_str,
+        url:        uri.to_s,
         status:     res.code.to_i,
         headers:    Headers.new(res.each),
         body:       res.body,

data/lib/fetch/connection_pool.rb

@@ -1,77 +1,129 @@
 require_relative '../fetch'
 
-require 'thread'
-
 module Fetch
   class ConnectionPool
+    SWEEP_INTERVAL = 1
+
     def initialize
       @connections = {}
       @mutex       = Mutex.new
-
-      @sweeper = Thread.new {
-        loop do
-          sweep
-
-          if @connections.empty?
-            Thread.stop
-          else
-            sleep 1
-          end
-        end
-      }
+      @pid         = Process.pid
     end
 
+    # The connection goes back to the pool only if the block completes. Otherwise it may still have a response in
+    # flight, which must not leak into the next request.
     def with_connection(uri, &block)
-      conn = checkout(uri)
+      conn      = checkout(uri)
+      completed = false
 
       begin
-        block.call(conn)
+        result    = block.call(conn)
+        completed = true
+
+        result
       ensure
-        checkin uri, conn
+        completed ? checkin(uri, conn) : close([conn])
       end
     end
 
     private
 
     def checkout(uri)
-      if entry = @mutex.synchronize { @connections.delete(uri.origin) }
-        entry.first
-      else
-        # @type var host: String
-        host = uri.host
+      expired, entry = synchronize { [sweep, @connections[uri.origin]&.pop] }
+
+      close expired
 
-        Net::HTTP.new(host, uri.port).tap {|http|
-          http.use_ssl = uri.scheme == 'https'
+      entry&.first || connect(uri)
+    end
 
-          Fetch.config.on_connection_create.call http, uri
+    def checkin(uri, conn)
+      synchronize do
+        (@connections[uri.origin] ||= []) << [conn, now]
 
-          http.start
-        }
+        start_sweeper unless @sweeper&.alive?
       end
     end
 
-    def checkin(uri, conn)
-      @mutex.synchronize do
-        @connections[uri.origin] = [conn, Time.now]
+    def connect(uri)
+      # @type var host: String
+      host = uri.host
+
+      Net::HTTP.new(host, uri.port).tap {|http|
+        http.use_ssl = uri.scheme == 'https'
+
+        Fetch.config.on_connection_create.call http, uri
+
+        http.start
+      }
+    end
+
+    def close(conns)
+      conns.each do |conn|
+        conn.finish
+      rescue StandardError
+        # The connection is discarded anyway.
       end
+    end
 
-      @sweeper.wakeup
+    def now
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
     end
 
-    def sweep
+    # Connections inherited from the parent process share their sockets with it, so a forked process starts over.
+    def synchronize(&block)
       @mutex.synchronize do
-        @connections.each do |origin, (conn, last_used)|
-          if last_used + Fetch.config.connection_max_idle_time < Time.now
-            begin
-              conn.finish
-            rescue IOError
-              # do nothing
-            end
-
-            @connections.delete origin
+        unless @pid == Process.pid
+          @connections = {}
+          @pid         = Process.pid
+        end
+
+        block.call
+      end
+    end
+
+    # The sweeper retires once the pool is empty. It clears @sweeper under the lock so that a concurrent checkin never
+    # mistakes a retiring sweeper for a running one.
+    def start_sweeper
+      @sweeper = Thread.new {
+        loop do
+          # @type var expired: Array[Net::HTTP]
+          expired = []
+          retired = false
+
+          synchronize do
+            expired  = sweep
+            retired  = @connections.empty?
+            @sweeper = nil if retired
           end
+
+          close expired
+
+          break if retired
+
+          sleep SWEEP_INTERVAL
         end
+      }
+    end
+
+    # Removes the expired connections from the pool and returns them. They are closed outside the lock because closing
+    # may take a while.
+    def sweep
+      max_idle_time = Fetch.config.connection_max_idle_time
+      current       = now
+
+      expired = @connections.flat_map {|_, entries|
+        expired, idle = entries.partition {|_, last_used| current - last_used > max_idle_time }
+
+        entries.replace idle
+
+        expired.map(&:first)
+      }
+
+      @connections.delete_if do |_, entries|
+        entries.empty?
       end
+
+      expired
     end
   end
 end

data/lib/fetch/headers.rb

@@ -35,7 +35,7 @@ module Fetch
     end
 
     def set(key, value)
-      @data[key.to_s.downcase] = [value]
+      @data[key.to_s.downcase] = [value.to_s]
     end
 
     def values

data/lib/fetch/multi_map.rb

@@ -41,8 +41,8 @@ module Fetch
       @entries.map(&:last)
     end
 
-    def each(&)
-      @entries.each(&)
+    def each(&block)
+      block ? @entries.each(&block) : @entries.each
     end
 
     private

data/lib/fetch/response.rb

@@ -1,5 +1,5 @@
 require 'json'
-require 'rack/utils'
+require 'net/http/status'
 
 module Fetch
   class Response
@@ -18,7 +18,7 @@ module Fetch
     end
 
     def status_text
-      Rack::Utils::HTTP_STATUS_CODES[status]
+      Net::HTTP::STATUS_CODES[status]
     end
 
     def json(**json_parse_options)

data/lib/fetch/version.rb

@@ -1,3 +1,3 @@
 module Fetch
-  VERSION = '0.6.0'
+  VERSION = '0.7.0'
 end

data/sig/fetch/api.rbs

@@ -1,5 +1,5 @@
 module Fetch
-  class API
+  module API
     def self?.fetch: (
       string,
       ?method: String | Symbol,

data/sig/fetch/client.rbs

@@ -2,19 +2,32 @@ module Fetch
   class Client
     include Singleton
 
+    MAX_REDIRECTS: Integer
+    REDIRECT_STATUSES: Array[Integer]
+    REDIRECT_MODES: Array[String]
+    BODY_HEADERS: Array[String]
+
+    type body = (String | FormData | URLSearchParams)?
+
+    @pool: ConnectionPool
+
+    def self.instance: () -> Client
+
     def fetch: (
       string,
       method: String | Symbol,
       headers: _Each[[_ToS, _ToS]],
-      body: (String | FormData | URLSearchParams)?,
-      redirect: 'follow' | 'error' | 'manual' | :follow | :error | :manual,
-      ?redirected: bool
+      body: body,
+      redirect: 'follow' | 'error' | 'manual' | :follow | :error | :manual
     ) -> Response
 
     private
 
     def initialize: () -> void
-    def pool: () -> ConnectionPool
-    def to_response: (string, Net::HTTPResponse, bool) -> Response
+    def request: (URI::HTTP, String, Headers, body) -> Net::HTTPResponse
+    def resolve: (URI::HTTP, String) -> URI::HTTP
+    def rewrite_to_get?: (Integer, String) -> bool
+    def rewind: (body) -> void
+    def to_response: (URI::HTTP, Net::HTTPResponse, bool) -> Response
   end
 end

data/sig/fetch/config.rbs

@@ -1,7 +1,7 @@
 module Fetch
   class Config
-    attr_accessor connection_max_idle_time: Integer
+    attr_accessor connection_max_idle_time: Integer | Float
     attr_accessor on_connection_create: ^(Net::HTTP, URI::HTTP) -> void
-    attr_accessor json_parse_options: json_options
+    attr_accessor json_parse_options: JSON::options
   end
 end

data/sig/fetch/connection_pool.rbs

@@ -1,5 +1,12 @@
 module Fetch
   class ConnectionPool
+    SWEEP_INTERVAL: Integer | Float
+
+    @connections: Hash[String, Array[[Net::HTTP, Float]]]
+    @mutex: Thread::Mutex
+    @pid: Integer
+    @sweeper: Thread?
+
     def initialize: () -> void
     def with_connection: [T] (URI::HTTP) { (Net::HTTP) -> T } -> T
 
@@ -7,6 +14,11 @@ module Fetch
 
     def checkout: (URI::HTTP) -> Net::HTTP
     def checkin: (URI::HTTP, Net::HTTP) -> void
-    def sweep: () -> void
+    def connect: (URI::HTTP) -> Net::HTTP
+    def close: (Array[Net::HTTP]) -> void
+    def now: () -> Float
+    def synchronize: [T] () { () -> T } -> T
+    def start_sweeper: () -> void
+    def sweep: () -> Array[Net::HTTP]
   end
 end

data/sig/fetch/headers.rbs

@@ -2,7 +2,9 @@ module Fetch
   class Headers
     include Enumerable[[String, String]]
 
-    def initialize: (_Each[[_ToS, _ToS]]) -> void
+    @data: Hash[String, Array[String]]
+
+    def initialize: (?_Each[[_ToS, _ToS]]) -> void
     def append: (_ToS, _ToS) -> void
     def delete: (_ToS) -> void
     def entries: () -> Array[[String, String]]

data/sig/fetch/multi_map.rbs

@@ -1,7 +1,9 @@
 module Fetch
-  class MultiMap[KIN, KOUT, VIN, VOUT]
+  class MultiMap[KIN, KOUT < Comparable, VIN, VOUT]
     include Enumerable[[KOUT, VOUT]]
 
+    @entries: Array[[KOUT, VOUT]]
+
     def initialize: () -> void
     def append: (KIN, VIN) -> void
     def delete: (KIN) -> void

data/sig/fetch/response.rbs

@@ -8,7 +8,7 @@ module Fetch
 
     def initialize: (url: String, status: Integer, headers: Headers, body: String?, redirected: bool) -> void
     def ok: () -> bool
-    def status_text: () -> String
-    def json: (**json_options) -> untyped
+    def status_text: () -> String?
+    def json: (**JSON::options) -> untyped
   end
 end

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: fetch-api
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Keita Urashima
-autorequire:
-bindir: exe
+bindir: bin
 cert_chain: []
-date: 2024-09-09 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -52,20 +51,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: rack
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: singleton
   requirement: !ruby/object:Gem::Requirement
@@ -94,7 +79,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description:
 email:
 - ursm@ursm.jp
 executables: []
@@ -134,7 +118,6 @@ licenses:
 metadata:
   homepage_uri: https://github.com/ursm/fetch-api
   source_code_uri: https://github.com/ursm/fetch-api.git
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -142,15 +125,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.1.0
+      version: 3.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.16
-signing_key:
+rubygems_version: 4.0.20
 specification_version: 4
 summary: Something like the Fetch API for Ruby
 test_files: []