files.com 1.1.813 → 1.1.815

data/lib/files.com/api_client.rb

@@ -76,6 +76,7 @@ module Files
     end
 
     def self.build_default_conn(force_net_http: false)
+      proxy = Files.proxy
       conn = Faraday.new do |builder|
         if Gem::Version.new(Faraday::VERSION) < Gem::Version.new("2.0.0") && defined?(Faraday::Request::Multipart)
           begin
@@ -98,12 +99,120 @@ module Files
         end
       end
 
-      conn.proxy = Files.proxy if Files.proxy
+      conn.proxy = proxy if proxy
       conn.ssl.verify = true
+      # How it was made, and with which proxy, so unchanged_default_conn? can tell whether anything about it has changed
+      # since.
+      conn.instance_variable_set(:@files_sdk_configuration, [ connection_configuration(conn), proxy_identity(proxy) ].freeze)
 
       conn
     end
 
+    # Whether conn is a connection build_default_conn made with proxy, as proxy_identity gives it (nil when it was given
+    # none, so it follows the proxy environment variables), with nothing about it changed since: the same middleware and
+    # adapter (the same objects, so neither another adapter nor one with a configuration block of its own), and the same
+    # TLS, request and proxy options, default headers and default parameters. An adaptive transfer sends over its own
+    # connections, made with its own proxy, in place of only such a connection with that same proxy; through any other,
+    # such as one with a minimum TLS version or another proxy, it sends as that connection does.
+    def self.unchanged_default_conn?(conn, proxy)
+      return false unless conn.is_a?(Faraday::Connection)
+
+      conn.instance_variable_get(:@files_sdk_configuration) == [ connection_configuration(conn), proxy ]
+    end
+
+    # A Files.proxy value, a URL or a Hash of its uri, user and password, as one frozen [URL, user, password], its user
+    # and password included, or nil for none.
+    def self.proxy_identity(proxy)
+      options = proxy && Faraday::ProxyOptions.from(proxy)
+      options&.uri && [ options.uri.to_s, options.user, options.password ].freeze
+    end
+
+    private_class_method def self.connection_configuration(conn)
+      [ conn.builder.handlers.map(&:object_id), conn.builder.adapter&.object_id, conn.ssl.to_hash, conn.options.to_hash,
+        conn.headers.to_h { |name, value| [ name.dup, value.dup ] }, conn.params.to_h { |name, value| [ name.dup, value.dup ] }, conn.proxy&.to_hash ].freeze
+    end
+
+    # Runs the block with each transfer request this thread sends through remote_request sent once: a failure is
+    # logged as usual but neither retried nor converted, so Faraday's own error, with any response it got, reaches
+    # the caller. An adaptive transfer sends its content requests through a caller's client this way, since it
+    # schedules and counts every retry itself.
+    def self.single_attempt
+      previous = Thread.current[:files_api_client_single_attempt]
+      Thread.current[:files_api_client_single_attempt] = true
+      yield
+    ensure
+      Thread.current[:files_api_client_single_attempt] = previous
+    end
+
+    # What an adaptive transfer's API requests use in place of the library-wide settings, fixed on the caller's thread
+    # as the transfer begins (transfer_settings) and passed as their request_settings option.
+    TransferSettings = Struct.new(:default_headers, :language, :app_info, :open_timeout, :read_timeout, :region_policy, :region_handler, keyword_init: true)
+
+    # The library-wide settings as they are now: frozen copies of Files.default_headers (a Proc is kept as it is, and
+    # still called for each request), Files.language and Files.app_info, the timeouts, Files.region_policy, and
+    # Files.region_handler, kept as it is. A request given them uses them instead of the library-wide ones, and never
+    # falls back to Files.api_key or Files.session_id: a credential it was not given stays absent.
+    def self.transfer_settings
+      TransferSettings.new(default_headers: settings_copy(Files.default_headers), language: settings_copy(Files.language), app_info: settings_copy(Files.app_info),
+                           open_timeout: Files.open_timeout, read_timeout: Files.read_timeout, region_policy: Files.region_policy,
+                           region_handler: Files.region_handler
+      ).freeze
+    end
+
+    # A frozen copy of value's Hashes, Arrays and Strings, all the way down; anything else, such as a Proc, as it is.
+    private_class_method def self.settings_copy(value)
+      case value
+      when Hash then value.to_h { |key, item| [ settings_copy(key), settings_copy(item) ] }.freeze
+      when Array then value.map { |item| settings_copy(item) }.freeze
+      when String then value.dup.freeze
+      else value
+      end
+    end
+
+    # The comment the SDK's own User-Agent ends with, and the most bytes that comment and the whole User-Agent may take.
+    USER_AGENT_METADATA_MARKER = "files-sdk-meta/1"
+    MAX_USER_AGENT_BYTES = 190
+    MAX_USER_AGENT_METADATA_BYTES = 128
+    private_constant :USER_AGENT_METADATA_MARKER, :MAX_USER_AGENT_BYTES, :MAX_USER_AGENT_METADATA_BYTES
+
+    # The User-Agent the SDK sends with its own API requests: identity, the SDK's name and version and any app info,
+    # followed by one comment naming the coarse family of the operating system host_os (RbConfig's) names and the Ruby
+    # runtime, with ruby_version for CRuby, such as "Files.com Ruby SDK v1.2.3 (files-sdk-meta/1; os=linux;
+    # runtime=ruby/3.4.4)". The comment carries nothing else: no host or user name, path, architecture or build. A
+    # version it cannot carry as given is left out rather than changed. When the whole User-Agent would be longer than
+    # 190 bytes, the version is left out first and then the whole comment, so identity is always sent as it is; an
+    # identity that is not printable ASCII, such as one whose app info is not, gets no comment. A User-Agent from
+    # Files.default_headers or a request's own headers is sent instead, exactly as given, and a transfer request
+    # (remote_request) sends only the headers it is given.
+    def self.user_agent_with_metadata(identity, host_os: RbConfig::CONFIG["host_os"], ruby_engine: RUBY_ENGINE, ruby_version: RUBY_VERSION)
+      return identity unless identity.is_a?(String) && identity.ascii_only? && identity.match?(/\A[\x20-\x7e]+\z/)
+
+      carried = ruby_engine == "ruby" && ruby_version.is_a?(String) && ruby_version.ascii_only? && ruby_version.match?(/\A[0-9][A-Za-z0-9.+_-]{0,47}\z/)
+      runtimes = carried ? [ "ruby/#{ruby_version}", "ruby" ] : [ "ruby" ]
+      os = user_agent_os(host_os)
+      runtimes.each do |runtime|
+        metadata = " (#{USER_AGENT_METADATA_MARKER}; os=#{os}; runtime=#{runtime})"
+        return "#{identity}#{metadata}" if metadata.bytesize <= MAX_USER_AGENT_METADATA_BYTES && identity.bytesize + metadata.bytesize <= MAX_USER_AGENT_BYTES
+      end
+      identity
+    end
+
+    # The coarse family of the operating system RbConfig's host_os names: "unknown" for none, and "other" for one the
+    # User-Agent's comment does not name.
+    private_class_method def self.user_agent_os(host_os)
+      return "unknown" unless host_os.is_a?(String) && !host_os.empty?
+      return "other" unless host_os.ascii_only?
+
+      case host_os
+      when /mswin|mingw|cygwin/i then "windows"
+      when /darwin/i then "macos"
+      when /android/i then "android"
+      when /linux/i then "linux"
+      when /freebsd/i then "freebsd"
+      else "other"
+      end
+    end
+
     def self.should_retry?(error, num_retries)
       return false if num_retries >= Files.max_network_retries
       return true if error.is_a?(Faraday::TimeoutError)
@@ -144,13 +253,15 @@ module Files
     end
 
     # Returns the response and the API key and session ID the request used. +region_http_pairs+, such as the pairs a
-    # returned record keeps, is used instead of this client's. The block, if given, receives the request's context once
-    # its response has arrived: the regional origin a lockout region mismatch moved it to, or nil, and the frozen
-    # region_http_pairs it used.
-    def execute_request(method, path, base_url: nil, api_key: nil, session_id: nil, workspace_id: nil, region_http_pairs: nil, headers: {}, params: {}, &on_request_context)
+    # returned record keeps, is used instead of this client's. +settings+, an adaptive transfer's TransferSettings, is
+    # used instead of the library-wide default headers, language, app info, timeouts and region policy and handler, and
+    # with it a credential the request was not given is not looked up in Files.api_key or Files.session_id. The block,
+    # if given, receives the request's context once its response has arrived: the regional origin a lockout region
+    # mismatch moved it to, or nil, and the frozen region_http_pairs it used.
+    def execute_request(method, path, base_url: nil, api_key: nil, session_id: nil, workspace_id: nil, region_http_pairs: nil, settings: nil, headers: {}, params: {}, &on_request_context)
       base_url ||= Files.base_url
       if api_key.nil?
-        session_id ||= Files.session_id
+        session_id ||= Files.session_id unless settings
       else
         session_id = nil
       end
@@ -159,7 +270,7 @@ module Files
       if session_id and session_id != ""
         check_session_id!(session_id)
       elsif path !~ /^\/sessions/ # TODO: automate this to refer to any unauthenticated endpoint
-        api_key ||= Files.api_key
+        api_key ||= Files.api_key unless settings
         check_api_key!(api_key)
       end
 
@@ -172,7 +283,7 @@ module Files
         body = params
       end
 
-      headers = request_headers(api_key, session_id, method, workspace_id).update(headers)
+      headers = request_headers(api_key, session_id, method, workspace_id, settings).update(headers)
       url = api_url(path, base_url)
 
       context = RequestLogContext.new
@@ -188,15 +299,15 @@ module Files
       # Fixed before the request starts, so later changes to the caller's or the library-wide pairs cannot affect it.
       http_pairs = region_http_pairs.nil? ? self.region_http_pairs : Util.checked_region_http_pairs(region_http_pairs)
       relocate = lambda do |error_response|
-        regional_base_url = region_continuation_base_url(error_response, base_url, body, http_pairs)
+        regional_base_url = region_continuation_base_url(error_response, base_url, body, http_pairs, settings)
         url = api_url(path, regional_base_url) if regional_base_url
         regional_base_url
       end
 
       http_resp = execute_request_with_rescues(base_url, context, relocate: relocate) do
         conn.run_request(method, url, body, headers) do |req|
-          req.options.open_timeout = Files.open_timeout
-          req.options.timeout = Files.read_timeout
+          req.options.open_timeout = setting(settings, :open_timeout)
+          req.options.timeout = setting(settings, :read_timeout)
           req.params = query_params unless query_params.nil?
         end
       end
@@ -212,26 +323,59 @@ module Files
       [ resp, api_key, session_id ]
     end
 
+    # Whether this client sends transfer requests just as the SDK's own default client does: an ApiClient itself, not
+    # a subclass or an object with methods of its own, with no logger or log level of its own. Only then may an
+    # adaptive transfer send its content requests over its own connections instead of through remote_request.
+    def default_transfer_behavior?
+      instance_of?(ApiClient) && singleton_methods.empty? && @logger.equal?(USE_GLOBAL) && @log_level.equal?(USE_GLOBAL)
+    end
+
+    # The workspace_id option that names, for every later request, the workspace this client's requests name now when
+    # their own workspace_id is nil: its own, or Files.workspace_id, and "" for none. An adaptive upload fixes it as it
+    # begins, so a later change to Files.workspace_id cannot move it.
+    def resolved_workspace_id
+      workspace_id = default_workspace_id
+      workspace_id.nil? ? "" : workspace_id
+    end
+
+    # Sends a request to a transfer URL, such as a signed storage URL, through this client's connection, with its
+    # middleware, adapter, settings and logging. The URL's query reaches the adapter as URI parses it (SignedQuery): the
+    # request's params, the connection's default ones included, are not added to it, unlike an API request's. URI keeps
+    # its order, its percent escapes and the characters a signed URL's query uses, so such a query arrives exactly as
+    # given, as its signature needs; it escapes others, such as a space, and drops tabs and line breaks. A URL URI cannot
+    # parse, as one with %zz in its query or a character outside ASCII, is left to Faraday as it is, which refuses it
+    # or sends its query decoded and encoded again, as it does any params.
     def remote_request(method, url, headers = {}, body = nil)
       context = RequestLogContext.new
       context.method       = method
       context.path         = "[transfer]"
       Util.log_debug("Transfer request", method: method, url: url)
+      query = transfer_query(url)
 
       execute_request_with_rescues(Files.base_url, context, skip_body_logging: true, is_transfer: true) do
         conn.run_request(method, url, body, headers) do |req|
           req.options.open_timeout = Files.open_timeout
           req.options.timeout = Files.read_timeout
+          req.options.params_encoder = SignedQuery.new(query, req.options.params_encoder || Faraday::Utils.default_params_encoder) if query
           yield(req) if block_given?
         end
       end
     end
 
+    # The query a request to a transfer URL carries to the adapter (SignedQuery): the one URI parses from the URL, ""
+    # for none, or nil when URI cannot parse it.
+    private def transfer_query(url)
+      ::URI.parse(url.to_s).query || ""
+    rescue ::URI::InvalidURIError
+      nil
+    end
+
     def stream_download(uri, io, range)
       if conn.adapter == Faraday::Adapter::NetHttp
         stream_download_with_net_http(URI(uri), io, range)
       else
         response = remote_request(:get, uri)
+        reject_frontend_response(response.headers.key?(Transfer::StorageRetry::FRONTEND_HEADER), io)
         io.fulfill_content_length(response.content_length) if io.respond_to?(:fulfill_content_length)
         io.write(response.body)
       end
@@ -261,6 +405,7 @@ module Files
 
     private def write_download_body(response, io)
       reject_unsuccessful_download(response, io)
+      reject_frontend_response(response.key?(Transfer::StorageRetry::FRONTEND_HEADER), io)
       io.fulfill_content_length(response.content_length) if io.respond_to?(:fulfill_content_length)
 
       expected_bytes = expected_download_bytes(response)
@@ -298,6 +443,16 @@ module Files
       raise
     end
 
+    # A response carrying X-Files-Frontend-App came from the Files.com web app, such as a sign-in page, not from file
+    # storage. It is refused before anything of it is written, and closes the destination as a refused transfer does.
+    private def reject_frontend_response(frontend, io)
+      return unless frontend
+
+      error = Transfer::FrontendResponseError.new
+      close_destination_after_failure(io, error)
+      raise error
+    end
+
     private def write_download_chunk(io, chunk)
       io << chunk
     rescue StandardError => e
@@ -344,9 +499,10 @@ module Files
     # with that error, naming the site's own origin in data.host or X-Files-Host, when the site is pinned to another
     # region or to a custom domain. Only the configured HTTPS origin, reached over this client's verified connection, may send the
     # credentials elsewhere, apart from an http:// origin to the destination +http_pairs+ pairs it with.
-    private def region_continuation_base_url(error_response, base_url, body, http_pairs)
-      return if Files.region_policy == :disabled
+    private def region_continuation_base_url(error_response, base_url, body, http_pairs, settings = nil)
+      return if setting(settings, :region_policy) == :disabled
 
+      handler = setting(settings, :region_handler)
       error_object = error_response.data
       return unless error_response.http_status == 403 && error_object[:type] == "not-authenticated/lockout-region-mismatch"
       return unless replayable?(body)
@@ -356,7 +512,7 @@ module Files
 
       host = data && data[:host]
       host = Faraday::Utils::Headers.new(error_response.http_headers || {})["X-Files-Host"] if host.nil? || host == ""
-      return http_pair_origin(base_url, host, http_pairs) unless verified_https_connection?(base_url)
+      return http_pair_origin(base_url, host, http_pairs, handler) unless verified_https_connection?(base_url)
 
       configured = Addressable::URI.parse(base_url)
       original_origin = https_origin(configured.normalized_host, configured.inferred_port)
@@ -364,8 +520,8 @@ module Files
       # A mismatch that names the origin it came from cannot be resolved by moving, so the handler is not asked.
       return if target_origin.nil? || target_origin == original_origin
 
-      if Files.region_handler
-        target_origin = approved_region_origin(original_origin, target_origin, approval_required) { |decision| region_target(decision)&.first }
+      if handler
+        target_origin = approved_region_origin(handler, original_origin, target_origin, approval_required) { |decision| region_target(decision)&.first }
       elsif approval_required
         return
       end
@@ -375,12 +531,13 @@ module Files
       target_origin
     end
 
-    # The origin Files.region_handler approves: the target for true, or another origin it returns that the block gives
-    # back unchanged, as the canonical form of an origin the request may move to. nil when it declines. The handler
-    # gets frozen copies, so it cannot change the origins used here.
-    private def approved_region_origin(original_origin, target_origin, approval_required)
+    # The origin the region handler (Files.region_handler, or the one an adaptive transfer began with) approves: the
+    # target for true, or another origin it returns that the block gives back unchanged, as the canonical form of an
+    # origin the request may move to. nil when it declines. The handler gets frozen copies, so it cannot change the
+    # origins used here.
+    private def approved_region_origin(handler, original_origin, target_origin, approval_required)
       event = { reason: :lockout_region_mismatch, original_origin: original_origin.dup.freeze, target_origin: target_origin.dup.freeze, approval_required: approval_required }.freeze
-      decision = Files.region_handler.call(event)
+      decision = handler.call(event)
       return target_origin if decision == true
       return unless decision.is_a?(String)
 
@@ -393,13 +550,13 @@ module Files
     # continue. Within it a bare host means that host over plain HTTP, on port 80 unless it names another port. The
     # region handler may still decline, or choose another destination paired with the same source; the request then
     # continues at that pair's own frozen destination, not at the String the handler returned.
-    private def http_pair_origin(base_url, host, http_pairs)
+    private def http_pair_origin(base_url, host, http_pairs, handler)
       original_origin = Util.http_origin(base_url)
       target_origin = Util.http_origin(host.is_a?(String) && !host.include?("://") ? "http://#{host}" : host)
       return unless original_origin && target_origin != original_origin && http_pairs.include?([ original_origin, target_origin ])
 
-      if Files.region_handler
-        target_origin = approved_region_origin(original_origin, target_origin, false) do |decision|
+      if handler
+        target_origin = approved_region_origin(handler, original_origin, target_origin, false) do |decision|
           http_pairs.find { |source, destination| source == original_origin && destination == decision }&.last
         end
       end
@@ -499,13 +656,15 @@ module Files
           log_response_error(error_context, request_start, e)
         end
 
-        if self.class.should_retry?(e, num_retries)
+        single_attempt = is_transfer && Thread.current[:files_api_client_single_attempt]
+        if !single_attempt && self.class.should_retry?(e, num_retries)
           num_retries += 1
           wait_seconds = self.class.sleep_time(num_retries)
           log_retry(context, num_retries, e, wait_seconds) unless is_transfer
           sleep wait_seconds
           retry
         end
+        raise if single_attempt
 
         Util.log_debug("Transfer error details", error_message: e.message) if is_transfer
 
@@ -692,25 +851,33 @@ module Files
       raise APIConnectionError, message
     end
 
-    private def request_headers(api_key, session_id, method, workspace_id)
+    # The library-wide setting name, or, for an adaptive transfer's request, what its TransferSettings hold for it.
+    private def setting(settings, name)
+      settings ? settings[name] : Files.public_send(name)
+    end
+
+    private def request_headers(api_key, session_id, method, workspace_id, settings = nil)
+      app_info = setting(settings, :app_info)
       user_agent = "Files.com Ruby SDK v#{Files::VERSION}"
-      user_agent += " #{format_app_info(Files.app_info)}" unless Files.app_info.nil?
+      user_agent += " #{format_app_info(app_info)}" unless app_info.nil?
 
-      headers = { "User-Agent" => user_agent }
+      headers = { "User-Agent" => self.class.user_agent_with_metadata(user_agent) }
       # A request body gets its Content-Type from the multipart and URL-encoding middleware, which choose
       # multipart/form-data when the body holds a file part and application/x-www-form-urlencoded otherwise.
       headers["Content-Type"] = "application/x-www-form-urlencoded" if QUERY_METHODS.include?(method.to_s.downcase.to_sym)
 
-      if Files.default_headers.is_a?(Proc)
-        proc_headers = Files.default_headers.call
+      default_headers = setting(settings, :default_headers)
+      if default_headers.is_a?(Proc)
+        proc_headers = default_headers.call
         headers.merge!(proc_headers) if proc_headers.is_a?(Hash)
       end
 
-      headers.merge!(Files.default_headers) if Files.default_headers.is_a?(Hash)
+      headers.merge!(default_headers) if default_headers.is_a?(Hash)
 
+      language = setting(settings, :language)
       headers["X-FilesAPI-Key"] = api_key if api_key
       headers["X-FilesAPI-Auth"] = session_id if session_id
-      headers["Accept-Language"] = Files.language if Files.language
+      headers["Accept-Language"] = language if language
       workspace_id = workspace_id.to_s
       headers["X-Files-Workspace-Id"] = workspace_id unless workspace_id.empty?
 
@@ -761,5 +928,26 @@ module Files
     class RequestLogContext
       attr_accessor :body, :api_key, :headers, :method, :path, :query_params, :session_id
     end
+
+    # The params encoder of one transfer request (remote_request). Faraday decodes a URL's query into the request's
+    # params and encodes them again as it builds the adapter's URL, sorted and escaped its own way, with the
+    # connection's default params among them. This encoder gives back the query remote_request read from the URL
+    # instead, character for character, whatever the params hold. Decoding, as an adapter may do to read the query
+    # back, is left to the encoder the request had.
+    class SignedQuery
+      def initialize(query, decoder)
+        @query = query
+        @decoder = decoder
+      end
+
+      def encode(_params)
+        @query
+      end
+
+      def decode(query)
+        @decoder.decode(query)
+      end
+    end
+    private_constant :SignedQuery
   end
 end