x-core 1.0.0

data/lib/x/core/response.rb

@@ -0,0 +1,196 @@
+# frozen_string_literal: true
+
+require "json"
+require "net/http"
+require_relative "built_response"
+require_relative "rate_limit"
+require_relative "response_headers"
+
+module X
+  module Core
+    # A summary of one API response, or one object of a stream, which a client passes to its on_response hook
+    # @api public
+    class ::X::Response
+      include ResponseHeaders
+
+      # The body of a response parsed, which the response and its copies share
+      #
+      # It is an object of its own, rather than the state of the response, so that a response that was frozen counts
+      # its resources as any other does, and a dup or clone of a response parses no body again: the body they hold
+      # is one. A response frozen deeply, as Ractor.make_shareable freezes one, freezes it too, and parses its body
+      # for each count, unless it was parsed before.
+      #
+      # @api private
+      class ParsedBody
+        # The body parsed as a JSON object
+        # @api private
+        # @return [Hash{String => Object}, nil] the parsed body, or nil for a body not yet parsed
+        attr_accessor :json
+      end
+      private_constant :ParsedBody
+
+      # @!method headers
+      #   The headers of the response
+      #
+      #   The names are lowercase, and a field the API sent more than once is joined with a comma.
+      #   @api public
+      #   @return [Hash{String => String}] the headers, frozen
+      #   @example Read how long the API took to answer
+      #     response.headers["x-response-time"]
+
+      # The HTTP method of the request
+      # @api public
+      # @return [Symbol] the HTTP method
+      # @example Get the HTTP method
+      #   response.http_method # => :get
+      attr_reader :http_method
+
+      # The URI of the request
+      # @api public
+      # @return [URI::Generic] the request URI
+      # @example Get the path of the request
+      #   response.uri.path # => "/2/users/me"
+      attr_reader :uri
+
+      # The response itself, as the client received it
+      #
+      # It is an escape hatch, for what a summary does not read: the status is {#status}, the headers are
+      # {#headers}, and the body is {#body}. It is the response of the transport the client sent the request with, a
+      # Net::HTTPResponse today, or the one built of the status, headers, and body the summary was given. Its class is
+      # not covered by the compatibility promise of 1.x: a later release of 1.x may send its requests with another
+      # library, whose response this then returns.
+      #
+      # @api public
+      # @return [Net::HTTPResponse] the response of the transport
+      # @example Read the reason phrase of the status line
+      #   response.http_response.message # => "OK"
+      attr_reader :http_response
+
+      # Summarize a response
+      #
+      # Public, so that an on_response hook can be tested with a summary built from the status, headers, and body of
+      # a response, or from a Net::HTTP response, as the client builds one for each response it reads.
+      #
+      # @api public
+      # @param http_method [Symbol, String] the HTTP method of the request, in any case, which is read as a lowercase
+      #   Symbol, as an error reads it
+      # @param uri [URI::Generic] the URI of the request
+      # @param http_response [Net::HTTPResponse, nil] the response of the transport, an escape hatch as
+      #   {#http_response} is, whose class is not covered by the compatibility promise of 1.x, or nil for one built of
+      #   the status, headers, and body, which every release of 1.x takes
+      # @param status [Integer, nil] the status of the response, from 100 to 599, when it is not given
+      # @param headers [Hash{String => String}, nil] the headers of the response, when it is not given
+      # @param body [String, nil] the part of the body summarized, such as one object of a stream, or nil for all of
+      #   it, which is the body of a response built of the status
+      # @return [Response] a new summary
+      # @raise [ArgumentError] if the HTTP response is given beside a status or headers, or neither it nor a status is
+      #   given, or the status is not from 100 to 599, or the headers are not a Hash of names to values
+      # @example Summarize a response
+      #   X::Response.new(http_method: :get, uri: URI("https://api.x.com/2/users/me"), status: 200,
+      #     headers: {"x-rate-limit-remaining" => "74"}, body: %({"data":{"id":"1"}}))
+      # @example Summarize a Net::HTTP response
+      #   X::Response.new(http_response:, http_method: :get, uri: URI("https://api.x.com/2/users/me"))
+      def initialize(http_method:, uri:, http_response: nil, status: nil, headers: nil, body: nil)
+        @http_method = http_method.downcase.to_sym
+        @uri = uri
+        @http_response = BuiltResponse.of(http_response, status:, headers:, body: (body if http_response.nil?))
+        @body = body.dup&.force_encoding(Encoding::UTF_8)
+        @parsed = ParsedBody.new
+      end
+
+      # The body summarized: one streamed object, or else the whole body
+      #
+      # It is tagged UTF-8, the encoding of the JSON the API sends. A body that is not valid UTF-8 keeps its bytes, so
+      # valid_encoding? tells it apart, and scrub replaces what is not UTF-8.
+      #
+      # @api public
+      # @return [String, nil] the body, tagged UTF-8
+      # @example Log the body
+      #   logger.debug(response.body)
+      def body = @body || http_response.body
+
+      # The HTTP status code
+      #
+      # @api public
+      # @return [Integer] the status code
+      # @example Get the status code
+      #   response.status # => 200
+      def status = Integer(http_response.code)
+
+      # Check whether the request succeeded
+      #
+      # @api public
+      # @return [Boolean] true for a 2xx status
+      # @example Count the failed requests
+      #   failures += 1 unless response.success?
+      def success? = http_response.is_a?(Net::HTTPSuccess)
+
+      # The rate limits the response reports in its headers
+      #
+      # @api public
+      # @return [Array<RateLimit>] the 15-minute limit, and the 24-hour app and user limits when reported
+      # @example Print how many requests remain in each window
+      #   response.rate_limits.each { |limit| puts "#{limit.type}: #{limit.remaining}" }
+      def rate_limits = RateLimit.__send__(:all_from, http_response)
+
+      # The 15-minute rate limit of the endpoint, which nearly every response reports
+      #
+      # @api public
+      # @return [RateLimit, nil] the rate limit, or nil if the response reports none
+      # @example Slow down near the limit
+      #   sleep response.rate_limit.reset_in if response.rate_limit&.remaining&.zero?
+      def rate_limit = rate_limits.find { |limit| limit.type.eql?(RateLimit::RATE_LIMIT_TYPE) }
+
+      # The number of resources the body holds, as data and as each kind of include
+      #
+      # The API bills reads by the resource, but once a UTC day for each, so these count what the response returned,
+      # not what X billed for it. The body is parsed once, however many times a summary is asked what it holds,
+      # unless the summary was frozen deeply, as Ractor.make_shareable freezes one, before it was first asked, which
+      # leaves it nowhere to keep what it parsed. A body whose includes is not an object holds no includes to count.
+      #
+      # @api public
+      # @return [Hash{String => Integer}] the count of data and of each include, such as users and posts
+      # @example Count the users a lookup returned
+      #   response.resource_counts # => {"data" => 1, "posts" => 1}
+      def resource_counts
+        body = parsed_body
+        data = body["data"]
+        counts = {"data" => Array.try_convert(data)&.size || [data].compact.size}
+        Hash.try_convert(body["includes"])&.each { |key, resources| counts[key] = Array(resources).size }
+        counts
+      end
+
+      # The number of resources the body holds, in data and includes together
+      #
+      # @api public
+      # @return [Integer] the resource count
+      # @example Total the resources a client has read
+      #   total += response.resource_count
+      def resource_count = resource_counts.values.sum
+
+      private
+
+      # The body parsed as a JSON object, read once and kept
+      #
+      # A hook that reads a summary reads a body the client has parsed already, so parsing it again for each count
+      # would parse every response of a client twice. It is kept apart from the state of the response, so a response
+      # that was frozen keeps it too. What holds it is frozen with a response that was frozen deeply, as
+      # Ractor.make_shareable freezes one, which then parses its body without keeping it, rather than raise.
+      #
+      # @api private
+      # @return [Hash{String => Object}] the parsed body
+      def parsed_body
+        @parsed.json || (@parsed.frozen? ? parse_body : @parsed.json = parse_body)
+      end
+
+      # The body parsed as a JSON object, or an empty one for a body that holds none
+      # @api private
+      # @return [Hash{String => Object}] the parsed body
+      def parse_body
+        Hash.try_convert(JSON.parse(body.to_s)) || {}
+      rescue JSON::ParserError
+        {}
+      end
+    end
+  end
+end

data/lib/x/core/response_headers.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module X
+  module Core
+    # The headers of a response, read from the response it holds, included into Response, HTTPError, and InvalidResponse
+    #
+    # Internal to x-core: it gives a summary of a response, and the error of a failed one, the same headers.
+    #
+    # @api private
+    module ResponseHeaders
+      # The headers of the response
+      #
+      # The names are lowercase, whatever case the API sent them in, since HTTP field names mean the same in any
+      # case, so a header is read here by the name it is written with here. A field the API sent more than once
+      # is joined with a comma, as HTTP joins the lines of a repeated field.
+      #
+      # Set-Cookie is joined the same way, though HTTP does not join it, since a cookie may hold a comma, as its
+      # Expires attribute does, so the cookies of a response that sets more than one cannot be read apart from its
+      # headers: read each of them from the http_response, with get_fields.
+      #
+      # @api public
+      # @return [Hash{String => String}] the headers, frozen
+      # @example Read how long the API took to answer
+      #   response.headers["x-response-time"]
+      # @example Read each cookie a response sets
+      #   response.http_response.get_fields("set-cookie")
+      def headers = http_response.to_hash.transform_values { |values| values.join(", ") }.freeze
+    end
+    private_constant :ResponseHeaders
+  end
+end

data/lib/x/core/response_parser.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+require "json"
+require "net/http"
+require_relative "errors/bad_gateway"
+require_relative "errors/bad_request"
+require_relative "errors/callback_error"
+require_relative "errors/conflict"
+require_relative "errors/http_error"
+require_relative "errors/invalid_response"
+require_relative "errors/forbidden"
+require_relative "errors/gateway_timeout"
+require_relative "errors/gone"
+require_relative "errors/internal_server_error"
+require_relative "errors/method_not_allowed"
+require_relative "errors/not_acceptable"
+require_relative "errors/not_found"
+require_relative "errors/payload_too_large"
+require_relative "errors/payment_required"
+require_relative "errors/request_timeout"
+require_relative "errors/service_unavailable"
+require_relative "errors/too_many_requests"
+require_relative "errors/unauthorized"
+require_relative "errors/unavailable_for_legal_reasons"
+require_relative "errors/unprocessable_entity"
+require_relative "errors/unsupported_media_type"
+require_relative "request_context"
+
+module X
+  module Core
+    # Parses HTTP responses from the X API
+    #
+    # Internal to x-core: Client parses responses with it.
+    #
+    # @api private
+    class ResponseParser
+      # Mapping of HTTP status codes to error classes
+      ERROR_MAP = {
+        400 => BadRequest,
+        401 => Unauthorized,
+        402 => PaymentRequired,
+        403 => Forbidden,
+        404 => NotFound,
+        405 => MethodNotAllowed,
+        406 => NotAcceptable,
+        408 => RequestTimeout,
+        409 => Conflict,
+        410 => Gone,
+        413 => PayloadTooLarge,
+        415 => UnsupportedMediaType,
+        422 => UnprocessableEntity,
+        429 => TooManyRequests,
+        451 => UnavailableForLegalReasons,
+        500 => InternalServerError,
+        502 => BadGateway,
+        503 => ServiceUnavailable,
+        504 => GatewayTimeout
+      }.freeze
+      # Error classes for the statuses ERROR_MAP does not name, keyed by the class of status: 4xx or 5xx
+      STATUS_CLASS_ERRORS = {4 => ClientError, 5 => ServerError}.freeze
+
+      # Parse an HTTP response
+      #
+      # A successful response without a body, such as 204 No Content, parses to nil. One whose body is not JSON
+      # raises, rather than parse to nil as though the response had no body.
+      #
+      # @api private
+      # @param response [Net::HTTPResponse] the HTTP response to parse
+      # @param array_class [Class, nil] the class for parsing JSON arrays
+      # @param object_class [Class, nil] the class for parsing JSON objects, or a class that builds objects from
+      #   the whole body (see {#decode})
+      # @param client [Client, nil] the client that made the request
+      # @param request [Net::HTTPRequest, nil] the request the response answers, which its error names
+      # @return [Object, nil] the parsed response body
+      # @raise [HTTPError] if the response is not successful
+      # @raise [InvalidResponse] if the body of a successful response is not JSON
+      # @example Parse a response
+      #   parser.parse(response: response, request: request)
+      def parse(response:, array_class: nil, object_class: nil, client: nil, request: nil)
+        raise error(response, request) unless response.is_a?(Net::HTTPSuccess)
+
+        body = response.body.to_s
+        return if blank?(body)
+
+        begin
+          decode(body, array_class:, object_class:, client:)
+        rescue JSON::ParserError
+          raise InvalidResponse.new(http_response: response, **RequestContext.of(request))
+        end
+      end
+
+      # Decode a JSON document into the classes a request asked for
+      #
+      # JSON gives every object in a document the same object_class, at every depth. A class that
+      # models a whole response instead responds to from_response, which receives the document
+      # parsed into Hashes and Arrays along with the client, and whatever it returns is the result;
+      # see {Client} for the protocol, which later versions of 1.x may pass other keywords to. An error from_response
+      # raises is tagged as a CallbackError, so that the handlers of a request do not take it for the request's own.
+      #
+      # @api private
+      # @param json [String] the JSON document
+      # @param array_class [Class, nil] the class for parsing JSON arrays
+      # @param object_class [Class, #from_response, nil] the class for parsing JSON objects, or one that
+      #   builds objects from the whole document
+      # @param client [Client, nil] the client that made the request, passed to from_response
+      # @return [Object] the decoded document
+      # @raise [JSON::ParserError] if the document is not valid JSON
+      # @raise [CallbackError] if from_response raises
+      # @example Decode a document into the default classes
+      #   parser.decode('{"data": {"id": "1"}}') # => {"data" => {"id" => "1"}}
+      def decode(json, array_class: nil, object_class: nil, client: nil)
+        return JSON.parse(json, array_class:, object_class:) unless object_class.respond_to?(:from_response)
+
+        document = JSON.parse(json)
+        CallbackError.tagging { object_class.from_response(document, client:) }
+      end
+
+      # Create the error of a response that is not successful
+      #
+      # Internal to x-core: TokenEndpoint raises it for a token request the endpoint failed to answer.
+      #
+      # @api private
+      # @param response [Net::HTTPResponse] the HTTP response
+      # @param request [Net::HTTPRequest, nil] the request the response answers, which the error names
+      # @return [HTTPError] the error
+      # @example Raise the error of a response
+      #   raise parser.error(response, request)
+      def error(response, request)
+        error_class(response).new(http_response: response, **RequestContext.of(request))
+      end
+
+      private
+
+      # Check whether a body holds nothing but whitespace
+      #
+      # A body that is not valid UTF-8 holds something other than whitespace, and a regular expression raises for it,
+      # so it is not matched against one: it is not JSON, and raises InvalidResponse, which holds it.
+      #
+      # @api private
+      # @param body [String] the body of the response, tagged UTF-8
+      # @return [Boolean] true for a body that is empty or holds whitespace alone
+      def blank?(body) = body.valid_encoding? && !body.match?(/\S/)
+
+      # Get the error class for a response, falling back on its class of status
+      # @api private
+      # @param response [Net::HTTPResponse] the HTTP response
+      # @return [Class] the error class
+      def error_class(response)
+        status = Integer(response.code)
+        ERROR_MAP.fetch(status) { STATUS_CLASS_ERRORS.fetch(status / 100, HTTPError) }
+      end
+    end
+    private_constant :ResponseParser
+  end
+end

data/lib/x/core/retry_handler.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "socket"
+require_relative "errors/callback_error"
+require_relative "errors/http_error"
+require_relative "errors/network_error"
+require_relative "errors/request_timeout"
+require_relative "errors/server_error"
+require_relative "setting_validator"
+
+module X
+  module Core
+    # Sends a request again after the API failed to answer it, or after its answer never arrived
+    #
+    # Internal to x-core: Client retries with it, and takes max_retries, and Client#with_retries sends a request
+    # again with it that a client sends no more than once, as it does any POST.
+    #
+    # @api private
+    class RetryHandler
+      # Default maximum number of retries, which sends an idempotent request twice more before it raises
+      DEFAULT_MAX_RETRIES = 2
+      # Seconds to wait before the first retry, doubled for each retry after, up to MAX_RETRY_AFTER
+      INITIAL_WAIT = 1
+      # The longest wait a response may ask for that a request waits out before it is sent again, and the longest the
+      # backoff grows to; a response that asks to be left alone for longer raises at once, rather than hold a caller for
+      # minutes on end
+      MAX_RETRY_AFTER = 60
+      # The failures a retry may follow, none of which the request itself is the reason for: a 408 says the API gave
+      # up waiting for the request, not that it refused it
+      RETRIABLE_ERRORS = [NetworkError, ServerError, RequestTimeout].freeze
+      # The errors of a socket, the cause of a NetworkError, that fail a request before any of it is written: a host
+      # that cannot be resolved or reached, a connection refused, and a connection or TLS handshake that timed out
+      UNSENT_ERRORS = [Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ENETUNREACH, Net::OpenTimeout, SocketError].freeze
+
+      # The maximum number of times to send an idempotent request again after a failure
+      # @api private
+      # @return [Integer] the maximum number of retries
+      # @example Read the maximum retries
+      #   handler.max_retries # => 2
+      attr_reader :max_retries
+
+      # Initialize a new retry handler
+      #
+      # @api private
+      # @param max_retries [Integer] the maximum number of times to send an idempotent request again
+      # @return [RetryHandler] a new instance
+      # @raise [ArgumentError] if the maximum number of retries is not an Integer of at least 0
+      # @example Create a handler that sends a failed request twice more
+      #   handler = X::Core::RetryHandler.new(max_retries: 2)
+      def initialize(max_retries: DEFAULT_MAX_RETRIES)
+        @max_retries = SettingValidator.count!(:max_retries, max_retries)
+      end
+
+      # Run a request, running it again after a failure of the API or of the network
+      #
+      # A request is sent again while retries remain, after waiting up to a second before the first retry and up to
+      # twice as long before each retry after, but never more than MAX_RETRY_AFTER, or for as long as the response asks
+      # when it carries a Retry-After header, whichever is longer. A response that asks to be left alone for longer than MAX_RETRY_AFTER raises
+      # at once, since waiting it out would hold the caller for minutes. Only an idempotent request is retried: the
+      # API may have acted on a POST whose answer never arrived, so sending that again could post twice. The block
+      # must build its request anew each time, so that each attempt is signed afresh.
+      #
+      # A TooManyRequests is not retried here: a request sent again before its rate limit resets is refused again, so
+      # RateLimitHandler waits for the reset instead, for as long as max_rate_limit_wait allows, well past
+      # MAX_RETRY_AFTER.
+      #
+      # A NetworkError is retried only when the request never left: a request that timed out reading its response,
+      # or whose connection dropped once it was written, may have been answered, and the API bills a read it answered
+      # whether or not the answer arrived, so sending it again could bill it again. resend_unanswered retries those
+      # too, for a request the API bills nothing for, such as the chunk of an upload.
+      #
+      # An error a callback of the request raised, such as on_response, is not the API's failure, and is raised as it
+      # is, so that a request Client#with_retries wraps is not sent again for it, though the API answered it.
+      #
+      # @api private
+      # @param idempotent [Boolean] whether sending the request again has the same effect as sending it once
+      # @param resend_unanswered [Boolean] whether to send the request again after a NetworkError that may have come
+      #   after the API received it
+      # @yield runs the request
+      # @return [Object] what the block returns
+      # @raise [NetworkError] if the request fails once more than the retries allow
+      # @raise [ServerError, RequestTimeout] if the API fails to answer once more than the retries allow, or asks for a
+      #   wait longer than MAX_RETRY_AFTER
+      # @example Retry a lookup
+      #   handler.handle(idempotent: true) { client.get("users/me") }
+      def handle(idempotent:, resend_unanswered: false)
+        retries = 0
+        begin
+          yield
+        rescue *RETRIABLE_ERRORS => e
+          retries += 1
+          requested = retry_after(e)
+          raise unless idempotent && retries <= max_retries && requested.to_i <= MAX_RETRY_AFTER && resendable?(e, resend_unanswered)
+
+          sleep([requested, backoff(retries)].compact.max)
+          retry
+        end
+      end
+
+      private
+
+      # Whether a failure leaves a request safe to send again
+      #
+      # A ServerError or a RequestTimeout is an answer, which says the API failed to act on the request. A NetworkError
+      # says the API never received it only when its cause is among UNSENT_ERRORS; any other may have come after the
+      # API answered. None of them says so when a callback raised it, rather than the API.
+      #
+      # @api private
+      # @param error [Error] the error the request raised
+      # @param resend_unanswered [Boolean] whether a request the API may have answered is sent again
+      # @return [Boolean] true if the request may be sent again
+      def resendable?(error, resend_unanswered)
+        return false if CallbackError.untagged?(error)
+
+        resend_unanswered || error.is_a?(HTTPError) || UNSENT_ERRORS.any? { |unsent| error.cause.is_a?(unsent) }
+      end
+
+      # The seconds a response asks a request to wait before it is sent again
+      #
+      # A request that never got a response, which raised a NetworkError, asks for no wait, and neither does a
+      # response that carries no Retry-After header.
+      #
+      # @api private
+      # @param error [Error] the error the request raised
+      # @return [Integer, nil] the seconds the response asks for, or nil if it asks for none
+      def retry_after(error)
+        error.retry_after if error.is_a?(HTTPError)
+      end
+
+      # The seconds to wait before a retry
+      #
+      # The wait doubles with each retry until it reaches MAX_RETRY_AFTER, and a random share of up to half of it is
+      # taken off. The share is what keeps the requests apart: a failure of the API fails every request in flight at
+      # once, and requests that waited the same time would be sent again together, to fail together once more. The
+      # cap keeps a client given many retries from sleeping for longer than a response may ask it to, as the waits
+      # of a doubling without end would: the tenth retry would otherwise wait over eight minutes.
+      #
+      # @api private
+      # @param retries [Integer] the number of the retry, counting from one
+      # @return [Float] the seconds to wait
+      def backoff(retries)
+        wait = [INITIAL_WAIT << (retries - 1), MAX_RETRY_AFTER].min
+        wait - (rand * wait / 2)
+      end
+    end
+    private_constant :RetryHandler
+  end
+end