esirb 0.3.0 → 0.4.0

data/lib/esirb/client.rb

@@ -2,50 +2,155 @@
 
 require "faraday"
 require "faraday-http-cache"
+require_relative "version"
+require_relative "configuration"
+require_relative "compatibility_date"
+require_relative "raise_error"
+require_relative "retry"
+require_relative "error_limit_guard"
+require_relative "vary_by_authorization"
 
 module Esirb
+  # Holds the ESI connection settings and starts building request paths.
   class Client
-    BASE_URL = "https://esi.evetech.net"
-    DEFAULT_HEADERS = {
-      "Accept" => "application/json",
-      "Accept-Language" => "en",
-      "If-Modified-Since" => "",
-      "If-None-Match" => "",
-      "X-Tenant" => "tranquility",
-      "X-Compatibility-Date" => "2026-06-09"
-    }.freeze
-
-    attr_reader :token, :path, :tenant, :language, :cache, :cache_store, :raise_errors
-
-    def initialize(token: nil, tenant: "tranquility", language: "en", cache: true, cache_store: nil, raise_errors: true)
+    # Sent after the app's own user_agent, so ESI can tell which library made the request.
+    USER_AGENT = "esirb/#{VERSION} (+https://github.com/sicks/esirb)".freeze
+
+    attr_reader :token, :base_url, :compatibility_date, :tenant, :language, :cache_store, :user_agent,
+                :timeout, :open_timeout, :retries, :connection
+
+    # token: an ESI access token, or something that responds to call and returns one. A callable
+    # is asked before every request, so an app can refresh its token without building a new client.
+    #
+    # Other options default to Esirb.configuration. compatibility_date can't be nil: without one,
+    # ESI would use its oldest behavior.
+    #
+    # A block is called with the Faraday connection builder after the configured faraday hook, so
+    # it can add middleware or change the adapter for this client only.
+    def initialize(token: nil, **options, &block)
+      check_arguments(token, options)
+
       @token = token
-      @tenant = tenant
-      @language = language
-      @cache = cache
-      @cache_store = cache_store
-      @raise_errors = raise_errors
+      apply_settings(Esirb.configuration.to_h.merge(options))
+      # Built here rather than on first use, so threads sharing a client never race to build it.
+      @connection = build_connection(block)
+    end
+
+    # Starts a path with a segment exactly as given, for segments that can't be reached by method name.
+    def segment(name, *params)
+      Path.new(client: self).segment(name, *params)
+    end
+
+    def method_missing(name, *, **)
+      return super unless Path.segment_name?(name)
+
+      Path.new(client: self).public_send(name, *, **)
+    end
+
+    def respond_to_missing?(name, include_private = false)
+      Path.segment_name?(name) || super
     end
 
-    def connection
-      @connection ||= Faraday.new(BASE_URL) do |c|
-        if cache
-          c.use :http_cache, store: cache_store, strategy: Faraday::HttpCache::Strategies::ByUrl, serializer: Marshal
-        end
+    private
 
-        c.headers = DEFAULT_HEADERS
-        c.headers["X-Tenant"] = tenant if tenant
-        c.headers["Accept-Language"] = language if language
+    def build_connection(block)
+      # ESI reads array params as repeated keys (categories=a&categories=b), not Faraday's default
+      # categories[]=a&categories[]=b.
+      request = { timeout:, open_timeout:, params_encoder: Faraday::FlatParamsEncoder }.compact
+      Faraday.new(base_url, request:) do |c|
+        use_middleware(c)
+        # Last, so app middleware sits closest to the network and sees real requests, not cache hits.
+        [@faraday, block].compact.each { |hook| hook.call(c) }
+      end
+    end
 
-        c.request :authorization, "Bearer", token if token
-        c.request :json
+    def use_middleware(builder)
+      # Before the cache, so the token is part of each cache key.
+      builder.request :authorization, "Bearer", access_token if token
+      builder.headers.update(default_headers)
 
-        c.response :raise_error if raise_errors
-        c.response :json, content_type: "application/json"
+      use_error_handling(builder)
+      builder.response :json
+      # Inside the JSON parser, so the cache stores raw response bodies.
+      use_cache(builder) if cache_store
+
+      builder.request :json
+    end
+
+    # What the authorization middleware sends: the string itself, or a lambda it calls before each
+    # request. The lambda refuses a blank token rather than sending "Bearer " to ESI.
+    def access_token
+      return token unless token.respond_to?(:call)
+
+      lambda do
+        value = token.call
+        raise Error, "token callable returned #{value.inspect} instead of an access token" if value.to_s.strip.empty?
+
+        value
       end
     end
 
-    def method_missing(m, *args, &block)
-      @path = Path.new(client: self).send(m.to_sym, *args)
+    def check_arguments(token, options)
+      unknown = options.keys - Configuration::DEFAULTS.keys
+      raise ArgumentError, "unknown option#{"s" if unknown.size > 1}: #{unknown.join(", ")}" if unknown.any?
+
+      check_callable(:faraday, options[:faraday])
+      check_retries(options.fetch(:retries, 0))
+      check_callable(:token, token, allow_string: true)
+    end
+
+    def check_retries(retries)
+      return if retries.is_a?(Integer) && retries >= 0
+
+      raise ArgumentError, "retries must be an Integer of 0 or more, got #{retries.inspect}"
+    end
+
+    def check_callable(name, value, allow_string: false)
+      return if value.nil? || value.respond_to?(:call) || (allow_string && value.is_a?(String))
+
+      expected = allow_string ? "a string or something that responds to call" : "something that responds to call"
+      raise ArgumentError, "#{name} must be #{expected}, got #{value.class}"
+    end
+
+    # Keys cache entries on the URL plus the headers in Vary: tenant, language, compatibility date
+    # and, through VaryByAuthorization, the token. Because every entry is per token, the store
+    # acts as a private cache and keeps ESI's `Cache-Control: private` responses too.
+    def use_cache(builder)
+      builder.use :http_cache, store: cache_store, strategy: Faraday::HttpCache::Strategies::ByVary,
+                               shared_cache: false
+      builder.use VaryByAuthorization
+    end
+
+    def apply_settings(settings)
+      (Configuration::DEFAULTS.keys - %i[user_agent compatibility_date]).each do |name|
+        instance_variable_set(:"@#{name}", settings[name])
+      end
+      @compatibility_date = CompatibilityDate.parse(settings[:compatibility_date])
+      app_user_agent = settings[:user_agent].to_s.strip
+      @user_agent = app_user_agent.empty? ? USER_AGENT : "#{app_user_agent} #{USER_AGENT}"
+    end
+
+    # Retry sits outside RaiseError, so it sees esirb's errors. ErrorLimitGuard sits inside Retry,
+    # so it checks and records every attempt; it needs the shared cache store.
+    def use_error_handling(builder)
+      builder.use Retry, retries: retries if retries.positive?
+      builder.use ErrorLimitGuard, store: cache_store, key: error_limit_key if cache_store
+      builder.use RaiseError
+    end
+
+    # One per ESI host and tenant: ESI counts the error limit per source IP, not per token.
+    def error_limit_key
+      "esirb:error-limit:#{base_url}:#{tenant}"
+    end
+
+    def default_headers
+      {
+        "Accept" => "application/json",
+        "Accept-Language" => language,
+        "X-Tenant" => tenant,
+        "X-Compatibility-Date" => compatibility_date,
+        "User-Agent" => user_agent
+      }.compact
     end
   end
 end

data/lib/esirb/compatibility_date.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require "date"
+
+module Esirb
+  # Checks the date sent as X-Compatibility-Date. Any date from ESI's oldest supported one to today
+  # works; ESI uses the latest published compatibility date on or before it. The oldest supported
+  # date isn't checked here, since CCP can raise it.
+  module CompatibilityDate
+    FORMAT = /\A\d{4}-\d{2}-\d{2}\z/
+
+    # Returns the date as a YYYY-MM-DD string, or raises ArgumentError.
+    def self.parse(value)
+      if value.nil?
+        raise ArgumentError, "compatibility_date can't be nil, since ESI would then use its oldest behavior. " \
+                             'Pass that date ("2020-01-01") explicitly if that\'s what you want.'
+      end
+
+      date = to_date(value)
+      raise ArgumentError, "compatibility_date #{date} is in the future; ESI's current date is #{today}" if date > today
+
+      date.strftime("%Y-%m-%d")
+    end
+
+    # ESI's date rolls over at 11:00 UTC.
+    def self.today
+      (Time.now.utc - (11 * 60 * 60)).to_date
+    end
+
+    def self.to_date(value)
+      # DateTime is a Date too. to_date drops its time, so a later hour today isn't counted as the future.
+      return value.to_date if value.is_a?(Date)
+
+      text = value.to_s
+      return Date.iso8601(text) if text.match?(FORMAT) && Date.valid_date?(*text.split("-").map(&:to_i))
+
+      raise ArgumentError, "compatibility_date must be a YYYY-MM-DD date, got #{value.inspect}"
+    end
+    private_class_method :to_date
+  end
+end

data/lib/esirb/configuration.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+# A light Ruby client for EVE Online's ESI API.
+module Esirb
+  # App-wide defaults for new clients. Set them once with Esirb.configure; options passed to
+  # Esirb::Client.new override them for that client.
+  class Configuration
+    DEFAULTS = {
+      base_url: "https://esi.evetech.net",
+      # Part of the API contract: a newer date can change response shapes. Only move it in a
+      # release that has been checked against that date.
+      compatibility_date: "2026-08-18",
+      tenant: "tranquility",
+      language: "en",
+      # Your app's name, version and contact info, e.g. "mappy/1.2 (admin@example.com)".
+      user_agent: nil,
+      # Where to cache responses, e.g. Rails.cache. Anything with read, write and delete works.
+      # nil turns caching off.
+      cache_store: nil,
+      # Seconds to wait for a response, and to open the connection. nil leaves it to the adapter.
+      timeout: 30,
+      open_timeout: 5,
+      # Times to retry a 502, 503 or 504, a timeout, a connection failure or a short 429, on GET,
+      # HEAD, PUT and DELETE. Retries stop when ESI's error limit is nearly used up. 0 turns them off.
+      retries: 2,
+      # Called with the Faraday connection builder after esirb's own setup, to add middleware,
+      # change the adapter or set other connection options.
+      faraday: nil
+    }.freeze
+
+    attr_accessor(*DEFAULTS.keys)
+
+    def initialize
+      DEFAULTS.each { |name, value| public_send(:"#{name}=", value) }
+    end
+
+    # The settings, keyed like DEFAULTS.
+    def to_h
+      DEFAULTS.keys.to_h { |name| [name, public_send(name)] }
+    end
+  end
+
+  class << self
+    # The defaults new clients start from.
+    def configuration
+      @configuration ||= Configuration.new
+    end
+
+    # Changes the defaults for clients created afterwards.
+    def configure
+      yield configuration
+    end
+
+    # Restores esirb's own defaults; mainly useful in tests.
+    def reset_configuration!
+      @configuration = Configuration.new
+    end
+  end
+end

data/lib/esirb/error_limit_guard.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require "faraday"
+require_relative "errors"
+require_relative "rate_limit"
+
+module Esirb
+  # Stops sending requests while ESI's error limit is nearly used up. Once that limit runs out,
+  # every ESI route returns 420 until the error limit window resets, and repeatedly ignoring it can get an app
+  # banned (https://developers.eveonline.com/docs/services/esi/rate-limiting/).
+  #
+  # The limit is counted per source IP, not per token, so its state is kept once per ESI host in
+  # the cache store, where every client in every process sees it. Apps on several servers that
+  # share one store will pause together, which errs on the side of sending less.
+  class ErrorLimitGuard < Faraday::Middleware
+    # How long to pause after a 420 that doesn't say when the error limit window resets. The window is a minute.
+    DEFAULT_RESET = 60
+
+    def initialize(app, store:, key:)
+      super(app)
+      @store = store
+      @key = key
+    end
+
+    def call(env)
+      raise_if_paused
+      response = @app.call(env)
+      record_error_limit(response.status, response.headers) unless cached?(response.env)
+      response
+    rescue ResponseError => e
+      record_error_limit(e.status, e.headers)
+      raise
+    end
+
+    private
+
+    def raise_if_paused
+      state = @store.read(@key)
+      return unless state
+
+      seconds = (state["reset_at"] - Time.now.to_f).ceil
+      return unless seconds.positive? && state["remaining"] < RateLimit::ERROR_LIMIT_FLOOR
+
+      raise ErrorLimitReached.new(remaining: state["remaining"], retry_after: seconds)
+    end
+
+    # Routes on the newer rate limit don't send error limit headers, and leave the state alone.
+    def record_error_limit(status, headers)
+      limit = RateLimit.new(headers)
+      remaining = status == 420 ? 0 : limit.error_limit_remaining
+      reset = limit.error_limit_reset || (DEFAULT_RESET if status == 420)
+      return if remaining.nil? || reset.nil?
+
+      @store.write(@key, { "remaining" => remaining, "reset_at" => Time.now.to_f + reset })
+    end
+
+    # A response served from the cache carries the error limit from when it was first fetched.
+    def cached?(env)
+      Array(env[:http_cache_trace]).include?(:fresh)
+    end
+  end
+end

data/lib/esirb/errors.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require_relative "rate_limit"
+
+module Esirb
+  # Base class for everything esirb raises.
+  class Error < StandardError; end
+
+  # Raised when the connection to ESI fails, including TLS errors.
+  class ConnectionFailed < Error; end
+
+  # Raised when ESI doesn't answer in time.
+  class TimeoutError < Error; end
+
+  # Raised for an error status from ESI. ESI's error bodies look like {"error" => "System not found"}.
+  class ResponseError < Error
+    # The response's status, headers and parsed body, and the full Faraday response.
+    attr_reader :status, :headers, :body, :response
+
+    def self.for_status(status)
+      STATUS_ERRORS.fetch(status) { status >= 500 ? ServerError : ClientError }
+    end
+
+    def initialize(env)
+      @status = env.status
+      @headers = env.response_headers
+      @body = env.body
+      @response = env.response
+      super("#{status} #{error_message(env)} (#{env.method.to_s.upcase} #{env.url})")
+    end
+
+    # ESI's rate or error limit as of this response.
+    def rate_limit
+      RateLimit.new(headers)
+    end
+
+    private
+
+    def error_message(env)
+      message = body["error"] if body.is_a?(Hash)
+      return message if message.is_a?(String) && !message.empty?
+
+      env.reason_phrase.to_s.empty? ? "error" : env.reason_phrase
+    end
+  end
+
+  # Raised without sending the request, because ESI's error limit is nearly used up and the error
+  # limit window hasn't reset yet. Only happens with a cache_store; see ErrorLimitGuard.
+  class ErrorLimitReached < Error
+    # Seconds until ESI's error limit window resets.
+    attr_reader :retry_after
+
+    def initialize(remaining:, retry_after:)
+      @retry_after = retry_after
+      super("ESI's error limit is nearly used up (#{remaining} errors left), " \
+            "so requests are paused for #{retry_after} more seconds")
+    end
+  end
+
+  # Raised by Path#get_all_pages when ESI refreshed the data while its pages were being fetched,
+  # twice in a row, so the pages can't be combined consistently. Trying again later usually works.
+  class PagesChanged < Error; end
+
+  # Any other 4xx status, and the parent of the classes for specific 4xx statuses.
+  class ClientError < ResponseError; end
+
+  class BadRequest < ClientError; end
+  class Unauthorized < ClientError; end
+  class Forbidden < ClientError; end
+  class NotFound < ClientError; end
+
+  # ESI's error limit (420) or rate limit (429).
+  class RateLimited < ClientError
+    # Seconds to wait before trying again, or nil if ESI didn't say.
+    def retry_after
+      value = status == 429 ? headers["Retry-After"] : headers["X-ESI-Error-Limit-Reset"]
+      Integer(value, exception: false) if value
+    end
+  end
+
+  # 420: too many errors in the error limit window. The limit is global, so every ESI route
+  # returns 420 until it resets. Never retried.
+  class ErrorLimited < RateLimited; end
+
+  # 429: a rate limit group's tokens are used up. Retried when Retry-After is short.
+  class TooManyRequests < RateLimited; end
+
+  # Any other 5xx status, and the parent of the classes for specific 5xx statuses.
+  class ServerError < ResponseError; end
+
+  # Usually transient on ESI's side, so esirb retries these (see the retries option).
+  class BadGateway < ServerError; end
+  class ServiceUnavailable < ServerError; end
+  class GatewayTimeout < ServerError; end
+
+  ResponseError::STATUS_ERRORS = {
+    400 => BadRequest,
+    401 => Unauthorized,
+    403 => Forbidden,
+    404 => NotFound,
+    420 => ErrorLimited,
+    429 => TooManyRequests,
+    502 => BadGateway,
+    503 => ServiceUnavailable,
+    504 => GatewayTimeout
+  }.freeze
+end

data/lib/esirb/pagination.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require_relative "errors"
+
+module Esirb
+  # Helpers for ESI's three pagination styles, included in Path. Each route uses one of them; its
+  # entry in ESI's API Explorer shows which by its parameters (page, before/after, or from_id).
+  # Pages are fetched one at a time, and each goes through the cache and retries like any request.
+  # See https://developers.eveonline.com/docs/services/esi/pagination/x-pages/
+  module Pagination
+    # Every record from a route paginated with `page` and the X-Pages header, e.g. market orders.
+    #
+    # All pages of one resource should share a Last-Modified. If it changes partway through, ESI
+    # refreshed the data between pages, so this starts over once, and raises PagesChanged if it happens again.
+    def get_all_pages(params: {}, headers: nil)
+      2.times do
+        records = pages_unless_changed(params, headers)
+        return records if records
+      end
+      raise PagesChanged, "#{self} changed while its pages were being fetched, twice in a row"
+    end
+
+    # Records from a route paginated with `before` and `after` cursors, e.g. freelance jobs,
+    # oldest first, as [records, after]. Save `after` and pass it back later to get only the records
+    # created or changed since. Without it, this fetches everything.
+    #
+    # Records can repeat across pages; each appears once, matched by its "id": the newest copy when
+    # following `after` toward newer records, and the one already fetched when following `before`
+    # toward older records.
+    def get_by_cursor(after: nil, params: {}, headers: nil)
+      after ? cursor_changes(after, params, headers) : cursor_everything(params, headers)
+    end
+
+    # Records from a route paginated with `from_id`, i.e. wallet transactions, newest first. Pass
+    # `stop_at:` with the newest ID you already have to fetch only the records after it.
+    def get_from_id(id_key: "transaction_id", stop_at: nil, params: {}, headers: nil)
+      records = []
+      from_id = nil
+      loop do
+        page = from_id_page(from_id, id_key, params, headers)
+        stop_index = stop_at && page.index { |record| record[id_key] == stop_at }
+        return records + page.first(stop_index) if stop_index
+        return records if page.empty?
+
+        records.concat(page)
+        from_id = page.last[id_key]
+      end
+    end
+
+    private
+
+    def pages_unless_changed(params, headers)
+      first_page = get(params: params.merge(page: 1), headers:)
+      records = first_page.body.dup
+      (2..first_page.headers["X-Pages"].to_i).each do |page|
+        response = get(params: params.merge(page:), headers:)
+        return nil unless response.headers["Last-Modified"] == first_page.headers["Last-Modified"]
+
+        records.concat(response.body)
+      end
+      records
+    end
+
+    # Each page after the first starts with the from_id record itself, which is dropped.
+    def from_id_page(from_id, id_key, params, headers)
+      return get(params:, headers:).body unless from_id
+
+      get(params: params.merge(from_id:), headers:).body.reject { |record| record[id_key] == from_id }
+    end
+
+    # Follows `before` until a page comes back empty, prepending older records.
+    def cursor_everything(params, headers)
+      records, cursor = cursor_page(params, headers)
+      after = cursor["after"]
+      while cursor["before"]
+        older, cursor = cursor_page(params.merge(before: cursor["before"]), headers)
+        break if older.empty?
+
+        ids = records.to_set { |record| record["id"] }
+        records = older.reject { |record| ids.include?(record["id"]) } + records
+      end
+      [records, after]
+    end
+
+    # Follows `after` until a page comes back empty, replacing records that changed.
+    def cursor_changes(after, params, headers)
+      records = []
+      loop do
+        newer, cursor = cursor_page(params.merge(after:), headers)
+        return [records, after] if newer.empty? || cursor["after"].nil?
+
+        ids = newer.to_set { |record| record["id"] }
+        records = records.reject { |record| ids.include?(record["id"]) } + newer
+        after = cursor["after"]
+      end
+    end
+
+    # A cursor page is {"cursor" => {...}, "<records>" => [...]}, where the records' key varies by route.
+    def cursor_page(params, headers)
+      body = get(params:, headers:).body
+      records = body.find { |key, value| key != "cursor" && value.is_a?(Array) }&.last || []
+      [records, body["cursor"] || {}]
+    end
+  end
+end