berrycrawl 0.1.0

data/lib/Berrycrawl/internal/http/raw_client.rb

@@ -0,0 +1,252 @@
+# frozen_string_literal: true
+
+module Berrycrawl
+  module Internal
+    module Http
+      # @api private
+      class RawClient
+        # Default HTTP status codes that trigger a retry
+        RETRYABLE_STATUSES = [408, 429, 500, 502, 503, 504, 521, 522, 524].freeze
+        # Initial delay between retries in seconds
+        INITIAL_RETRY_DELAY = 0.5
+        # Maximum delay between retries in seconds
+        MAX_RETRY_DELAY = 60.0
+        # Jitter factor for randomizing retry delays (20%)
+        JITTER_FACTOR = 0.2
+
+        # @return [String] The base URL for requests
+        attr_reader :base_url
+
+        # @param base_url [String] The base url for the request.
+        # @param max_retries [Integer] The number of times to retry a failed request, defaults to 2.
+        # @param timeout [Float] The timeout for the request, defaults to 60.0 seconds.
+        # @param headers [Hash] The headers for the request.
+        # @param auth_provider [Object, nil] An optional auth provider responding to
+        #   `auth_headers`. When present its headers are resolved on every request so
+        #   token-based schemes (e.g. OAuth) can refresh an expired token mid-session.
+        def initialize(base_url:, max_retries: 2, timeout: 60.0, headers: {}, auth_provider: nil)
+          @base_url = base_url
+          @max_retries = max_retries
+          @timeout = timeout
+          @auth_provider = auth_provider
+          @default_headers = {
+            "X-Fern-Language": "Ruby",
+            "X-Fern-SDK-Name": "berrycrawl",
+            "X-Fern-SDK-Version": "0.0.1"
+          }.merge(headers)
+        end
+
+        # @param request [Berrycrawl::Internal::Http::BaseRequest] The HTTP request.
+        # @return [HTTP::Response] The HTTP response.
+        def send(request)
+          url = build_url(request)
+          # Resolve auth headers once per request (not per retry) so token-based
+          # providers refresh at most once here; static providers are cheap.
+          auth_headers = resolve_auth_headers
+          attempt = 0
+          response = nil
+
+          loop do
+            http_request = build_http_request(
+              url:,
+              method: request.method,
+              headers: request.encode_headers(protected_keys: @default_headers.keys + auth_headers.keys),
+              body: request.encode_body,
+              auth_headers: auth_headers
+            )
+
+            conn = connect(url)
+            conn.open_timeout = @timeout
+            conn.read_timeout = @timeout
+            conn.write_timeout = @timeout
+            conn.continue_timeout = @timeout
+
+            response = conn.request(http_request)
+
+            break unless should_retry?(response, attempt)
+
+            delay = retry_delay(response, attempt)
+            sleep(delay)
+            attempt += 1
+          end
+
+          response
+        end
+
+        # Determines if a request should be retried based on the response status code.
+        # @param response [Net::HTTPResponse] The HTTP response.
+        # @param attempt [Integer] The current retry attempt (0-indexed).
+        # @return [Boolean] Whether the request should be retried.
+        def should_retry?(response, attempt)
+          return false if attempt >= @max_retries
+
+          status = response.code.to_i
+          RETRYABLE_STATUSES.include?(status)
+        end
+
+        # Calculates the delay before the next retry attempt using exponential backoff with jitter.
+        # Respects Retry-After header if present.
+        # @param response [Net::HTTPResponse] The HTTP response.
+        # @param attempt [Integer] The current retry attempt (0-indexed).
+        # @return [Float] The delay in seconds before the next retry.
+        def retry_delay(response, attempt)
+          # Check for Retry-After header (can be seconds or HTTP date)
+          retry_after = response["Retry-After"]
+          if retry_after
+            delay = parse_retry_after(retry_after)
+            return [delay, MAX_RETRY_DELAY].min if delay&.positive?
+          end
+
+          # Exponential backoff with jitter: base_delay * 2^attempt
+          base_delay = INITIAL_RETRY_DELAY * (2**attempt)
+          add_jitter([base_delay, MAX_RETRY_DELAY].min)
+        end
+
+        # Parses the Retry-After header value.
+        # @param value [String] The Retry-After header value (seconds or HTTP date).
+        # @return [Float, nil] The delay in seconds, or nil if parsing fails.
+        def parse_retry_after(value)
+          # Try parsing as integer (seconds)
+          seconds = Integer(value, exception: false)
+          return seconds.to_f if seconds
+
+          # Try parsing as HTTP date
+          begin
+            retry_time = Time.httpdate(value)
+            delay = retry_time - Time.now
+            delay.positive? ? delay : nil
+          rescue ArgumentError
+            nil
+          end
+        end
+
+        # Adds random jitter to a delay value.
+        # @param delay [Float] The base delay in seconds.
+        # @return [Float] The delay with jitter applied.
+        def add_jitter(delay)
+          jitter = delay * JITTER_FACTOR * (rand - 0.5) * 2
+          [delay + jitter, 0].max
+        end
+
+        LOCALHOST_HOSTS = %w[localhost 127.0.0.1 [::1]].freeze
+
+        # @param request [Berrycrawl::Internal::Http::BaseRequest] The HTTP request.
+        # @return [URI::Generic] The URL.
+        def build_url(request)
+          encoded_query = request.encode_query
+
+          # If the path is already an absolute URL, use it directly
+          if request.path.start_with?("http://", "https://")
+            url = request.path
+            url = "#{url}?#{encode_query(encoded_query)}" if encoded_query&.any?
+            parsed = URI.parse(url)
+            validate_https!(parsed)
+            return parsed
+          end
+
+          path = request.path.start_with?("/") ? request.path[1..] : request.path
+          base = request.base_url || @base_url
+          url = "#{base.chomp("/")}/#{path}"
+          url = "#{url}?#{encode_query(encoded_query)}" if encoded_query&.any?
+          parsed = URI.parse(url)
+          validate_https!(parsed)
+          parsed
+        end
+
+        # Raises if the URL uses http:// for a non-localhost host, which would
+        # send authentication credentials in plaintext.
+        # @param url [URI::Generic] The parsed URL.
+        def validate_https!(url)
+          return if url.scheme != "http"
+          return if LOCALHOST_HOSTS.include?(url.host)
+
+          raise ArgumentError,
+                "Refusing to send request to non-HTTPS URL: #{url}. " \
+                "HTTP is only allowed for localhost. Use HTTPS or pass a localhost URL."
+        end
+
+        # Resolves the auth headers to send with the next request. Delegates to the
+        # configured auth provider (if any) on every call so that token-based
+        # providers (e.g. OAuth client-credentials) can refresh an expired token
+        # before the request is sent. Returns an empty hash when no provider is set,
+        # which keeps the api-key / basic / bearer / no-auth paths unchanged.
+        # @return [Hash] The auth headers for the current request.
+        def resolve_auth_headers
+          return {} if @auth_provider.nil?
+
+          @auth_provider.auth_headers
+        end
+
+        # @param url [URI::Generic] The url to the resource.
+        # @param method [String] The HTTP method to use.
+        # @param headers [Hash] The headers for the request.
+        # @param body [String, nil] The body for the request.
+        # @param auth_headers [Hash] The auth headers resolved for this request. These
+        #   take precedence over the static default headers but not over per-request
+        #   headers, mirroring the previous baked-header precedence.
+        # @return [HTTP::Request] The HTTP request.
+        def build_http_request(url:, method:, headers: {}, body: nil, auth_headers: {})
+          request = Net::HTTPGenericRequest.new(
+            method,
+            !body.nil?,
+            method != "HEAD",
+            url
+          )
+
+          request_headers = @default_headers.merge(auth_headers).merge(headers)
+          request_headers.each { |name, value| request[name] = value }
+          request.body = body if body
+
+          # Net::HTTP disables its transparent gzip/deflate decoding as soon as an
+          # Accept-Encoding header is set explicitly on the request. Re-enable it so
+          # that compressed response bodies are still inflated.
+          request.extend(DecodeContent) if request_headers.keys.any? { |name| name.to_s.casecmp("accept-encoding").zero? }
+
+          request
+        end
+
+        # Keeps Net::HTTP's transparent gzip/deflate response decoding enabled
+        # even when an Accept-Encoding header is set explicitly on the request.
+        # @api private
+        module DecodeContent
+          def decode_content # rubocop:disable Naming/PredicateMethod
+            true
+          end
+        end
+
+        # @param query [Hash] The query for the request.
+        # @return [String, nil] The encoded query.
+        def encode_query(query)
+          query.to_h.empty? ? nil : URI.encode_www_form(query)
+        end
+
+        # @param url [URI::Generic] The url to connect to.
+        # @return [Net::HTTP] The HTTP connection.
+        def connect(url)
+          is_https = (url.scheme == "https")
+
+          port = if url.port
+                   url.port
+                 elsif is_https
+                   Net::HTTP.https_default_port
+                 else
+                   Net::HTTP.http_default_port
+                 end
+
+          http = Net::HTTP.new(url.host, port)
+          http.use_ssl = is_https
+          http.verify_mode = OpenSSL::SSL::VERIFY_PEER if is_https
+          # NOTE: We handle retries at the application level with HTTP status code awareness,
+          # so we set max_retries to 0 to disable Net::HTTP's built-in network-level retries.
+          http.max_retries = 0
+          http
+        end
+
+        # @return [String]
+        def inspect
+          "#<#{self.class.name}:0x#{object_id.to_s(16)} @base_url=#{@base_url.inspect}>"
+        end
+      end
+    end
+  end
+end

data/lib/Berrycrawl/internal/iterators/cursor_item_iterator.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Berrycrawl
+  module Internal
+    class CursorItemIterator < ItemIterator
+      # Instantiates a CursorItemIterator, an Enumerable class which wraps calls to a cursor-based paginated API and yields individual items from it.
+      #
+      # @param initial_cursor [String] The initial cursor to use when iterating, if any.
+      # @param cursor_field [Symbol] The field in API responses to extract the next cursor from.
+      # @param item_field [Symbol] The field in API responses to extract the items to iterate over.
+      # @param block [Proc] A block which is responsible for receiving a cursor to use and returning the given page from the API.
+      # @return [Berrycrawl::Internal::CursorItemIterator]
+      def initialize(initial_cursor:, cursor_field:, item_field:, &)
+        super()
+        @item_field = item_field
+        @page_iterator = CursorPageIterator.new(initial_cursor:, cursor_field:, &)
+        @page = nil
+      end
+
+      # Returns the CursorPageIterator mediating access to the underlying API.
+      #
+      # @return [Berrycrawl::Internal::CursorPageIterator]
+      def pages
+        @page_iterator
+      end
+    end
+  end
+end

data/lib/Berrycrawl/internal/iterators/cursor_page_iterator.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Berrycrawl
+  module Internal
+    class CursorPageIterator
+      include Enumerable
+
+      # The raw HTTP response from the most recent page response.
+      # @return [Net::HTTPResponse, nil]
+      attr_reader :http_response
+
+      # Instantiates a CursorPageIterator, an Enumerable class which wraps calls to a cursor-based paginated API and yields pages of items.
+      #
+      # @param initial_cursor [String] The initial cursor to use when iterating, if any.
+      # @param cursor_field [Symbol] The name of the field in API responses to extract the next cursor from.
+      # @param block [Proc] A block which is responsible for receiving a cursor to use and returning the given page from the API.
+      #   The block should return a two-element array: [parsed_page, raw_http_response].
+      # @return [Berrycrawl::Internal::CursorPageIterator]
+      def initialize(initial_cursor:, cursor_field:, &block)
+        @need_initial_load = initial_cursor.nil?
+        @cursor = initial_cursor
+        @cursor_field = cursor_field
+        @get_next_page = block
+        @http_response = nil
+      end
+
+      # Iterates over each page returned by the API.
+      #
+      # @param block [Proc] The block which each retrieved page is yielded to.
+      # @return [NilClass]
+      def each(&block)
+        while (page = next_page)
+          block.call(page)
+        end
+      end
+
+      # Whether another page will be available from the API.
+      #
+      # @return [Boolean]
+      def next?
+        @need_initial_load || !@cursor.nil?
+      end
+
+      # Retrieves the next page from the API.
+      #
+      # @return [Object, nil]
+      def next_page
+        return if !@need_initial_load && @cursor.nil?
+
+        @need_initial_load = false
+        result = @get_next_page.call(@cursor)
+        if result.is_a?(Array)
+          fetched_page, raw_response = result
+          @http_response = raw_response
+        else
+          fetched_page = result
+        end
+        @cursor = fetched_page.send(@cursor_field)
+        fetched_page
+      end
+    end
+  end
+end

data/lib/Berrycrawl/internal/iterators/item_iterator.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Berrycrawl
+  module Internal
+    class ItemIterator
+      include Enumerable
+
+      # The raw HTTP response from the most recent page response.
+      # @return [Net::HTTPResponse, nil]
+      def http_response
+        @page_iterator&.http_response
+      end
+
+      # Iterates over each item returned by the API.
+      #
+      # @param block [Proc] The block which each retrieved item is yielded to.
+      # @return [NilClass]
+      def each(&block)
+        while (item = next_element)
+          block.call(item)
+        end
+      end
+
+      # Whether another item will be available from the API.
+      #
+      # @return [Boolean]
+      def next?
+        load_next_page if @page.nil?
+        return false if @page.nil?
+
+        return true if any_items_in_cached_page?
+
+        load_next_page
+        any_items_in_cached_page?
+      end
+
+      # Retrieves the next item from the API.
+      def next_element
+        item = next_item_from_cached_page
+        return item if item
+
+        load_next_page
+        next_item_from_cached_page
+      end
+
+      private
+
+      def next_item_from_cached_page
+        return unless @page
+
+        @page.send(@item_field).shift
+      end
+
+      def any_items_in_cached_page?
+        return false unless @page
+
+        !@page.send(@item_field).empty?
+      end
+
+      def load_next_page
+        @page = @page_iterator.next_page
+      end
+    end
+  end
+end

data/lib/Berrycrawl/internal/iterators/offset_item_iterator.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Berrycrawl
+  module Internal
+    class OffsetItemIterator < ItemIterator
+      # Instantiates an OffsetItemIterator, an Enumerable class which wraps calls to an offset-based paginated API and yields the individual items from it.
+      #
+      # @param initial_page [Integer] The initial page or offset to start from when iterating.
+      # @param item_field [Symbol] The name of the field in API responses to extract the items to iterate over.
+      # @param has_next_field [Symbol] The name of the field in API responses containing a boolean of whether another page exists.
+      # @param step [Boolean] If true, treats the page number as a true offset (i.e. increments the page number by the number of items returned from each call rather than just 1)
+      # @param block [Proc] A block which is responsible for receiving a page number to use and returning the given page from the API.
+      #
+      # @return [Berrycrawl::Internal::OffsetItemIterator]
+      def initialize(initial_page:, item_field:, has_next_field:, step:, &)
+        super()
+        @item_field = item_field
+        @page_iterator = OffsetPageIterator.new(initial_page:, item_field:, has_next_field:, step:, &)
+        @page = nil
+      end
+
+      # Returns the OffsetPageIterator that is mediating access to the underlying API.
+      #
+      # @return [Berrycrawl::Internal::OffsetPageIterator]
+      def pages
+        @page_iterator
+      end
+    end
+  end
+end

data/lib/Berrycrawl/internal/iterators/offset_page_iterator.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module Berrycrawl
+  module Internal
+    class OffsetPageIterator
+      include Enumerable
+
+      # The raw HTTP response from the most recent page response.
+      # @return [Net::HTTPResponse, nil]
+      attr_reader :http_response
+
+      # Instantiates an OffsetPageIterator, an Enumerable class which wraps calls to an offset-based paginated API and yields pages of items from it.
+      #
+      # @param initial_page [Integer] The initial page to use when iterating, if any.
+      # @param item_field [Symbol] The field to pull the list of items to iterate over.
+      # @param has_next_field [Symbol] The field to pull the boolean of whether a next page exists from, if any.
+      # @param step [Boolean] If true, treats the page number as a true offset (i.e. increments the page number by the number of items returned from each call rather than just 1)
+      # @param block [Proc] A block which is responsible for receiving a page number to use and returning the given page from the API.
+      #   The block should return a two-element array: [parsed_page, raw_http_response].
+      # @return [Berrycrawl::Internal::OffsetPageIterator]
+      def initialize(initial_page:, item_field:, has_next_field:, step:, &block)
+        @page_number = initial_page || (step ? 0 : 1)
+        @item_field = item_field
+        @has_next_field = has_next_field
+        @step = step
+        @get_next_page = block
+
+        # A cache of whether the API has another page, if it gives us that information...
+        @next_page = nil
+        # ...or the actual next page, preloaded, if it doesn't.
+        @has_next_page = nil
+
+        @http_response = nil
+      end
+
+      # Iterates over each page returned by the API.
+      #
+      # @param block [Proc] The block which each retrieved page is yielded to.
+      # @return [NilClass]
+      def each(&block)
+        while (page = next_page)
+          block.call(page)
+        end
+      end
+
+      # Whether another page will be available from the API.
+      #
+      # @return [Boolean]
+      def next?
+        return @has_next_page unless @has_next_page.nil?
+        return true if @next_page
+
+        fetched_page = fetch_page(@page_number)
+        fetched_page_items = fetched_page&.send(@item_field)
+        if fetched_page_items.nil? || fetched_page_items.empty?
+          @has_next_page = false
+        else
+          @next_page = fetched_page
+          true
+        end
+      end
+
+      # Returns the next page from the API.
+      def next_page
+        return nil if @page_number.nil?
+
+        if @next_page
+          this_page = @next_page
+          @next_page = nil
+        else
+          this_page = fetch_page(@page_number)
+        end
+
+        @has_next_page = this_page&.send(@has_next_field) if @has_next_field
+
+        items = this_page.send(@item_field)
+        if items.nil? || items.empty?
+          @page_number = nil
+          return nil
+        elsif @step
+          @page_number += items.length
+        else
+          @page_number += 1
+        end
+
+        this_page
+      end
+
+      private
+
+      def fetch_page(page_number)
+        result = @get_next_page.call(page_number)
+        if result.is_a?(Array)
+          fetched_page, raw_response = result
+          @http_response = raw_response
+          fetched_page
+        else
+          result
+        end
+      end
+    end
+  end
+end

data/lib/Berrycrawl/internal/json/request.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Berrycrawl
+  module Internal
+    module JSON
+      # @api private
+      class Request < Berrycrawl::Internal::Http::BaseRequest
+        attr_reader :body
+
+        # @param base_url [String] The base URL for the request
+        # @param path [String] The path for the request
+        # @param method [Symbol] The HTTP method for the request (:get, :post, etc.)
+        # @param headers [Hash] Additional headers for the request (optional)
+        # @param query [Hash] Query parameters for the request (optional)
+        # @param body [Object, nil] The JSON request body (optional)
+        # @param request_options [Berrycrawl::RequestOptions, Hash{Symbol=>Object}, nil]
+        def initialize(base_url:, path:, method:, headers: {}, query: {}, body: nil, request_options: {})
+          super(base_url:, path:, method:, headers:, query:, request_options:)
+
+          @body = body
+        end
+
+        # @return [Hash] The encoded HTTP request headers.
+        # @param protected_keys [Array<String>] Header keys set by the SDK client (e.g. auth, metadata)
+        #   that must not be overridden by additional_headers from request_options.
+        def encode_headers(protected_keys: [])
+          sdk_headers = {
+            "Content-Type" => "application/json",
+            "Accept" => "application/json"
+          }.merge(@headers)
+          merge_additional_headers(sdk_headers, protected_keys:)
+        end
+
+        # @return [String, nil] The encoded HTTP request body.
+        def encode_body
+          @body.nil? ? nil : ::JSON.generate(@body)
+        end
+      end
+    end
+  end
+end

data/lib/Berrycrawl/internal/json/serializable.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Berrycrawl
+  module Internal
+    module JSON
+      module Serializable
+        # Loads data from JSON into its deserialized form
+        #
+        # @param str [String] Raw JSON to load into an object
+        # @return [Object]
+        def load(str)
+          raise NotImplementedError
+        end
+
+        # Dumps data from its deserialized form into JSON
+        #
+        # @param value [Object] The deserialized value
+        # @return [String]
+        def dump(value)
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end