berrycrawl 0.1.0

data/lib/Berrycrawl/client.rb

@@ -0,0 +1,268 @@
+# frozen_string_literal: true
+
+module Berrycrawl
+  class Client
+    # @param request_options [Hash]
+    # @param params [Berrycrawl::Types::CrawlDto]
+    # @option request_options [String] :base_url
+    # @option request_options [Hash{String => Object}] :additional_headers
+    # @option request_options [Hash{String => Object}] :additional_query_parameters
+    # @option request_options [Hash{String => Object}] :additional_body_parameters
+    # @option request_options [Integer] :timeout_in_seconds
+    #
+    # @return [Berrycrawl::Types::JobCreatedResponse]
+    def crawl(request_options: {}, **params)
+      params = Berrycrawl::Internal::Types::Utils.normalize_keys(params)
+      request = Berrycrawl::Internal::JSON::Request.new(
+        base_url: request_options[:base_url],
+        method: "POST",
+        path: "crawl",
+        body: Berrycrawl::Types::CrawlDto.new(params).to_h,
+        request_options: request_options
+      )
+      begin
+        response = @client.send(request)
+      rescue Net::HTTPRequestTimeout
+        raise Berrycrawl::Errors::TimeoutError
+      end
+      code = response.code.to_i
+      if code.between?(200, 299)
+        Berrycrawl::Types::JobCreatedResponse.load(response.body)
+      else
+        error_class = Berrycrawl::Errors::ResponseError.subclass_for_code(code)
+        raise error_class.new(response.body, code: code)
+      end
+    end
+
+    # @param request_options [Hash]
+    # @param params [Berrycrawl::Types::ExtractDto]
+    # @option request_options [String] :base_url
+    # @option request_options [Hash{String => Object}] :additional_headers
+    # @option request_options [Hash{String => Object}] :additional_query_parameters
+    # @option request_options [Hash{String => Object}] :additional_body_parameters
+    # @option request_options [Integer] :timeout_in_seconds
+    #
+    # @return [Berrycrawl::Types::JobCreatedResponse]
+    def extract(request_options: {}, **params)
+      params = Berrycrawl::Internal::Types::Utils.normalize_keys(params)
+      request = Berrycrawl::Internal::JSON::Request.new(
+        base_url: request_options[:base_url],
+        method: "POST",
+        path: "extract",
+        body: Berrycrawl::Types::ExtractDto.new(params).to_h,
+        request_options: request_options
+      )
+      begin
+        response = @client.send(request)
+      rescue Net::HTTPRequestTimeout
+        raise Berrycrawl::Errors::TimeoutError
+      end
+      code = response.code.to_i
+      if code.between?(200, 299)
+        Berrycrawl::Types::JobCreatedResponse.load(response.body)
+      else
+        error_class = Berrycrawl::Errors::ResponseError.subclass_for_code(code)
+        raise error_class.new(response.body, code: code)
+      end
+    end
+
+    # @param request_options [Hash]
+    # @param params [Berrycrawl::Types::MapDto]
+    # @option request_options [String] :base_url
+    # @option request_options [Hash{String => Object}] :additional_headers
+    # @option request_options [Hash{String => Object}] :additional_query_parameters
+    # @option request_options [Hash{String => Object}] :additional_body_parameters
+    # @option request_options [Integer] :timeout_in_seconds
+    #
+    # @return [Berrycrawl::Types::MapResponse]
+    def map(request_options: {}, **params)
+      params = Berrycrawl::Internal::Types::Utils.normalize_keys(params)
+      request = Berrycrawl::Internal::JSON::Request.new(
+        base_url: request_options[:base_url],
+        method: "POST",
+        path: "map",
+        body: Berrycrawl::Types::MapDto.new(params).to_h,
+        request_options: request_options
+      )
+      begin
+        response = @client.send(request)
+      rescue Net::HTTPRequestTimeout
+        raise Berrycrawl::Errors::TimeoutError
+      end
+      code = response.code.to_i
+      if code.between?(200, 299)
+        Berrycrawl::Types::MapResponse.load(response.body)
+      else
+        error_class = Berrycrawl::Errors::ResponseError.subclass_for_code(code)
+        raise error_class.new(response.body, code: code)
+      end
+    end
+
+    # @param request_options [Hash]
+    # @param params [Berrycrawl::Types::ParseDto]
+    # @option request_options [String] :base_url
+    # @option request_options [Hash{String => Object}] :additional_headers
+    # @option request_options [Hash{String => Object}] :additional_query_parameters
+    # @option request_options [Hash{String => Object}] :additional_body_parameters
+    # @option request_options [Integer] :timeout_in_seconds
+    #
+    # @return [Berrycrawl::Types::ScrapeResponse]
+    def parse(request_options: {}, **params)
+      params = Berrycrawl::Internal::Types::Utils.normalize_keys(params)
+      request = Berrycrawl::Internal::JSON::Request.new(
+        base_url: request_options[:base_url],
+        method: "POST",
+        path: "parse",
+        body: Berrycrawl::Types::ParseDto.new(params).to_h,
+        request_options: request_options
+      )
+      begin
+        response = @client.send(request)
+      rescue Net::HTTPRequestTimeout
+        raise Berrycrawl::Errors::TimeoutError
+      end
+      code = response.code.to_i
+      if code.between?(200, 299)
+        Berrycrawl::Types::ScrapeResponse.load(response.body)
+      else
+        error_class = Berrycrawl::Errors::ResponseError.subclass_for_code(code)
+        raise error_class.new(response.body, code: code)
+      end
+    end
+
+    # @param request_options [Hash]
+    # @param params [Berrycrawl::Types::ScrapeDto]
+    # @option request_options [String] :base_url
+    # @option request_options [Hash{String => Object}] :additional_headers
+    # @option request_options [Hash{String => Object}] :additional_query_parameters
+    # @option request_options [Hash{String => Object}] :additional_body_parameters
+    # @option request_options [Integer] :timeout_in_seconds
+    #
+    # @return [Berrycrawl::Types::ScrapeResponse]
+    def scrape(request_options: {}, **params)
+      params = Berrycrawl::Internal::Types::Utils.normalize_keys(params)
+      request = Berrycrawl::Internal::JSON::Request.new(
+        base_url: request_options[:base_url],
+        method: "POST",
+        path: "scrape",
+        body: Berrycrawl::Types::ScrapeDto.new(params).to_h,
+        request_options: request_options
+      )
+      begin
+        response = @client.send(request)
+      rescue Net::HTTPRequestTimeout
+        raise Berrycrawl::Errors::TimeoutError
+      end
+      code = response.code.to_i
+      if code.between?(200, 299)
+        Berrycrawl::Types::ScrapeResponse.load(response.body)
+      else
+        error_class = Berrycrawl::Errors::ResponseError.subclass_for_code(code)
+        raise error_class.new(response.body, code: code)
+      end
+    end
+
+    # Cookie banners, blocking overlays, chat widgets, and lazy loading are handled by default. Every cleanup pass can
+    # be controlled per request.
+    #
+    # @param request_options [Hash]
+    # @param params [Berrycrawl::Types::ScreenshotDto]
+    # @option request_options [String] :base_url
+    # @option request_options [Hash{String => Object}] :additional_headers
+    # @option request_options [Hash{String => Object}] :additional_query_parameters
+    # @option request_options [Hash{String => Object}] :additional_body_parameters
+    # @option request_options [Integer] :timeout_in_seconds
+    #
+    # @return [Berrycrawl::Types::ScrapeResponse]
+    def screenshot(request_options: {}, **params)
+      params = Berrycrawl::Internal::Types::Utils.normalize_keys(params)
+      request = Berrycrawl::Internal::JSON::Request.new(
+        base_url: request_options[:base_url],
+        method: "POST",
+        path: "screenshot",
+        body: Berrycrawl::Types::ScreenshotDto.new(params).to_h,
+        request_options: request_options
+      )
+      begin
+        response = @client.send(request)
+      rescue Net::HTTPRequestTimeout
+        raise Berrycrawl::Errors::TimeoutError
+      end
+      code = response.code.to_i
+      if code.between?(200, 299)
+        Berrycrawl::Types::ScrapeResponse.load(response.body)
+      else
+        error_class = Berrycrawl::Errors::ResponseError.subclass_for_code(code)
+        raise error_class.new(response.body, code: code)
+      end
+    end
+
+    # @param request_options [Hash]
+    # @param params [Berrycrawl::Types::SearchDto]
+    # @option request_options [String] :base_url
+    # @option request_options [Hash{String => Object}] :additional_headers
+    # @option request_options [Hash{String => Object}] :additional_query_parameters
+    # @option request_options [Hash{String => Object}] :additional_body_parameters
+    # @option request_options [Integer] :timeout_in_seconds
+    #
+    # @return [Berrycrawl::Types::SearchResponse]
+    def search(request_options: {}, **params)
+      params = Berrycrawl::Internal::Types::Utils.normalize_keys(params)
+      request = Berrycrawl::Internal::JSON::Request.new(
+        base_url: request_options[:base_url],
+        method: "POST",
+        path: "search",
+        body: Berrycrawl::Types::SearchDto.new(params).to_h,
+        request_options: request_options
+      )
+      begin
+        response = @client.send(request)
+      rescue Net::HTTPRequestTimeout
+        raise Berrycrawl::Errors::TimeoutError
+      end
+      code = response.code.to_i
+      if code.between?(200, 299)
+        Berrycrawl::Types::SearchResponse.load(response.body)
+      else
+        error_class = Berrycrawl::Errors::ResponseError.subclass_for_code(code)
+        raise error_class.new(response.body, code: code)
+      end
+    end
+
+    # @param base_url [String, nil]
+    # @param token [String]
+    # @param max_retries [Integer]
+    #
+    # @return [void]
+    def initialize(base_url: nil, token: ENV.fetch("BERRYCRAWL_API_KEY", nil), max_retries: 2)
+      @raw_client = Berrycrawl::Internal::Http::RawClient.new(
+        base_url: base_url || Berrycrawl::Environment::PRODUCTION,
+        headers: {
+          "X-Fern-Language" => "Ruby",
+          Authorization: "Bearer #{token}"
+        },
+        max_retries: max_retries
+      )
+    end
+
+    # @return [Berrycrawl::Account::Client]
+    def account
+      @account ||= Berrycrawl::Account::Client.new(client: @raw_client)
+    end
+
+    # @return [Berrycrawl::Brand::Client]
+    def brand
+      @brand ||= Berrycrawl::Brand::Client.new(client: @raw_client)
+    end
+
+    # @return [Berrycrawl::Jobs::Client]
+    def jobs
+      @jobs ||= Berrycrawl::Jobs::Client.new(client: @raw_client)
+    end
+
+    # @return [Berrycrawl::Webhooks::Client]
+    def webhooks
+      @webhooks ||= Berrycrawl::Webhooks::Client.new(client: @raw_client)
+    end
+  end
+end

data/lib/Berrycrawl/environment.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Berrycrawl
+  class Environment
+    PRODUCTION = "https://api.berrycrawl.com/api/v1"
+  end
+end

data/lib/Berrycrawl/errors/api_error.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Berrycrawl
+  module Errors
+    class ApiError < StandardError
+    end
+  end
+end

data/lib/Berrycrawl/errors/client_error.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Berrycrawl
+  module Errors
+    class ClientError < ResponseError
+    end
+
+    class UnauthorizedError < ClientError
+    end
+
+    class ForbiddenError < ClientError
+    end
+
+    class NotFoundError < ClientError
+    end
+  end
+end

data/lib/Berrycrawl/errors/redirect_error.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Berrycrawl
+  module Errors
+    class RedirectError < ResponseError
+    end
+  end
+end

data/lib/Berrycrawl/errors/response_error.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Berrycrawl
+  module Errors
+    class ResponseError < ApiError
+      attr_reader :code
+
+      def initialize(msg, code:)
+        @code = code
+        super(msg)
+      end
+
+      def inspect
+        "#<#{self.class.name} @code=#{code} @body=#{message}>"
+      end
+
+      # Returns the most appropriate error class for the given code.
+      #
+      # @return [Class]
+      def self.subclass_for_code(code)
+        case code
+        when 300..399
+          RedirectError
+        when 401
+          UnauthorizedError
+        when 403
+          ForbiddenError
+        when 404
+          NotFoundError
+        when 400..499
+          ClientError
+        when 503
+          ServiceUnavailableError
+        when 500..599
+          ServerError
+        else
+          ResponseError
+        end
+      end
+    end
+  end
+end

data/lib/Berrycrawl/errors/server_error.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Berrycrawl
+  module Errors
+    class ServerError < ResponseError
+    end
+
+    class ServiceUnavailableError < ApiError
+    end
+  end
+end

data/lib/Berrycrawl/errors/timeout_error.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Berrycrawl
+  module Errors
+    class TimeoutError < ApiError
+    end
+  end
+end

data/lib/Berrycrawl/internal/errors/constraint_error.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Berrycrawl
+  module Internal
+    module Errors
+      class ConstraintError < StandardError
+      end
+    end
+  end
+end

data/lib/Berrycrawl/internal/errors/type_error.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Berrycrawl
+  module Internal
+    module Errors
+      class TypeError < StandardError
+      end
+    end
+  end
+end

data/lib/Berrycrawl/internal/http/base_request.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Berrycrawl
+  module Internal
+    module Http
+      # @api private
+      class BaseRequest
+        attr_reader :base_url, :path, :method, :headers, :query, :request_options
+
+        # @param base_url [String] The base URL for the request
+        # @param path [String] The path for the request
+        # @param method [String] The HTTP method for the request (:get, :post, etc.)
+        # @param headers [Hash] Additional headers for the request (optional)
+        # @param query [Hash] Query parameters for the request (optional)
+        # @param request_options [Berrycrawl::RequestOptions, Hash{Symbol=>Object}, nil]
+        def initialize(base_url:, path:, method:, headers: {}, query: {}, request_options: {})
+          @base_url = base_url
+          @path = path
+          @method = method
+          @headers = headers
+          @query = query
+          @request_options = request_options
+        end
+
+        # @return [Hash] The query parameters merged with additional query parameters from request options.
+        def encode_query
+          additional_query = @request_options&.dig(:additional_query_parameters) || @request_options&.dig("additional_query_parameters") || {}
+          @query.merge(additional_query)
+        end
+
+        # Child classes should implement:
+        # - encode_headers: Returns the encoded HTTP request headers.
+        # - encode_body: Returns the encoded HTTP request body.
+
+        private
+
+        # Merges additional_headers from request_options into sdk_headers, filtering out
+        # any keys that collide with SDK-set or client-protected headers (case-insensitive).
+        # @param sdk_headers [Hash] Headers set by the SDK for this request type.
+        # @param protected_keys [Array<String>] Additional header keys that must not be overridden.
+        # @return [Hash] The merged headers.
+        def merge_additional_headers(sdk_headers, protected_keys: [])
+          additional_headers = @request_options&.dig(:additional_headers) || @request_options&.dig("additional_headers") || {}
+          all_protected = (sdk_headers.keys + protected_keys).to_set { |k| k.to_s.downcase }
+          filtered = additional_headers.reject { |key, _| all_protected.include?(key.to_s.downcase) }
+          sdk_headers.merge(filtered)
+        end
+      end
+    end
+  end
+end