ghostcrawl 2.2.0

data/lib/ghostcrawl/error_codes.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+# Mirrors ghostcrawl/errors/codes.json (the single source of truth). Do not hand-diverge the set.
+#
+# Two channels:
+#   PROBLEM — OUR failure  -> non-2xx application/problem+json
+#   RESULT  — TARGET failure -> HTTP 200 + result_error{code,retryable}
+module Ghostcrawl
+  # Canonical error-code constants + lookup tables. The 23 codes, their HTTP
+  # status, retryability, and channel mirror the server-side catalog so that
+  # every typed error the SDK raises carries a stable, documented +code+.
+  module ErrorCodes
+    # --- PROBLEM channel (our failure -> non-2xx) ----------------------------
+    BAD_REQUEST                = "bad_request"
+    UNAUTHORIZED               = "unauthorized"
+    FORBIDDEN                  = "forbidden"
+    PAYMENT_REQUIRED           = "payment_required"
+    NOT_FOUND                  = "not_found"
+    CONFLICT                   = "conflict"
+    BYO_PROXY_INVALID          = "byo_proxy_invalid"
+    TIER_UNAVAILABLE           = "tier_unavailable"
+    RATE_LIMITED               = "rate_limited"
+    QUOTA_BACKEND_UNAVAILABLE  = "quota_backend_unavailable"
+    POOL_EXHAUSTED             = "pool_exhausted"
+    EGRESS_INTEGRITY_FAILED    = "egress_integrity_failed"
+    RENDER_HUNG                = "render_hung"
+    ENGINE_CRASHED             = "engine_crashed"
+    RENDER_TIMEOUT             = "render_timeout"
+    FLEET_TIMEOUT              = "fleet_timeout"
+    SERVICE_UNAVAILABLE        = "service_unavailable"
+    INTERNAL_ERROR             = "internal_error"
+
+    # --- RESULT channel (target failure -> HTTP 200 + result_error) ----------
+    TARGET_HTTP_ERROR          = "target_http_error"
+    NAVIGATION_FAILED          = "navigation_failed"
+    BLOCKED                    = "blocked"
+    CAPTCHA_REQUIRED           = "captcha_required"
+    EMPTY_CONTENT              = "empty_content"
+
+    # Full catalog: code => {http:, retryable:, channel:}. Mirrors codes.json.
+    CATALOG = {
+      BAD_REQUEST               => { http: 400, retryable: false, channel: "problem" },
+      UNAUTHORIZED              => { http: 401, retryable: false, channel: "problem" },
+      FORBIDDEN                 => { http: 403, retryable: false, channel: "problem" },
+      PAYMENT_REQUIRED          => { http: 402, retryable: false, channel: "problem" },
+      NOT_FOUND                 => { http: 404, retryable: false, channel: "problem" },
+      CONFLICT                  => { http: 409, retryable: false, channel: "problem" },
+      BYO_PROXY_INVALID         => { http: 422, retryable: false, channel: "problem" },
+      TIER_UNAVAILABLE          => { http: 400, retryable: false, channel: "problem" },
+      RATE_LIMITED              => { http: 429, retryable: true,  channel: "problem" },
+      QUOTA_BACKEND_UNAVAILABLE => { http: 503, retryable: true,  channel: "problem" },
+      POOL_EXHAUSTED            => { http: 503, retryable: true,  channel: "problem" },
+      EGRESS_INTEGRITY_FAILED   => { http: 503, retryable: true,  channel: "problem" },
+      RENDER_HUNG               => { http: 503, retryable: true,  channel: "problem" },
+      ENGINE_CRASHED            => { http: 503, retryable: true,  channel: "problem" },
+      RENDER_TIMEOUT            => { http: 504, retryable: true,  channel: "problem" },
+      FLEET_TIMEOUT             => { http: 504, retryable: true,  channel: "problem" },
+      SERVICE_UNAVAILABLE       => { http: 503, retryable: true,  channel: "problem" },
+      INTERNAL_ERROR            => { http: 500, retryable: true,  channel: "problem" },
+      TARGET_HTTP_ERROR         => { http: 200, retryable: false, channel: "result" },
+      NAVIGATION_FAILED         => { http: 200, retryable: false, channel: "result" },
+      BLOCKED                   => { http: 200, retryable: true,  channel: "result" },
+      CAPTCHA_REQUIRED          => { http: 200, retryable: true,  channel: "result" },
+      EMPTY_CONTENT             => { http: 200, retryable: false, channel: "result" }
+    }.freeze
+
+    # code => retryable? boolean lookup, derived from {CATALOG}.
+    RETRYABLE = CATALOG.transform_values { |meta| meta[:retryable] }.freeze
+
+    # Codes carried on a successful (HTTP 200) response that signal a target-side
+    # failure — the RESULT channel. A top-level +code+ is only treated as an
+    # error when it appears in this set.
+    RESULT_CODES = CATALOG.select { |_code, meta| meta[:channel] == "result" }.keys.freeze
+
+    # Whether +code+ is a known RESULT-channel (HTTP-200 target-failure) code.
+    # @param code [String, nil]
+    # @return [Boolean]
+    def self.result_channel?(code)
+      RESULT_CODES.include?(code)
+    end
+
+    # Fallback status -> canonical code map, used when the response body carries
+    # no +code+ (the common Kiota case where the body is dropped before the
+    # adapter raises). Keeps every typed error keyed to a canonical code.
+    # @param status [Integer] HTTP status code
+    # @return [String] canonical error code
+    def self.code_for_status(status)
+      case status
+      when 400 then BAD_REQUEST
+      when 401 then UNAUTHORIZED
+      when 402 then PAYMENT_REQUIRED
+      when 403 then FORBIDDEN
+      when 404 then NOT_FOUND
+      when 409 then CONFLICT
+      when 422 then BYO_PROXY_INVALID
+      when 429 then RATE_LIMITED
+      when 503 then SERVICE_UNAVAILABLE
+      when 504 then FLEET_TIMEOUT
+      else
+        status.to_i >= 500 ? INTERNAL_ERROR : BAD_REQUEST
+      end
+    end
+  end
+end

data/lib/ghostcrawl/errors.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "error_codes"
+
+module Ghostcrawl
+  # Base error for all GhostCrawl API errors.
+  class GhostcrawlError < StandardError
+    # @return [Integer] HTTP status code
+    attr_reader :status_code
+    # @return [String] raw response body
+    attr_reader :body
+    # @return [String, nil] canonical error code (see {Ghostcrawl::ErrorCodes})
+    attr_reader :code
+    # @return [Boolean] whether the operation is safe to retry
+    attr_reader :retryable
+    # @return [String, nil] request id (the problem+json +instance+) for support
+    attr_reader :request_id
+
+    def initialize(message, status_code: 0, body: nil, code: nil, retryable: false, request_id: nil)
+      super(message)
+      @status_code = status_code
+      @body = body
+      @code = code
+      @retryable = retryable
+      @request_id = request_id
+    end
+  end
+
+  # Raised on 401 — missing or invalid API key.
+  class AuthenticationError < GhostcrawlError; end
+
+  # Raised on 402 — usage or spend limit reached.
+  class PaymentRequiredError < GhostcrawlError; end
+
+  # Raised on 422 — bad request parameters.
+  class InvalidRequestError < GhostcrawlError; end
+
+  # Raised on 429 — rate limit reached.
+  class RateLimitError < GhostcrawlError; end
+
+  # Raised on 5xx server errors.
+  class APIError < GhostcrawlError; end
+
+  # Raised when a request succeeds at the transport layer (HTTP 200) but the
+  # target page could not be scraped — the RESULT channel. Carries the canonical
+  # result-error +code+ (e.g. "blocked", "captcha_required") plus the target's
+  # own HTTP status when one is available.
+  class ScrapeError < GhostcrawlError
+    # @return [Integer, nil] the target site's HTTP status, when reported
+    attr_reader :target_status
+
+    def initialize(message, status_code: 200, body: nil, code: nil, retryable: false,
+                   request_id: nil, target_status: nil)
+      super(message, status_code: status_code, body: body, code: code,
+                     retryable: retryable, request_id: request_id)
+      @target_status = target_status
+    end
+  end
+
+  # Translates a low-level transport exception (raised by the Kiota Faraday
+  # adapter) into a typed {GhostcrawlError}. The adapter raises a bare
+  # +MicrosoftKiotaAbstractions::ApiError+ whose message embeds the HTTP status
+  # code (e.g. "...status code...:402"), and a typed +HTTPValidationError+ for
+  # 422 responses. Neither subclasses {GhostcrawlError}, so the documented
+  # +rescue Ghostcrawl::GhostcrawlError+ contract would otherwise never fire.
+  #
+  # @param err [StandardError] the original transport exception
+  # @return [GhostcrawlError] never returns normally — always raises
+  # @api private
+  def self.raise_translated(err)
+    # A typed HTTPValidationError is the adapter's 422 mapping — it carries no
+    # ":<code>" in its message, so key off the class name.
+    validation = err.class.name.to_s.include?("HTTPValidationError")
+    status = validation ? 422 : extract_status(err)
+    body = err.message
+
+    # The problem+json body is frequently dropped by the Kiota runtime before it
+    # raises (no spec error-mapping for the status). Try to read a canonical
+    # +code+ and +instance+ (request id) from whatever message/body text exists;
+    # otherwise fall back to a status -> code map so the caller still gets a
+    # canonical, documented code + retryability keyed off the status line.
+    problem = parse_problem_body(body)
+    code = problem["code"] || ErrorCodes.code_for_status(status)
+    request_id = problem["instance"]
+    retryable = ErrorCodes::RETRYABLE.fetch(code, false)
+
+    # Keep the existing status -> class selection for back-compat; just enrich
+    # the raised instance with the canonical code/retryable/request_id.
+    klass =
+      case status
+      when 401, 403 then AuthenticationError
+      when 402      then PaymentRequiredError
+      when 400, 422 then InvalidRequestError
+      when 429      then RateLimitError
+      when 500..599 then APIError
+      else GhostcrawlError
+      end
+    raise klass.new(friendly_message(status, err), status_code: status, body: body,
+                                                   code: code, retryable: retryable,
+                                                   request_id: request_id)
+  end
+
+  # Best-effort parse of a problem+json body/message string. Returns the parsed
+  # Hash when the text is a JSON object, or an empty Hash otherwise — never
+  # raises, so callers can read +["code"]+/+["instance"]+ unconditionally.
+  # @api private
+  def self.parse_problem_body(body)
+    return {} if body.nil?
+
+    parsed = JSON.parse(body.to_s)
+    parsed.is_a?(Hash) ? parsed : {}
+  rescue StandardError
+    {}
+  end
+
+  # @api private
+  def self.extract_status(err)
+    # The Faraday adapter appends ":<code>" to its ApiError message.
+    m = err.message.to_s.match(/:(\d{3})\b/)
+    m ? m[1].to_i : 0
+  end
+
+  # @api private
+  def self.friendly_message(status, err)
+    return "validation error (HTTP 422)" if err.class.name.to_s.include?("HTTPValidationError")
+
+    case status
+    when 401, 403 then "authentication failed — check your API key (HTTP #{status})"
+    when 402      then "payment required — usage or spend limit reached (HTTP 402)"
+    when 400, 422 then "invalid request parameters (HTTP #{status})"
+    when 429      then "rate limit reached — retry after a short delay (HTTP 429)"
+    when 500..599 then "server error (HTTP #{status})"
+    else err.message.to_s
+    end
+  end
+end

data/lib/ghostcrawl/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Ghostcrawl
+  VERSION = "2.2.0"
+end

data/lib/ghostcrawl.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require_relative "ghostcrawl/version"
+require_relative "ghostcrawl/error_codes"
+require_relative "ghostcrawl/errors"
+require_relative "ghostcrawl/client"
+
+# GhostCrawl SDK — collect web data at scale.
+module Ghostcrawl
+  # Convenience constructor.
+  # @param token [String, nil] API key
+  # @param base_url [String, nil] optional base URL override
+  # @return [Client]
+  def self.new(token: nil, base_url: nil)
+    Client.new(token: token, base_url: base_url)
+  end
+end