ghostcrawl 2.2.0

data/lib/ghostcrawl/client.rb

@@ -0,0 +1,761 @@
+# frozen_string_literal: true
+
+# GhostCrawl Ruby SDK — idiomatic facade over the Kiota-generated core.
+#
+# Architecture:
+#
+#   _generated/ Kiota core  — spec-faithful 98-op request-builder (models, transport, auth)
+#   This FACADE              — thin idiomatic layer delegating to the generated builders
+#
+# All HTTP transport, URL routing, serialization, and auth come from the generated core.
+# The facade maps idiomatic calls (client.scrape) to generated builders via
+# BaseBearerTokenAuthenticationProvider + FaradayRequestAdapter.
+#
+# Usage:
+#   require "ghostcrawl"
+#   client = Ghostcrawl::Client.new(token: "gck_live_YOUR_KEY")
+#   result = client.scrape(url: "https://example.com")
+
+require "microsoft_kiota_abstractions"
+require "microsoft_kiota_faraday"
+require "microsoft_kiota_serialization_json"
+require_relative "../../_generated/ghostcrawl_client"
+require_relative "../../_generated/models/scrape_request"
+require_relative "../../_generated/models/search_request"
+require_relative "../../_generated/models/extract_request"
+require_relative "../../_generated/models/deep_crawl_body"
+require_relative "../../_generated/models/map_body"
+require_relative "../../_generated/models/session_create_request"
+require_relative "../../_generated/models/extend_body"
+require_relative "../../_generated/models/profile_create_request"
+require_relative "../../_generated/models/profile_update_request"
+require_relative "../../_generated/models/webhook_create_request"
+require_relative "../../_generated/models/schedule_create_request"
+require_relative "errors"
+
+module Ghostcrawl
+  # ---------------------------------------------------------------------------
+  # Fix: microsoft_kiota_serialization_json 0.9.2 has a bug where
+  # write_object_value(nil, body) creates a temp writer, serializes body into
+  # temp, then RETURNS temp without merging temp's content into self.@writer.
+  # set_content_from_parsable calls write_object_value(nil, body) and discards
+  # the return value, so @content = {}.to_json = "{}".
+  #
+  # Fix: monkey-patch write_object_value so that when key is nil, it serializes
+  # body into self directly (not a temp writer) — same as what the non-nil branch
+  # does, but without a key.
+  # ---------------------------------------------------------------------------
+
+  # @api private
+  module KiotaWriterFix
+    def write_object_value(key, value)
+      return unless value
+      if key.nil?
+        # Fix: serialize into self, not a temp. Merges all fields into @writer.
+        value.serialize(self)
+      else
+        super
+      end
+    end
+  end
+
+  # Patch the writer class once, idempotently
+  MicrosoftKiotaSerializationJson::JsonSerializationWriter.prepend(KiotaWriterFix) \
+    unless MicrosoftKiotaSerializationJson::JsonSerializationWriter.ancestors.include?(KiotaWriterFix)
+
+  # ---------------------------------------------------------------------------
+  # Fix 2: Some generated request builders pass a Module (e.g. `Binary`) as
+  # the factory to `send_async`. The parse node's `get_object_value(factory)`
+  # calls `factory.call(self)`, which fails for a Module.
+  #
+  # Fix: when factory doesn't respond to `call`, fall back to parsing the
+  # response body as JSON and returning the raw hash. This gives us plain
+  # Ruby hashes (which ResponseHelper.to_hash handles correctly) without
+  # requiring generated response models.
+  # ---------------------------------------------------------------------------
+
+  # @api private
+  module KiotaParseNodeFix
+    def get_object_value(factory)
+      return super if factory.respond_to?(:call)
+      # Fallback: return the raw parsed node for non-callable factories (e.g. Binary).
+      # @current_node is already a Ruby Hash/Array/primitive parsed from JSON.
+      @current_node
+    rescue StandardError
+      super
+    end
+  end
+
+  MicrosoftKiotaSerializationJson::JsonParseNode.prepend(KiotaParseNodeFix) \
+    unless MicrosoftKiotaSerializationJson::JsonParseNode.ancestors.include?(KiotaParseNodeFix)
+
+  # ---------------------------------------------------------------------------
+  # AdditionalDataBody — a minimal Parsable that wraps an arbitrary Hash.
+  #
+  # Used for all POST bodies. The generated models set typed defaults in their
+  # constructors that would be serialized as null/empty-enum values and cause
+  # 422 validation errors. AdditionalDataBody only serializes the fields we
+  # explicitly pass, producing clean JSON.
+  #
+  # Depends on KiotaWriterFix above to correctly handle the nil-key
+  # write_object_value(nil, body) call from set_content_from_parsable.
+  # ---------------------------------------------------------------------------
+
+  # @api private
+  class AdditionalDataBody
+    include MicrosoftKiotaAbstractions::Parsable
+
+    def initialize(data = {})
+      @data = data.transform_keys(&:to_s)
+    end
+
+    def get_field_deserializers
+      {}
+    end
+
+    def serialize(writer)
+      writer.write_additional_data(@data)
+    end
+
+    def additional_data
+      @data
+    end
+
+    def additional_data=(hash)
+      @data = hash.transform_keys(&:to_s)
+    end
+
+    def self.create_from_discriminator_value(_parse_node)
+      AdditionalDataBody.new
+    end
+  end
+end
+
+module Ghostcrawl
+  DEFAULT_BASE_URL = "https://api.ghostcrawl.io"
+
+  # ---------------------------------------------------------------------------
+  # Static bearer token provider — implements AccessTokenProvider
+  # ---------------------------------------------------------------------------
+
+  # @api private
+  class StaticTokenProvider
+    include MicrosoftKiotaAbstractions::AccessTokenProvider
+
+    def initialize(token)
+      @token = token
+    end
+
+    # The Kiota Faraday adapter calls .resume on the return value of this method,
+    # expecting a Fiber. Wrap the token in a Fiber to satisfy the contract.
+    def get_authorization_token(_uri, _additional_authentication_context = nil)
+      token = @token
+      Fiber.new { token }
+    end
+
+    def get_allowed_hosts_validator
+      MicrosoftKiotaAbstractions::AllowedHostsValidator.new([])
+    end
+  end
+
+  # ---------------------------------------------------------------------------
+  # Response helpers — convert Kiota Parsable/Hash responses to plain Hash
+  # ---------------------------------------------------------------------------
+
+  # @api private
+  module ResponseHelper
+    # Converts any Kiota response value to a plain Hash or Array.
+    # The Kiota Faraday adapter returns Fibers for async responses; call .resume
+    # to execute the request synchronously and get the actual response value.
+    def self.to_hash(value)
+      # Resolve Fibers (the Kiota Faraday adapter returns a Fiber for every
+      # request; .resume is what actually executes the HTTP call). Any non-2xx
+      # surfaces here as a raw transport exception — translate it into a typed
+      # Ghostcrawl error so the documented rescue contract works.
+      if value.is_a?(Fiber)
+        begin
+          value = value.resume
+        rescue Ghostcrawl::GhostcrawlError
+          raise
+        rescue StandardError => e
+          Ghostcrawl.raise_translated(e)
+        end
+      end
+
+      case value
+      when Hash
+        value.transform_values { |v| to_hash(v) }
+      when Array
+        value.map { |v| to_hash(v) }
+      when NilClass
+        {}
+      else
+        # Typed Parsable: try serializing to JSON then parsing back
+        if value.respond_to?(:additional_data) && value.additional_data
+          value.additional_data.transform_values { |v| to_hash(v) }
+        elsif value.respond_to?(:to_h)
+          value.to_h.transform_values { |v| to_hash(v) }
+        else
+          value
+        end
+      end
+    end
+
+    # Inspects a decoded HTTP-200 response hash for a RESULT-channel failure (the
+    # target page could not be scraped) and raises {Ghostcrawl::ScrapeError} when
+    # one is present. This is the reliable, highest-value error path: the body is
+    # always available here (unlike the dropped problem+json body on non-2xx).
+    #
+    # A failure is signalled by any of:
+    #   * a +result_error+ Hash carrying a +code+
+    #   * +ok+ explicitly +false+ (always a failure, even with no code)
+    #   * a top-level +code+ that is a known RESULT-channel code
+    #
+    # A genuinely OK hash (ok: true, no +ok+ key, or no error code) is returned
+    # untouched and never raises.
+    #
+    # @param hash [Hash] the decoded response
+    # @return [Hash] the same hash, when it is not a failure
+    # @api private
+    def self.raise_on_result_error!(hash)
+      return hash unless hash.is_a?(Hash)
+
+      # Descend into a `results` envelope (scrape/extract wrap per-URL results) —
+      # the target failure lives on the INNER result, not the envelope top level.
+      inner = hash["results"]
+      if inner.is_a?(Array)
+        inner.each { |item| raise_on_result_error!(item) }
+        return hash
+      end
+
+      result_error = hash["result_error"]
+      result_error = nil unless result_error.is_a?(Hash)
+      top_code = hash["code"]
+      ok_false = hash["ok"] == false
+
+      # Pull the code: result_error wins, then a top-level RESULT-channel code.
+      code = nil
+      code = result_error["code"] if result_error
+      code ||= top_code if Ghostcrawl::ErrorCodes.result_channel?(top_code)
+
+      # The flat markdown-build envelope reports a target failure ONLY via
+      # status="failed" (no ok/result_error) — don't count it as a success.
+      status_failed = hash["status"] == "failed"
+      code ||= top_code if status_failed && top_code.is_a?(String)
+
+      # Only raise when there is a concrete result-channel failure signal.
+      return hash unless code || ok_false || status_failed
+
+      # ok: false with no usable code -> treat as empty/unusable content.
+      code ||= Ghostcrawl::ErrorCodes::EMPTY_CONTENT
+
+      retryable =
+        if result_error && result_error.key?("retryable")
+          result_error["retryable"]
+        else
+          Ghostcrawl::ErrorCodes::RETRYABLE.fetch(code, false)
+        end
+
+      target_status = nil
+      target_status = result_error["target_status"] if result_error
+      target_status ||= hash["target_status"]  # flat markdown-envelope path
+      reason = result_error && result_error["reason"]
+
+      msg = "scrape failed (#{code})"
+      msg += ": #{reason}" if reason && !reason.to_s.empty?
+      msg += " (target HTTP #{target_status})" if target_status
+
+      raise Ghostcrawl::ScrapeError.new(
+        msg,
+        status_code: 200,
+        body: nil,
+        code: code,
+        retryable: retryable,
+        request_id: hash["request_id"],
+        target_status: target_status
+      )
+    end
+  end
+
+  # ---------------------------------------------------------------------------
+  # Sub-clients — each delegates to the generated v1 request builders
+  # ---------------------------------------------------------------------------
+
+  # Manage crawl runs — /v1/crawl-runs.
+  class CrawlRunsClient
+    def initialize(v1)
+      @v1 = v1
+    end
+
+    # Start a new crawl run from a seed URL.
+    # Delegates to POST /v1/crawl-runs via the generated CrawlRunsRequestBuilder.
+    # The endpoint is a tagged union: a start request requires +action: "start"+
+    # and a +seed_urls+ array (not a bare +url+).
+    # @param url [String] seed URL
+    # @param max_depth [Integer] maximum crawl depth (default 2)
+    # @param max_pages [Integer] maximum pages (default 100)
+    # @param raise_on_result_error [Boolean] raise {Ghostcrawl::ScrapeError} on a
+    #   target-side (HTTP-200) failure instead of returning the raw hash (default true)
+    # @return [Hash] crawl run record with +run_id+ and +status+
+    def start(url:, max_depth: 2, max_pages: 100, raise_on_result_error: true, **opts)
+      body = AdditionalDataBody.new({ "action" => "start", "seed_urls" => [url],
+                                      "max_depth" => max_depth, "max_pages" => max_pages }
+                                      .merge(opts.transform_keys(&:to_s)))
+      hash = ResponseHelper.to_hash(@v1.crawl_runs.post(body))
+      raise_on_result_error ? ResponseHelper.raise_on_result_error!(hash) : hash
+    end
+
+    # List crawl runs.
+    # Delegates to GET /v1/crawl-runs via the generated CrawlRunsRequestBuilder.
+    def list
+      ResponseHelper.to_hash(@v1.crawl_runs.get)
+    end
+
+    # Get a single crawl run by ID.
+    # Delegates to GET /v1/crawl-runs/{run_id} via the generated builder.
+    def get(run_id)
+      ResponseHelper.to_hash(@v1.crawl_runs.by_run_id(run_id).get)
+    end
+
+    # Cancel a running crawl run.
+    # Delegates to POST /v1/crawl-runs/{run_id}/cancel via the generated builder.
+    def cancel(run_id)
+      ResponseHelper.to_hash(@v1.crawl_runs.by_run_id(run_id).cancel.post)
+    end
+  end
+
+  # Manage browser sessions — /v1/sessions.
+  class SessionsClient
+    def initialize(v1)
+      @v1 = v1
+    end
+
+    # List all active sessions.
+    # Delegates to GET /v1/sessions via the generated SessionsRequestBuilder.
+    def list
+      ResponseHelper.to_hash(@v1.sessions.get)
+    end
+
+    # Create a new browser session.
+    # Delegates to POST /v1/sessions/create via the generated builder.
+    # @param profile_name [String] identity profile to use
+    def create(profile_name:, **opts)
+      data = { "profile" => profile_name }.merge(opts.transform_keys(&:to_s))
+      ResponseHelper.to_hash(@v1.sessions.create.post(AdditionalDataBody.new(data)))
+    end
+
+    # Extend a session's TTL.
+    # Delegates to POST /v1/sessions/{id}/extend via the generated builder.
+    def extend(session_id, duration_seconds: 300)
+      ResponseHelper.to_hash(@v1.sessions.by_profile__id(session_id).extend.post(
+        AdditionalDataBody.new({ "ttl_seconds" => duration_seconds })
+      ))
+    end
+
+    # Release a session back to the pool.
+    # Delegates to POST /v1/sessions/{id}/release via the generated builder.
+    def release(session_id)
+      ResponseHelper.to_hash(@v1.sessions.by_profile__id(session_id).release.post)
+    end
+  end
+
+  # Manage identity profiles — /v1/profiles.
+  class ProfilesClient
+    def initialize(v1)
+      @v1 = v1
+    end
+
+    # List all profiles.
+    # Delegates to GET /v1/profiles via the generated ProfilesRequestBuilder.
+    def list
+      ResponseHelper.to_hash(@v1.profiles.get)
+    end
+
+    # Get a profile by name.
+    # Delegates to GET /v1/profiles/{name} via the generated builder.
+    def get(name)
+      ResponseHelper.to_hash(@v1.profiles.by_name(name).get)
+    end
+
+    # Create a new profile.
+    # Delegates to POST /v1/profiles via the generated ProfilesRequestBuilder.
+    def create(name:, **config)
+      body = AdditionalDataBody.new({ "name" => name }.merge(config.transform_keys(&:to_s)))
+      ResponseHelper.to_hash(@v1.profiles.post(body))
+    end
+
+    # Update a profile.
+    # Delegates to PUT /v1/profiles/{name} via the generated builder.
+    def update(name, **config)
+      ResponseHelper.to_hash(@v1.profiles.by_name(name).put(AdditionalDataBody.new(config.transform_keys(&:to_s))))
+    end
+
+    # Delete a profile.
+    # Delegates to DELETE /v1/profiles/{name} via the generated builder.
+    def delete(name)
+      ResponseHelper.to_hash(@v1.profiles.by_name(name).delete)
+    end
+  end
+
+  # Manage webhooks — /v1/webhooks.
+  class WebhooksClient
+    def initialize(v1)
+      @v1 = v1
+    end
+
+    # List all webhooks.
+    # Delegates to GET /v1/webhooks via the generated WebhooksRequestBuilder.
+    def list
+      ResponseHelper.to_hash(@v1.webhooks.get)
+    end
+
+    # Get a webhook by ID.
+    # Delegates to GET /v1/webhooks/{id} via the generated builder.
+    def get(webhook_id)
+      ResponseHelper.to_hash(@v1.webhooks.by_webhook_id(webhook_id).get)
+    end
+
+    # Register a new webhook endpoint.
+    # Delegates to POST /v1/webhooks via the generated WebhooksRequestBuilder.
+    def create(url:, event_types: nil, events: nil, **opts)
+      data = { "url" => url }.merge(opts.transform_keys(&:to_s))
+      # API field is "event_types"; "events" kept as a back-compat alias.
+      et = event_types.nil? ? events : event_types
+      data["event_types"] = et unless et.nil?
+      ResponseHelper.to_hash(@v1.webhooks.post(AdditionalDataBody.new(data)))
+    end
+
+    # Delete a webhook.
+    # Delegates to DELETE /v1/webhooks/{id} via the generated builder.
+    def delete(webhook_id)
+      @v1.webhooks.by_webhook_id(webhook_id).delete
+      {}
+    end
+
+    # Rotate the signing secret for a webhook.
+    # Delegates to POST /v1/webhooks/{id}/rotate-secret via the generated builder.
+    def rotate_secret(webhook_id)
+      ResponseHelper.to_hash(@v1.webhooks.by_webhook_id(webhook_id).rotate_secret.post)
+    end
+  end
+
+  # Manage schedules — /v1/schedules.
+  class SchedulesClient
+    def initialize(v1)
+      @v1 = v1
+    end
+
+    # List all schedules.
+    # Delegates to GET /v1/schedules via the generated SchedulesRequestBuilder.
+    def list
+      ResponseHelper.to_hash(@v1.schedules.get)
+    end
+
+    # Get a schedule by ID.
+    # Delegates to GET /v1/schedules/{id} via the generated builder.
+    def get(schedule_id)
+      ResponseHelper.to_hash(@v1.schedules.by_schedule_id(schedule_id).get)
+    end
+
+    # Create a new schedule.
+    # Delegates to POST /v1/schedules via the generated SchedulesRequestBuilder.
+    def create(cron:, task:, **opts)
+      data = { "cron_expr" => cron, "task" => task }.merge(opts.transform_keys(&:to_s))
+      ResponseHelper.to_hash(@v1.schedules.post(AdditionalDataBody.new(data)))
+    end
+
+    # Delete a schedule.
+    # Delegates to DELETE /v1/schedules/{id} via the generated builder.
+    def delete(schedule_id)
+      @v1.schedules.by_schedule_id(schedule_id).delete
+      {}
+    end
+  end
+
+  # Manage datasets — /v1/datasets.
+  class DatasetsClient
+    def initialize(v1)
+      @v1 = v1
+    end
+
+    # List all datasets.
+    # Delegates to GET /v1/datasets via the generated DatasetsRequestBuilder.
+    def list
+      ResponseHelper.to_hash(@v1.datasets.get)
+    end
+
+    # Get a dataset by name.
+    # Delegates to GET /v1/datasets/{name} via the generated builder.
+    def get(name)
+      ResponseHelper.to_hash(@v1.datasets.by_name(name).get)
+    end
+
+    # Create a new dataset.
+    # Delegates to POST /v1/datasets via the generated DatasetsRequestBuilder.
+    def create(name:, **opts)
+      body = AdditionalDataBody.new({ "name" => name }.merge(opts.transform_keys(&:to_s)))
+      ResponseHelper.to_hash(@v1.datasets.post(body))
+    end
+
+    # Delete a dataset.
+    # Delegates to DELETE /v1/datasets/{name} via the generated builder.
+    def delete(name)
+      ResponseHelper.to_hash(@v1.datasets.by_name(name).delete)
+    end
+
+    # Get rows from a dataset.
+    # Delegates to GET /v1/datasets/{name}/rows via the generated builder.
+    def rows(name)
+      ResponseHelper.to_hash(@v1.datasets.by_name(name).rows.get)
+    end
+
+    # Append rows to a dataset.
+    # Delegates to POST /v1/datasets/{name}/rows/append via the generated builder.
+    def append(name, rows)
+      body = AdditionalDataBody.new({ "rows" => rows })
+      ResponseHelper.to_hash(@v1.datasets.by_name(name).rows.append.post(body))
+    end
+  end
+
+  # Manage session recordings — /v1/recordings.
+  class RecordingsClient
+    def initialize(v1)
+      @v1 = v1
+    end
+
+    # List all recordings.
+    # Delegates to GET /v1/recordings via the generated RecordingsRequestBuilder.
+    def list
+      ResponseHelper.to_hash(@v1.recordings.get)
+    end
+
+    # Get a recording by ID.
+    # Delegates to GET /v1/recordings/{id} via the generated builder.
+    def get(recording_id)
+      ResponseHelper.to_hash(@v1.recordings.by_recording__id(recording_id).get)
+    end
+
+    # Delete a recording.
+    # Delegates to DELETE /v1/recordings/{id} via the generated builder.
+    def delete(recording_id)
+      @v1.recordings.by_recording__id(recording_id).delete
+      {}
+    end
+  end
+
+  # Key-value store — /v1/kv.
+  class KVClient
+    def initialize(v1)
+      @v1 = v1
+    end
+
+    # Get a value by key.
+    # Delegates to GET /v1/kv/{key} via the generated KvRequestBuilder.
+    def get(key)
+      ResponseHelper.to_hash(@v1.kv.by_key(key).get)
+    end
+
+    # Set a key-value pair.
+    # Delegates to PUT /v1/kv/{key} via the generated builder.
+    def set(key, value)
+      body = AdditionalDataBody.new({ "value" => value })
+      ResponseHelper.to_hash(@v1.kv.by_key(key).put(body))
+    end
+
+    # Delete a key.
+    # Delegates to DELETE /v1/kv/{key} via the generated builder.
+    def delete(key)
+      ResponseHelper.to_hash(@v1.kv.by_key(key).delete)
+    end
+  end
+
+  # ---------------------------------------------------------------------------
+  # Main facade — Client
+  # ---------------------------------------------------------------------------
+
+  # GhostCrawl idiomatic API client.
+  #
+  # Delegates all HTTP transport, URL routing, serialization, and auth to the
+  # Kiota-generated canonical core (_generated/). This facade is the shipped API.
+  #
+  # @example
+  #   require "ghostcrawl"
+  #   client = Ghostcrawl::Client.new(token: "gck_live_YOUR_KEY")
+  #   result = client.scrape(url: "https://example.com")
+  class Client
+    # Default read timeout (seconds). Browser-rendered scrapes/crawls are slow,
+    # and the underlying net/http default (60s) is too short for them.
+    DEFAULT_TIMEOUT = 300
+
+    # @param token [String, nil] API key. Falls back to +GHOSTCRAWL_API_KEY+ env var.
+    # @param base_url [String, nil] Override API base URL. Falls back to +GHOSTCRAWL_BASE_URL+ env var.
+    # @param timeout [Integer, nil] Per-request read timeout in seconds. Falls back
+    #   to +GHOSTCRAWL_TIMEOUT+ env var, then {DEFAULT_TIMEOUT}.
+    def initialize(token: nil, base_url: nil, timeout: nil)
+      resolved_token = token || ENV.fetch("GHOSTCRAWL_API_KEY", nil)
+      if resolved_token.nil? || resolved_token.empty?
+        raise ArgumentError,
+              "token is required — pass token: or set GHOSTCRAWL_API_KEY. " \
+              "Get your key at https://ghostcrawl.io"
+      end
+
+      resolved_base = (base_url ||
+                       ENV.fetch("GHOSTCRAWL_BASE_URL", nil) ||
+                       DEFAULT_BASE_URL).gsub(%r{/+$}, "")
+
+      resolved_timeout = (timeout ||
+                          ENV.fetch("GHOSTCRAWL_TIMEOUT", nil) ||
+                          DEFAULT_TIMEOUT).to_i
+
+      # Build the Kiota core via BaseBearerTokenAuthenticationProvider + FaradayRequestAdapter.
+      # All HTTP, auth, serialization, and URL routing delegate to the generated core.
+      auth_provider = MicrosoftKiotaAbstractions::BaseBearerTokenAuthenticationProvider.new(
+        StaticTokenProvider.new(resolved_token)
+      )
+      adapter = MicrosoftKiotaFaraday::FaradayRequestAdapter.new(auth_provider)
+      adapter.set_base_url(resolved_base)
+
+      # The default Faraday connection sets no timeout, so it inherits net/http's
+      # 60s read timeout — too short for browser-rendered work. Raise it.
+      if adapter.client.respond_to?(:options) && resolved_timeout.positive?
+        adapter.client.options.timeout = resolved_timeout
+        adapter.client.options.open_timeout = 30
+      end
+
+      @core    = Ghostcrawl::GhostcrawlClient.new(adapter)
+      @v1      = @core.v1
+      # Kept so #me can re-issue the GET /v1/me request with the raw-JSON
+      # (Binary) response factory instead of the typed MeResponse model.
+      @adapter = adapter
+
+      @crawl_runs  = CrawlRunsClient.new(@v1)
+      @sessions    = SessionsClient.new(@v1)
+      @profiles    = ProfilesClient.new(@v1)
+      @webhooks    = WebhooksClient.new(@v1)
+      @schedules   = SchedulesClient.new(@v1)
+      @datasets    = DatasetsClient.new(@v1)
+      @recordings  = RecordingsClient.new(@v1)
+      @kv          = KVClient.new(@v1)
+    end
+
+    # @return [CrawlRunsClient]
+    attr_reader :crawl_runs
+    # @return [SessionsClient]
+    attr_reader :sessions
+    # @return [ProfilesClient]
+    attr_reader :profiles
+    # @return [WebhooksClient]
+    attr_reader :webhooks
+    # @return [SchedulesClient]
+    attr_reader :schedules
+    # @return [DatasetsClient]
+    attr_reader :datasets
+    # @return [RecordingsClient]
+    attr_reader :recordings
+    # @return [KVClient]
+    attr_reader :kv
+
+    # ---------------------------------------------------------------------------
+    # Top-level facade methods — delegate to generated builders
+    # ---------------------------------------------------------------------------
+
+    # Scrape a single URL and return the rendered content.
+    # Delegates to POST /v1/scrape via the generated ScrapeRequestBuilder.
+    # @param url [String] target URL
+    # @param format [String] output format: "markdown" (default), "html", "text"
+    # @param engine [String] browser engine: "auto" (default), "chrome", "firefox", "webkit"
+    # @param javascript [Boolean] enable JavaScript rendering (default true)
+    # @param extract_schema [Hash, nil] JSON Schema for structured extraction
+    # @param raise_on_result_error [Boolean] raise {Ghostcrawl::ScrapeError} on a
+    #   target-side (HTTP-200) failure instead of returning the raw hash (default true)
+    # @return [Hash] response with +markdown+, +status+, and other fields
+    def scrape(url:, format: "markdown", engine: "auto", javascript: true, extract_schema: nil,
+               raise_on_result_error: true, **opts)
+      # Use AdditionalDataBody to send only the fields we specify — the generated
+      # ScrapeRequest model would serialize typed defaults (nulls + empty enums) that
+      # cause 422 validation errors on the server.
+      data = { "url" => url, "format" => format, "engine" => engine,
+               "javascript_enabled" => javascript }.merge(opts.transform_keys(&:to_s))
+      data["extract_schema"] = extract_schema unless extract_schema.nil?
+      hash = ResponseHelper.to_hash(@v1.scrape.post(AdditionalDataBody.new(data)))
+      raise_on_result_error ? ResponseHelper.raise_on_result_error!(hash) : hash
+    end
+
+    # Search the web and return results.
+    # Delegates to POST /v1/search via the generated SearchRequestBuilder.
+    #
+    # /v1/search requires your own search-backend API key (BYO; GhostCrawl
+    # charges no markup). Pass it as +provider_key+ — it is sent as the
+    # +X-Provider-Authorization: Bearer <provider_key>+ header the backend
+    # requires. Without it the API replies 401 search_backend_key_missing.
+    # @param query [String] search query
+    # @param engine [String] search engine: "google" (default), "bing", "duckduckgo"
+    # @param limit [Integer] maximum results (default 10)
+    # @param provider_key [String, nil] BYO search-backend key (sent as X-Provider-Authorization)
+    # @return [Hash] response with +results+ list
+    def search(query:, engine: "google", limit: 10, provider_key: nil, **opts)
+      data = { "query" => query, "engine" => engine,
+               "limit" => limit }.merge(opts.transform_keys(&:to_s))
+      config = nil
+      unless provider_key.nil?
+        config = MicrosoftKiotaAbstractions::RequestConfiguration.new
+        headers = MicrosoftKiotaAbstractions::RequestHeaders.new
+        headers.add("X-Provider-Authorization", "Bearer #{provider_key}")
+        config.headers = headers
+      end
+      ResponseHelper.to_hash(@v1.search.post(AdditionalDataBody.new(data), config))
+    end
+
+    # Extract structured data from a URL using a JSON Schema.
+    # Delegates to POST /v1/extract via the generated ExtractRequestBuilder.
+    # @param url [String] target URL
+    # @param schema [Hash] JSON Schema describing the shape to extract
+    # @param raise_on_result_error [Boolean] raise {Ghostcrawl::ScrapeError} on a
+    #   target-side (HTTP-200) failure instead of returning the raw hash (default true)
+    # @return [Hash] extracted data
+    def extract(url:, schema:, raise_on_result_error: true, **opts)
+      data = { "url" => url, "schema" => schema }.merge(opts.transform_keys(&:to_s))
+      hash = ResponseHelper.to_hash(@v1.extract.post(AdditionalDataBody.new(data)))
+      raise_on_result_error ? ResponseHelper.raise_on_result_error!(hash) : hash
+    end
+
+    # Start a deep crawl from a seed URL.
+    # Delegates to POST /v1/crawl/deep via the generated CrawlDeepRequestBuilder.
+    # @param url [String] seed URL
+    # @param max_depth [Integer] maximum crawl depth (default 2)
+    # @param max_pages [Integer] maximum pages (default 100)
+    # @param raise_on_result_error [Boolean] raise {Ghostcrawl::ScrapeError} on a
+    #   target-side (HTTP-200) failure instead of returning the raw hash (default true)
+    # @return [Hash] crawl run record
+    def crawl(url:, max_depth: 2, max_pages: 100, raise_on_result_error: true, **opts)
+      data = { "seed_urls" => [url], "max_depth" => max_depth,
+               "max_urls" => max_pages }.merge(opts.transform_keys(&:to_s))
+      hash = ResponseHelper.to_hash(@v1.crawl.deep.post(AdditionalDataBody.new(data)))
+      raise_on_result_error ? ResponseHelper.raise_on_result_error!(hash) : hash
+    end
+
+    # Map all URLs reachable from a seed URL.
+    # Delegates to POST /v1/map via the generated MapRequestBuilder.
+    # @param url [String] seed URL
+    # @return [Hash] response with +urls+ list
+    def map(url:, **opts)
+      data = { "url" => url }.merge(opts.transform_keys(&:to_s))
+      ResponseHelper.to_hash(@v1.map.post(AdditionalDataBody.new(data)))
+    end
+
+    # Get the current account's profile.
+    # Delegates to GET /v1/me via the generated MeRequestBuilder.
+    #
+    # The generated +@v1.me.get+ deserializes into the typed +MeResponse+ model,
+    # whose +created_at+ composed-type member fails under the pinned Kiota JSON
+    # parser ("Error during deserialization"). To avoid that, we re-issue the
+    # SAME request (reusing the builder's URL/auth/header wiring via
+    # +to_get_request_information+) with the +Binary+ response factory — the
+    # exact raw-JSON path the +scrape+ builder already uses (see KiotaParseNodeFix).
+    # @return [Hash] account info with +user_id+, +email+, +created_at+, +primary_team+
+    def me
+      request_info = @v1.me.to_get_request_information(nil)
+      ResponseHelper.to_hash(@adapter.send_async(request_info, Ghostcrawl::V1::Binary, {}))
+    end
+  end
+end