ghostcrawl 2.3.4

data/lib/ghostcrawl/client.rb

@@ -0,0 +1,1351 @@
+# frozen_string_literal: true
+
+# GhostCrawl Ruby SDK — idiomatic facade over the Kiota-generated core.
+#
+# Architecture:
+#
+#   _generated/ Kiota core  — spec-faithful 98-op request-builder (models, transport, auth)
+#   This FACADE              — thin idiomatic layer delegating to the generated builders
+#
+# All HTTP transport, URL routing, serialization, and auth come from the generated core.
+# The facade maps idiomatic calls (client.scrape) to generated builders via
+# BaseBearerTokenAuthenticationProvider + FaradayRequestAdapter.
+#
+# Usage:
+#   require "ghostcrawl"
+#   client = GhostCrawl::Client.new(token: "gck_live_YOUR_KEY")
+#   result = client.scrape(url: "https://example.com")
+
+require "json"
+require "microsoft_kiota_abstractions"
+require "microsoft_kiota_faraday"
+require "microsoft_kiota_serialization_json"
+require_relative "../../_generated/ghostcrawl_client"
+require_relative "../../_generated/models/scrape_request"
+require_relative "../../_generated/models/search_request"
+require_relative "../../_generated/models/extract_request"
+require_relative "../../_generated/models/deep_crawl_body"
+require_relative "../../_generated/models/map_body"
+require_relative "../../_generated/models/session_create_request"
+require_relative "../../_generated/models/extend_body"
+require_relative "../../_generated/models/profile_create_request"
+require_relative "../../_generated/models/profile_update_request"
+require_relative "../../_generated/models/webhook_create_request"
+require_relative "../../_generated/models/schedule_create_request"
+require_relative "errors"
+
+module GhostCrawl
+  # ---------------------------------------------------------------------------
+  # Fix: microsoft_kiota_serialization_json 0.9.2 has a bug where
+  # write_object_value(nil, body) creates a temp writer, serializes body into
+  # temp, then RETURNS temp without merging temp's content into self.@writer.
+  # set_content_from_parsable calls write_object_value(nil, body) and discards
+  # the return value, so @content = {}.to_json = "{}".
+  #
+  # Fix: monkey-patch write_object_value so that when key is nil, it serializes
+  # body into self directly (not a temp writer) — same as what the non-nil branch
+  # does, but without a key.
+  # ---------------------------------------------------------------------------
+
+  # @api private
+  module KiotaWriterFix
+    def write_object_value(key, value)
+      return unless value
+      if key.nil?
+        # Fix: serialize into self, not a temp. Merges all fields into @writer.
+        value.serialize(self)
+      else
+        super
+      end
+    end
+
+    # Fix 1b: the pinned Kiota writer's write_collection_of_object_values calls
+    # `self.write_object_value(nil, v).writer` on each element — it relies on
+    # write_object_value(nil, ...) returning a *writer object* that responds to
+    # `.writer`. Our write_object_value override above (Fix 1) changes that
+    # nil-key contract to serialize-into-self and return the serialize result
+    # (a Hash / nil), so the upstream `.writer` call raises
+    # `NoMethodError: undefined method 'writer' for {}:Hash`. That aborts
+    # serialize for ANY response model carrying a collection of typed objects
+    # (WebhookListResponse#items, etc.), which ResponseHelper.serialize_parsable
+    # then swallows -> the facade returns {} and silently drops the whole list.
+    #
+    # Fix: serialize each element into its OWN fresh writer and collect the raw
+    # per-element hashes directly, never depending on the write_object_value
+    # return contract. Mirrors the non-nil upstream branch (line 154) without the
+    # broken `.writer` deref.
+    def write_collection_of_object_values(key, values)
+      return unless values
+      hashes = values.map do |v|
+        temp = MicrosoftKiotaSerializationJson::JsonSerializationWriter.new
+        v.serialize(temp)
+        temp.writer
+      end
+      if key.nil?
+        hashes
+      else
+        writer[key] = hashes
+      end
+    end
+
+    # Fix 1c: the pinned Kiota writer's write_any_value (used by
+    # write_additional_data, i.e. EVERY field AdditionalDataBody sends) has no
+    # case for a plain Hash and mis-handles an Array of Hashes:
+    #   * a Hash hits the `value.is_a? Object` branch -> `return value.to_s`,
+    #     which returns a string but NEVER assigns @writer[key] -> the whole
+    #     nested object is SILENTLY DROPPED from the request body.
+    #   * an Array is sent to write_collection_of_primitive_values, which mangles
+    #     any non-primitive (Hash) element.
+    # This drops/mangles every nested field the facade sends: scrape(extract_schema),
+    # extract(schema), crawl/crawl_runs opts, datasets.append(rows: [ {..} ]),
+    # schedules.create(job_params: {..}) — the last of which the API REQUIRES,
+    # so the request 422s ("job_params required") even though the caller supplied it.
+    #
+    # Fix: intercept Hash and Array here and write a fully-recursed, JSON-native
+    # structure straight into @writer[key]. The writer emits @writer.to_json at the
+    # end, so plain nested Hash/Array values serialize correctly. Everything else
+    # falls through to the upstream primitive handling.
+    def write_any_value(key, value)
+      if value.is_a?(Hash) || value.is_a?(Array)
+        return super unless key # nil-key arrays keep upstream behavior
+        writer[key] = KiotaWriterFix.jsonable(value)
+        return
+      end
+      super
+    end
+
+    # Recursively converts a value into a JSON-native structure (Hash/Array of
+    # plain scalars), so it round-trips through @writer.to_json intact.
+    # @api private
+    def self.jsonable(value)
+      case value
+      when Hash  then value.each_with_object({}) { |(k, v), h| h[k.to_s] = jsonable(v) }
+      when Array then value.map { |v| jsonable(v) }
+      else value
+      end
+    end
+  end
+
+  # Patch the writer class once, idempotently
+  MicrosoftKiotaSerializationJson::JsonSerializationWriter.prepend(KiotaWriterFix) \
+    unless MicrosoftKiotaSerializationJson::JsonSerializationWriter.ancestors.include?(KiotaWriterFix)
+
+  # ---------------------------------------------------------------------------
+  # Fix 2: Some generated request builders pass a Module (e.g. `Binary`) as
+  # the factory to `send_async`. The parse node's `get_object_value(factory)`
+  # calls `factory.call(self)`, which fails for a Module.
+  #
+  # Fix: when factory doesn't respond to `call`, fall back to parsing the
+  # response body as JSON and returning the raw hash. This gives us plain
+  # Ruby hashes (which ResponseHelper.to_hash handles correctly) without
+  # requiring generated response models.
+  # ---------------------------------------------------------------------------
+
+  # @api private
+  module KiotaParseNodeFix
+    def get_object_value(factory)
+      return super if factory.respond_to?(:call)
+      # Fallback: return the raw parsed node for non-callable factories (e.g. Binary).
+      # @current_node is already a Ruby Hash/Array/primitive parsed from JSON.
+      @current_node
+    rescue StandardError
+      super
+    end
+  end
+
+  MicrosoftKiotaSerializationJson::JsonParseNode.prepend(KiotaParseNodeFix) \
+    unless MicrosoftKiotaSerializationJson::JsonParseNode.ancestors.include?(KiotaParseNodeFix)
+
+  # ---------------------------------------------------------------------------
+  # AdditionalDataBody — a minimal Parsable that wraps an arbitrary Hash.
+  #
+  # Used for all POST bodies. The generated models set typed defaults in their
+  # constructors that would be serialized as null/empty-enum values and cause
+  # 422 validation errors. AdditionalDataBody only serializes the fields we
+  # explicitly pass, producing clean JSON.
+  #
+  # Depends on KiotaWriterFix above to correctly handle the nil-key
+  # write_object_value(nil, body) call from set_content_from_parsable.
+  # ---------------------------------------------------------------------------
+
+  # @api private
+  class AdditionalDataBody
+    include MicrosoftKiotaAbstractions::Parsable
+
+    def initialize(data = {})
+      @data = data.transform_keys(&:to_s)
+    end
+
+    def get_field_deserializers
+      {}
+    end
+
+    def serialize(writer)
+      writer.write_additional_data(@data)
+    end
+
+    def additional_data
+      @data
+    end
+
+    def additional_data=(hash)
+      @data = hash.transform_keys(&:to_s)
+    end
+
+    def self.create_from_discriminator_value(_parse_node)
+      AdditionalDataBody.new
+    end
+  end
+end
+
+module GhostCrawl
+  DEFAULT_BASE_URL = "https://api.ghostcrawl.io"
+
+  # ---------------------------------------------------------------------------
+  # Static bearer token provider — implements AccessTokenProvider
+  # ---------------------------------------------------------------------------
+
+  # @api private
+  class StaticTokenProvider
+    include MicrosoftKiotaAbstractions::AccessTokenProvider
+
+    def initialize(token)
+      @token = token
+    end
+
+    # The Kiota Faraday adapter calls .resume on the return value of this method,
+    # expecting a Fiber. Wrap the token in a Fiber to satisfy the contract.
+    def get_authorization_token(_uri, _additional_authentication_context = nil)
+      token = @token
+      Fiber.new { token }
+    end
+
+    def get_allowed_hosts_validator
+      MicrosoftKiotaAbstractions::AllowedHostsValidator.new([])
+    end
+  end
+
+  # ---------------------------------------------------------------------------
+  # Response helpers — convert Kiota Parsable/Hash responses to plain Hash
+  # ---------------------------------------------------------------------------
+
+  # @api private
+  module ResponseHelper
+    # Converts any Kiota response value to a plain Hash or Array.
+    # The Kiota Faraday adapter returns Fibers for async responses; call .resume
+    # to execute the request synchronously and get the actual response value.
+    def self.to_hash(value)
+      # Resolve Fibers (the Kiota Faraday adapter returns a Fiber for every
+      # request; .resume is what actually executes the HTTP call). Any non-2xx
+      # surfaces here as a raw transport exception — translate it into a typed
+      # GhostCrawl error so the documented rescue contract works.
+      if value.is_a?(Fiber)
+        begin
+          value = value.resume
+        rescue GhostCrawl::GhostCrawlError
+          raise
+        rescue StandardError => e
+          GhostCrawl.raise_translated(e)
+        end
+      end
+
+      case value
+      when Hash
+        value.transform_values { |v| to_hash(v) }
+      when Array
+        value.map { |v| to_hash(v) }
+      when NilClass
+        {}
+      else
+        # Typed Parsable (Kiota-generated model). Kiota models keep their spec
+        # fields in typed instance variables, NOT in +additional_data+ (which is
+        # only the overflow bucket for unmapped keys). Reading +additional_data+
+        # alone therefore returns +{}+ for a fully-typed response (e.g.
+        # MapResponse{@links,@success}, WebhookListResponse{@items,@total}),
+        # silently dropping the real payload.
+        #
+        # Recover the typed fields by round-tripping the model through its own
+        # +serialize+ (the Parsable contract) into a JSON writer and parsing the
+        # result back to a plain Hash. Then overlay any +additional_data+ (unmapped
+        # keys the server sent that the model didn't declare). Degrade gracefully:
+        # a model whose +serialize+ raises (e.g. an unresolved composed-type member)
+        # falls back to the prior additional_data/to_h behavior so nothing regresses.
+        parsable_to_hash(value)
+      end
+    end
+
+    # Converts a typed Kiota Parsable into a plain Hash by serializing it via its
+    # own +serialize+ and parsing the emitted JSON, then merging +additional_data+.
+    # Falls back to +additional_data+ / +to_h+ / the value itself on any failure.
+    # @api private
+    def self.parsable_to_hash(value)
+      serialized = serialize_parsable(value)
+      if serialized.is_a?(Hash)
+        extra = (value.additional_data if value.respond_to?(:additional_data)) || {}
+        merged = serialized.merge(extra.transform_keys(&:to_s))
+        return merged.transform_values { |v| to_hash(v) }
+      end
+
+      if value.respond_to?(:additional_data) && value.additional_data && !value.additional_data.empty?
+        value.additional_data.transform_values { |v| to_hash(v) }
+      elsif value.respond_to?(:to_h)
+        value.to_h.transform_values { |v| to_hash(v) }
+      elsif value.respond_to?(:additional_data) && value.additional_data
+        value.additional_data.transform_values { |v| to_hash(v) }
+      else
+        value
+      end
+    end
+
+    # Serializes a Parsable to a Ruby Hash using the pinned Kiota JSON writer.
+    # Returns the parsed Hash, or +nil+ when the model isn't a serializable
+    # Parsable or serialization/parse fails (caller then falls back).
+    # @api private
+    def self.serialize_parsable(value)
+      return nil unless value.respond_to?(:serialize)
+
+      writer = MicrosoftKiotaSerializationJson::JsonSerializationWriter.new
+      value.serialize(writer)
+      content = writer.get_serialized_content
+      json = content.is_a?(String) ? content : content.read
+      return nil if json.nil? || json.empty?
+
+      parsed = JSON.parse(json)
+      parsed.is_a?(Hash) ? parsed : nil
+    rescue StandardError
+      nil
+    end
+
+    # Executes a request that returns NO response body (an HTTP 204, or any DELETE
+    # whose spec maps the success response to +void+) and returns a plain Hash.
+    #
+    # Why this exists — two upstream defects in the pinned Kiota Ruby runtime that
+    # make the generated void-DELETE builders unusable:
+    #
+    #   1. The generated +delete+ for a void response calls
+    #      +send_async(request_info, nil, ...)+, and the Faraday adapter hard-raises
+    #      +"factory cannot be null"+ BEFORE the request is ever sent — so the DELETE
+    #      never reaches the server and the caller gets a bare +StandardError+
+    #      (not a {GhostCrawlError}).
+    #   2. For void DELETEs the adapter DOES send maps to +Binary+, the response is an
+    #      empty 204 body, and +get_root_parse_node+ then feeds +""+ to the JSON parser,
+    #      raising +JSON::ParserError+ on a perfectly successful delete.
+    #
+    # This helper reuses the adapter's OWN public request pipeline
+    # (+convert_to_native_request_async+ applies base-url + auth, +run_request+
+    # sends it) but skips body deserialization entirely: a 2xx returns +{}+ (or the
+    # decoded JSON when the server did send a body, e.g. +{"deleted":true}+), and a
+    # >=400 is translated into the documented typed {GhostCrawlError} hierarchy.
+    #
+    # @param adapter [MicrosoftKiotaFaraday::FaradayRequestAdapter]
+    # @param request_info [MicrosoftKiotaAbstractions::RequestInformation]
+    # @return [Hash] +{}+ on an empty 2xx, or the decoded body when one is present
+    # @api private
+    def self.void_request!(adapter, request_info)
+      # Replicate the first half of the adapter's send_async by hand — apply the
+      # base URL, run the bearer-auth Fiber (which mutates request_info's headers),
+      # then build the native Faraday request. We deliberately do NOT call the
+      # adapter's convert_to_native_request_async helper: its body uses a bare
+      # +return+ inside a Fiber, which raises LocalJumpError("unexpected return")
+      # when resumed under Ruby 3.x.
+      request_info.path_parameters["baseurl"] = adapter.get_base_url
+      # authenticate_request returns a Fiber only when the Authorization header is
+      # not already present; it returns nil otherwise. Guard the resume.
+      auth_fiber = adapter.authentication_provider.authenticate_request(request_info)
+      auth_fiber.resume if auth_fiber.respond_to?(:resume)
+      request = adapter.get_request_from_request_info(request_info)
+      response = adapter.client.run_request(
+        request.http_method, request.path, request.body, request.headers
+      )
+
+      status = response.status
+      if status >= 400
+        # Reuse the typed-error translation: synthesize a message carrying the
+        # ":<status>" token raise_translated keys off, plus any body text.
+        body = response.body.to_s
+        err = MicrosoftKiotaAbstractions::ApiError.new(
+          "The server returned an unexpected status code:#{status}" \
+          "#{body.empty? ? '' : " #{body}"}"
+        )
+        GhostCrawl.raise_translated(err)
+      end
+
+      body = response.body.to_s
+      return {} if body.strip.empty?
+
+      parsed = (JSON.parse(body) rescue nil)
+      parsed.is_a?(Hash) ? parsed.transform_values { |v| to_hash(v) } : {}
+    rescue GhostCrawl::GhostCrawlError
+      raise
+    rescue StandardError => e
+      GhostCrawl.raise_translated(e)
+    end
+
+    # Executes a request whose success response is RAW BINARY (e.g. the
+    # +application/pdf+ body from +POST /v1/pdf+) and returns the bytes verbatim.
+    #
+    # Mirrors {void_request!}: it reuses the adapter's own request pipeline
+    # (base URL + bearer auth via +authenticate_request+, then +run_request+) but
+    # deliberately skips Kiota body deserialization — the response is not JSON, so
+    # there is no parse node to build. A 2xx returns the raw response body (an
+    # ASCII-8BIT String of bytes, ready to write to a file); a >=400 is translated
+    # into the documented typed {GhostCrawlError} hierarchy (the problem+json
+    # +detail+ rides along in the message).
+    #
+    # @param adapter [MicrosoftKiotaFaraday::FaradayRequestAdapter]
+    # @param request_info [MicrosoftKiotaAbstractions::RequestInformation]
+    # @return [String] the raw response bytes (ASCII-8BIT)
+    # @api private
+    def self.binary_request!(adapter, request_info)
+      request_info.path_parameters["baseurl"] = adapter.get_base_url
+      auth_fiber = adapter.authentication_provider.authenticate_request(request_info)
+      auth_fiber.resume if auth_fiber.respond_to?(:resume)
+      request = adapter.get_request_from_request_info(request_info)
+      response = adapter.client.run_request(
+        request.http_method, request.path, request.body, request.headers
+      )
+
+      status = response.status
+      if status >= 400
+        body = response.body.to_s
+        err = MicrosoftKiotaAbstractions::ApiError.new(
+          "The server returned an unexpected status code:#{status}" \
+          "#{body.empty? ? '' : " #{body}"}"
+        )
+        GhostCrawl.raise_translated(err)
+      end
+
+      response.body.to_s.b
+    rescue GhostCrawl::GhostCrawlError
+      raise
+    rescue StandardError => e
+      GhostCrawl.raise_translated(e)
+    end
+
+    # Inspects a decoded HTTP-200 response hash for a RESULT-channel failure (the
+    # target page could not be scraped) and raises {GhostCrawl::ScrapeError} when
+    # one is present. This is the reliable, highest-value error path: the body is
+    # always available here (unlike the dropped problem+json body on non-2xx).
+    #
+    # A failure is signalled by any of:
+    #   * a +result_error+ Hash carrying a +code+
+    #   * +ok+ explicitly +false+ (always a failure, even with no code)
+    #   * a top-level +code+ that is a known RESULT-channel code
+    #
+    # A genuinely OK hash (ok: true, no +ok+ key, or no error code) is returned
+    # untouched and never raises.
+    #
+    # @param hash [Hash] the decoded response
+    # @return [Hash] the same hash, when it is not a failure
+    # @api private
+    def self.raise_on_result_error!(hash)
+      return hash unless hash.is_a?(Hash)
+
+      # Descend into a `results` envelope (scrape/extract wrap per-URL results) —
+      # the target failure lives on the INNER result, not the envelope top level.
+      inner = hash["results"]
+      if inner.is_a?(Array)
+        inner.each { |item| raise_on_result_error!(item) }
+        return hash
+      end
+
+      result_error = hash["result_error"]
+      result_error = nil unless result_error.is_a?(Hash)
+      top_code = hash["code"]
+      ok_false = hash["ok"] == false
+
+      # Pull the code: result_error wins, then a top-level RESULT-channel code.
+      code = nil
+      code = result_error["code"] if result_error
+      code ||= top_code if GhostCrawl::ErrorCodes.result_channel?(top_code)
+
+      # The flat markdown-build envelope reports a target failure ONLY via
+      # status="failed" (no ok/result_error) — don't count it as a success.
+      status_failed = hash["status"] == "failed"
+      code ||= top_code if status_failed && top_code.is_a?(String)
+
+      # Only raise when there is a concrete result-channel failure signal.
+      return hash unless code || ok_false || status_failed
+
+      # ok: false with no usable code -> treat as empty/unusable content.
+      code ||= GhostCrawl::ErrorCodes::EMPTY_CONTENT
+
+      retryable =
+        if result_error && result_error.key?("retryable")
+          result_error["retryable"]
+        else
+          GhostCrawl::ErrorCodes::RETRYABLE.fetch(code, false)
+        end
+
+      target_status = nil
+      target_status = result_error["target_status"] if result_error
+      target_status ||= hash["target_status"]  # flat markdown-envelope path
+      reason = result_error && result_error["reason"]
+
+      msg = "scrape failed (#{code})"
+      msg += ": #{reason}" if reason && !reason.to_s.empty?
+      msg += " (target HTTP #{target_status})" if target_status
+
+      raise GhostCrawl::ScrapeError.new(
+        msg,
+        status_code: 200,
+        body: nil,
+        code: code,
+        retryable: retryable,
+        request_id: hash["request_id"],
+        target_status: target_status
+      )
+    end
+  end
+
+  # ---------------------------------------------------------------------------
+  # Sub-clients — each delegates to the generated v1 request builders
+  # ---------------------------------------------------------------------------
+
+  # Manage crawl runs — /v1/crawl-runs.
+  class CrawlRunsClient
+    # Terminal run states — a crawl run in any of these will never change again,
+    # so a wait can stop. Both spellings of "cancelled" are accepted.
+    TERMINAL_STATUSES = %w[completed failed cancelled canceled].freeze
+
+    # Per-request server-block window (seconds) for the long-poll wait. Each
+    # +GET ...?wait=true&timeout_s=WAIT_WINDOW_S+ makes the SERVER block for up
+    # to this long, so the client never sleeps between checks. Kept well under
+    # the client read timeout ({Client::DEFAULT_TIMEOUT}) so a single blocking
+    # request can never trip the transport timeout.
+    WAIT_WINDOW_S = 30
+
+    def initialize(v1, adapter = nil)
+      @v1 = v1
+      @adapter = adapter
+    end
+
+    # Start a new crawl run from a seed URL.
+    # Delegates to POST /v1/crawl-runs via the generated CrawlRunsRequestBuilder.
+    # The endpoint is a tagged union: a start request requires +action: "start"+
+    # and a +seed_urls+ array (not a bare +url+).
+    #
+    # Pass +wait: true+ to block until the run reaches a terminal state
+    # (completed | failed | cancelled) or +timeout+ seconds elapse. This sends
+    # +wait_until: "completed"+ so the SERVER blocks — no client-side poll loop,
+    # no +sleep+. When the run is still running at +timeout+ the current
+    # (non-terminal) record is returned; call {#wait_for_completion} again to
+    # keep waiting.
+    #
+    # @param url [String] seed URL
+    # @param max_depth [Integer] maximum crawl depth (default 2)
+    # @param max_pages [Integer] maximum pages (default 100)
+    # @param wait [Boolean] block until the run is terminal (default false)
+    # @param timeout [Integer] total seconds to wait when +wait: true+ (default 300)
+    # @param raise_on_result_error [Boolean] raise {GhostCrawl::ScrapeError} on a
+    #   target-side (HTTP-200) failure instead of returning the raw hash (default true).
+    #   Ignored on the +wait: true+ path, which always returns the terminal run so
+    #   the caller can inspect a +failed+ status.
+    # @return [Hash] crawl run record with +run_id+ and +status+ (results present
+    #   when it completed while waiting)
+    def start(url:, max_depth: 2, max_pages: 100, wait: false, timeout: 300,
+              raise_on_result_error: true, **opts)
+      data = { "action" => "start", "seed_urls" => [url],
+               "max_depth" => max_depth, "max_pages" => max_pages }
+             .merge(opts.transform_keys(&:to_s))
+
+      unless wait
+        hash = ResponseHelper.to_hash(@v1.crawl_runs.post(AdditionalDataBody.new(data)))
+        return raise_on_result_error ? ResponseHelper.raise_on_result_error!(hash) : hash
+      end
+
+      # Start-and-wait: ask the server to block until terminal. Bound the first
+      # server block to a safe window, then long-poll the rest via GET so a low
+      # per-request read timeout can never abort the whole wait.
+      window = [timeout, WAIT_WINDOW_S].min
+      data["wait_until"] = "completed"
+      data["timeout_s"]  = window
+      hash = ResponseHelper.to_hash(@v1.crawl_runs.post(AdditionalDataBody.new(data)))
+      return hash if terminal?(hash)
+
+      run_id = hash["run_id"]
+      remaining = timeout - window
+      return hash if run_id.nil? || run_id.to_s.empty? || remaining <= 0
+      wait_for_completion(run_id, timeout: remaining)
+    end
+
+    # Block until an existing crawl run reaches a terminal state, or +timeout+
+    # seconds elapse. Event-driven: each iteration issues a SERVER-blocking
+    # +GET /v1/crawl-runs/{run_id}?wait=true&timeout_s=N+ that returns the moment
+    # the run goes terminal (or after its own window). There is no client +sleep+
+    # — the wait cost lives on the server, and successive blocking windows are
+    # chained until the caller's +timeout+ deadline.
+    #
+    # On timeout the current non-terminal run is returned (never raises for a
+    # slow run); a terminal run is returned as soon as it is observed.
+    #
+    # @param run_id [String] the run to wait on
+    # @param timeout [Integer] total seconds to wait (default 300)
+    # @return [Hash] the run record (terminal when it finished in time)
+    def wait_for_completion(run_id, timeout: 300)
+      if run_id.nil? || run_id.to_s.empty?
+        raise ArgumentError, "run_id is required"
+      end
+
+      deadline = monotonic + timeout
+      loop do
+        remaining = deadline - monotonic
+        # Deadline reached — one final non-blocking read of the current state.
+        return get(run_id) if remaining <= 0
+
+        window = [remaining, WAIT_WINDOW_S].min
+        run = fetch_waiting(run_id, window)
+        return run if terminal?(run)
+        # The server just blocked for ~window seconds; loop straight into the
+        # next blocking window. No client-side delay.
+      end
+    end
+
+    # List crawl runs.
+    # Delegates to GET /v1/crawl-runs via the generated CrawlRunsRequestBuilder.
+    def list
+      ResponseHelper.to_hash(@v1.crawl_runs.get)
+    end
+
+    # Get a single crawl run by ID.
+    # Delegates to GET /v1/crawl-runs/{run_id} via the generated builder.
+    def get(run_id)
+      ResponseHelper.to_hash(@v1.crawl_runs.by_run_id(run_id).get)
+    end
+
+    # Cancel a running crawl run.
+    # Delegates to POST /v1/crawl-runs/{run_id}/cancel via the generated builder.
+    def cancel(run_id)
+      ResponseHelper.to_hash(@v1.crawl_runs.by_run_id(run_id).cancel.post)
+    end
+
+    private
+
+    # True when +run+ is a Hash carrying a terminal +status+.
+    def terminal?(run)
+      run.is_a?(Hash) && TERMINAL_STATUSES.include?(run["status"].to_s)
+    end
+
+    # Issues one SERVER-blocking GET on the run with +?wait=true&timeout_s=N+.
+    #
+    # The generated item builder's +to_get_request_information+ does not wire
+    # query parameters (its URL template has no query placeholder), so we take
+    # the request it builds, extend the template with +{?wait,timeout_s}+, and
+    # set the params before sending through the raw adapter — reusing the
+    # Binary (raw-JSON) response factory the generated builders already use.
+    def fetch_waiting(run_id, timeout_s)
+      request_info = @v1.crawl_runs.by_run_id(run_id).to_get_request_information(nil)
+      request_info.url_template = "{+baseurl}/v1/crawl-runs/{run_id}{?wait,timeout_s}"
+      request_info.query_parameters["wait"] = "true"
+      request_info.query_parameters["timeout_s"] = timeout_s.to_i.to_s
+      ResponseHelper.to_hash(@adapter.send_async(request_info, GhostCrawl::V1::Binary, {}))
+    end
+
+    # Monotonic clock (seconds) — immune to wall-clock adjustments during a wait.
+    def monotonic
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+  end
+
+  # Manage browser sessions — /v1/sessions.
+  class SessionsClient
+    def initialize(v1)
+      @v1 = v1
+    end
+
+    # List all active sessions.
+    # Delegates to GET /v1/sessions via the generated SessionsRequestBuilder.
+    def list
+      ResponseHelper.to_hash(@v1.sessions.get)
+    end
+
+    # Create a new browser session.
+    # Delegates to POST /v1/sessions/create via the generated builder.
+    # @param profile_name [String] identity profile to use
+    def create(profile_name:, **opts)
+      data = { "profile" => profile_name }.merge(opts.transform_keys(&:to_s))
+      ResponseHelper.to_hash(@v1.sessions.create.post(AdditionalDataBody.new(data)))
+    end
+
+    # Extend a session's TTL.
+    # Delegates to POST /v1/sessions/{id}/extend via the generated builder.
+    def extend(session_id, duration_seconds: 300)
+      ResponseHelper.to_hash(@v1.sessions.by_profile_id(session_id).extend.post(
+        AdditionalDataBody.new({ "ttl_seconds" => duration_seconds })
+      ))
+    end
+
+    # Release a session back to the pool.
+    # Delegates to POST /v1/sessions/{id}/release via the generated builder.
+    def release(session_id)
+      ResponseHelper.to_hash(@v1.sessions.by_profile_id(session_id).release.post)
+    end
+  end
+
+  # Manage identity profiles — /v1/profiles.
+  class ProfilesClient
+    def initialize(v1, adapter = nil)
+      @v1 = v1
+      @adapter = adapter
+    end
+
+    # List all profiles.
+    # Delegates to GET /v1/profiles via the generated ProfilesRequestBuilder.
+    def list
+      ResponseHelper.to_hash(@v1.profiles.get)
+    end
+
+    # Get a profile by name.
+    # Delegates to GET /v1/profiles/{name} via the generated builder.
+    def get(name)
+      ResponseHelper.to_hash(@v1.profiles.by_name(name).get)
+    end
+
+    # Create a new profile.
+    # Delegates to POST /v1/profiles via the generated ProfilesRequestBuilder.
+    def create(name:, **config)
+      body = AdditionalDataBody.new({ "name" => name }.merge(config.transform_keys(&:to_s)))
+      ResponseHelper.to_hash(@v1.profiles.post(body))
+    end
+
+    # Update a profile.
+    # Delegates to PUT /v1/profiles/{name} via the generated builder.
+    def update(name, **config)
+      ResponseHelper.to_hash(@v1.profiles.by_name(name).put(AdditionalDataBody.new(config.transform_keys(&:to_s))))
+    end
+
+    # Delete a profile.
+    # Delegates to DELETE /v1/profiles/{name} via the generated builder.
+    # The endpoint answers 204 No Content; routing through {ResponseHelper.void_request!}
+    # avoids the Kiota JSON parser choking on the empty body.
+    def delete(name)
+      ResponseHelper.void_request!(@adapter, @v1.profiles.by_name(name).to_delete_request_information(nil))
+    end
+  end
+
+  # Manage webhooks — /v1/webhooks.
+  class WebhooksClient
+    def initialize(v1, adapter = nil)
+      @v1 = v1
+      @adapter = adapter
+    end
+
+    # List all webhooks.
+    # Delegates to GET /v1/webhooks via the generated WebhooksRequestBuilder.
+    def list
+      ResponseHelper.to_hash(@v1.webhooks.get)
+    end
+
+    # Get a webhook by ID.
+    # Delegates to GET /v1/webhooks/{id} via the generated builder.
+    def get(webhook_id)
+      ResponseHelper.to_hash(@v1.webhooks.by_webhook_id(webhook_id).get)
+    end
+
+    # Register a new webhook endpoint.
+    # Delegates to POST /v1/webhooks via the generated WebhooksRequestBuilder.
+    def create(url:, event_types: nil, events: nil, **opts)
+      data = { "url" => url }.merge(opts.transform_keys(&:to_s))
+      # API field is "event_types"; "events" kept as a back-compat alias.
+      et = event_types.nil? ? events : event_types
+      data["event_types"] = et unless et.nil?
+      ResponseHelper.to_hash(@v1.webhooks.post(AdditionalDataBody.new(data)))
+    end
+
+    # Delete a webhook.
+    # Delegates to DELETE /v1/webhooks/{id}. The generated builder passes a +nil+
+    # response factory (204 void), which the Kiota adapter rejects with a bare
+    # StandardError BEFORE sending the request — so we build the request info and
+    # run it through {ResponseHelper.void_request!} instead, which actually fires
+    # the DELETE and returns +{}+ on success.
+    def delete(webhook_id)
+      ResponseHelper.void_request!(@adapter, @v1.webhooks.by_webhook_id(webhook_id).to_delete_request_information(nil))
+    end
+
+    # Rotate the signing secret for a webhook.
+    # Delegates to POST /v1/webhooks/{id}/rotate-secret via the generated builder.
+    def rotate_secret(webhook_id)
+      ResponseHelper.to_hash(@v1.webhooks.by_webhook_id(webhook_id).rotate_secret.post)
+    end
+  end
+
+  # Manage schedules — /v1/schedules.
+  class SchedulesClient
+    def initialize(v1, adapter = nil)
+      @v1 = v1
+      @adapter = adapter
+    end
+
+    # List all schedules.
+    # Delegates to GET /v1/schedules via the generated SchedulesRequestBuilder.
+    def list
+      ResponseHelper.to_hash(@v1.schedules.get)
+    end
+
+    # Get a schedule by ID.
+    # Delegates to GET /v1/schedules/{id} via the generated builder.
+    def get(schedule_id)
+      ResponseHelper.to_hash(@v1.schedules.by_schedule_id(schedule_id).get)
+    end
+
+    # Create a new schedule.
+    # Delegates to POST /v1/schedules via the generated SchedulesRequestBuilder.
+    #
+    # The API's ScheduleCreateRequest requires +name+, +job_type+
+    # ("scrape" | "crawl" | "change_monitor"), +cron_expr+, and +job_params+
+    # (the full scrape/crawl request body). It rejects any other top-level field
+    # with a 422 — so a legacy +task+ key must be TRANSLATED, never forwarded.
+    #
+    # @param cron [String] cron expression (sent as +cron_expr+)
+    # @param name [String] schedule name (required by the API)
+    # @param job_type [String, nil] "scrape" | "crawl" | "change_monitor"
+    # @param job_params [Hash, nil] full job request body (e.g. { url: ... })
+    # @param task [Hash, nil] DEPRECATED legacy shape { "action" => ..., ...rest };
+    #   when given (and job_type/job_params are absent) it is split into
+    #   +job_type+ (from +task["action"]+) and +job_params+ (the remaining keys).
+    def create(cron:, name: nil, job_type: nil, job_params: nil, task: nil, **opts)
+      # Back-compat: derive job_type/job_params from a legacy `task` hash.
+      if task.is_a?(Hash)
+        t = task.transform_keys(&:to_s)
+        job_type ||= t["action"] || t["job_type"] || t["type"]
+        job_params ||= t.reject { |k, _| %w[action job_type type].include?(k) }
+      end
+
+      data = { "cron_expr" => cron }
+      data["name"]       = name       unless name.nil?
+      data["job_type"]   = job_type   unless job_type.nil?
+      data["job_params"] = job_params unless job_params.nil?
+      # opts may override/supply name/job_type/job_params/notify_webhook/monitor_mode.
+      # Never forward a raw `task` field — the API 422s on it.
+      data.merge!(opts.transform_keys(&:to_s).reject { |k, _| k == "task" })
+
+      ResponseHelper.to_hash(@v1.schedules.post(AdditionalDataBody.new(data)))
+    end
+
+    # Delete a schedule.
+    # Delegates to DELETE /v1/schedules/{id}. The generated builder returns a Fiber
+    # that must be resumed to actually send the request (the old +; {}+ discarded it,
+    # so the DELETE never fired). {ResponseHelper.void_request!} runs it and tolerates
+    # the empty 204 body.
+    def delete(schedule_id)
+      ResponseHelper.void_request!(@adapter, @v1.schedules.by_schedule_id(schedule_id).to_delete_request_information(nil))
+    end
+  end
+
+  # Manage datasets — /v1/datasets.
+  class DatasetsClient
+    def initialize(v1)
+      @v1 = v1
+    end
+
+    # List all datasets.
+    # Delegates to GET /v1/datasets via the generated DatasetsRequestBuilder.
+    def list
+      ResponseHelper.to_hash(@v1.datasets.get)
+    end
+
+    # Get a dataset by name.
+    # Delegates to GET /v1/datasets/{name} via the generated builder.
+    def get(name)
+      ResponseHelper.to_hash(@v1.datasets.by_name(name).get)
+    end
+
+    # Create a new dataset.
+    # Delegates to POST /v1/datasets via the generated DatasetsRequestBuilder.
+    def create(name:, **opts)
+      body = AdditionalDataBody.new({ "name" => name }.merge(opts.transform_keys(&:to_s)))
+      ResponseHelper.to_hash(@v1.datasets.post(body))
+    end
+
+    # Delete a dataset.
+    # Delegates to DELETE /v1/datasets/{name} via the generated builder.
+    def delete(name)
+      ResponseHelper.to_hash(@v1.datasets.by_name(name).delete)
+    end
+
+    # Get rows from a dataset.
+    # Delegates to GET /v1/datasets/{name}/rows via the generated builder.
+    def rows(name)
+      ResponseHelper.to_hash(@v1.datasets.by_name(name).rows.get)
+    end
+
+    # Append rows to a dataset.
+    # Delegates to POST /v1/datasets/{name}/rows/append via the generated builder.
+    def append(name, rows)
+      body = AdditionalDataBody.new({ "rows" => rows })
+      ResponseHelper.to_hash(@v1.datasets.by_name(name).rows.append.post(body))
+    end
+  end
+
+  # Manage session recordings — /v1/recordings.
+  class RecordingsClient
+    def initialize(v1, adapter = nil)
+      @v1 = v1
+      @adapter = adapter
+    end
+
+    # List all recordings.
+    # Delegates to GET /v1/recordings via the generated RecordingsRequestBuilder.
+    def list
+      ResponseHelper.to_hash(@v1.recordings.get)
+    end
+
+    # Get a recording by ID.
+    # Delegates to GET /v1/recordings/{id} via the generated builder.
+    # (The generated accessor is +by_recording_id+, single underscore.)
+    def get(recording_id)
+      ResponseHelper.to_hash(@v1.recordings.by_recording_id(recording_id).get)
+    end
+
+    # Delete a recording.
+    # Delegates to DELETE /v1/recordings/{id}. Same void-DELETE defect as webhooks
+    # (generated builder passes a +nil+ factory → adapter raises before sending), so
+    # we route through {ResponseHelper.void_request!}.
+    def delete(recording_id)
+      ResponseHelper.void_request!(@adapter, @v1.recordings.by_recording_id(recording_id).to_delete_request_information(nil))
+    end
+  end
+
+  # Key-value store — /v1/kv.
+  class KVClient
+    def initialize(v1)
+      @v1 = v1
+    end
+
+    # Get a value by key.
+    # Delegates to GET /v1/kv/{key} via the generated KvRequestBuilder.
+    def get(key)
+      ResponseHelper.to_hash(@v1.kv.by_key(key).get)
+    end
+
+    # Set a key-value pair.
+    # Delegates to PUT /v1/kv/{key} via the generated builder.
+    def set(key, value)
+      body = AdditionalDataBody.new({ "value" => value })
+      ResponseHelper.to_hash(@v1.kv.by_key(key).put(body))
+    end
+
+    # Delete a key.
+    # Delegates to DELETE /v1/kv/{key} via the generated builder.
+    def delete(key)
+      ResponseHelper.to_hash(@v1.kv.by_key(key).delete)
+    end
+  end
+
+  # ---------------------------------------------------------------------------
+  # Main facade — Client
+  # ---------------------------------------------------------------------------
+
+  # GhostCrawl idiomatic API client.
+  #
+  # Delegates all HTTP transport, URL routing, serialization, and auth to the
+  # Kiota-generated canonical core (_generated/). This facade is the shipped API.
+  #
+  # @example
+  #   require "ghostcrawl"
+  #   client = GhostCrawl::Client.new(token: "gck_live_YOUR_KEY")
+  #   result = client.scrape(url: "https://example.com")
+  class Client
+    # Default read timeout (seconds). Browser-rendered scrapes/crawls are slow,
+    # and the underlying net/http default (60s) is too short for them.
+    DEFAULT_TIMEOUT = 300
+
+    # @param token [String, nil] API key. Falls back to +GHOSTCRAWL_API_KEY+ env var.
+    # @param base_url [String, nil] Override API base URL. Falls back to +GHOSTCRAWL_BASE_URL+ env var.
+    # @param timeout [Integer, nil] Per-request read timeout in seconds. Falls back
+    #   to +GHOSTCRAWL_TIMEOUT+ env var, then {DEFAULT_TIMEOUT}.
+    def initialize(token: nil, base_url: nil, timeout: nil)
+      resolved_token = token || ENV.fetch("GHOSTCRAWL_API_KEY", nil)
+      if resolved_token.nil? || resolved_token.empty?
+        raise ArgumentError,
+              "token is required — pass token: or set GHOSTCRAWL_API_KEY. " \
+              "Get your key at https://ghostcrawl.io"
+      end
+
+      resolved_base = (base_url ||
+                       ENV.fetch("GHOSTCRAWL_BASE_URL", nil) ||
+                       DEFAULT_BASE_URL).gsub(%r{/+$}, "")
+
+      resolved_timeout = (timeout ||
+                          ENV.fetch("GHOSTCRAWL_TIMEOUT", nil) ||
+                          DEFAULT_TIMEOUT).to_i
+
+      # Build the Kiota core via BaseBearerTokenAuthenticationProvider + FaradayRequestAdapter.
+      # All HTTP, auth, serialization, and URL routing delegate to the generated core.
+      auth_provider = MicrosoftKiotaAbstractions::BaseBearerTokenAuthenticationProvider.new(
+        StaticTokenProvider.new(resolved_token)
+      )
+      adapter = MicrosoftKiotaFaraday::FaradayRequestAdapter.new(auth_provider)
+      adapter.set_base_url(resolved_base)
+
+      # The default Faraday connection sets no timeout, so it inherits net/http's
+      # 60s read timeout — too short for browser-rendered work. Raise it.
+      if adapter.client.respond_to?(:options) && resolved_timeout.positive?
+        adapter.client.options.timeout = resolved_timeout
+        adapter.client.options.open_timeout = 30
+      end
+
+      @core    = GhostCrawl::GhostCrawlClient.new(adapter)
+      @v1      = @core.v1
+      # Kept for the raw-adapter dispatch paths (void DELETEs, binary PDF/
+      # screenshot, the long-poll crawl-run wait, and the ad-hoc JSON routes
+      # the generated core has no builder for).
+      @adapter = adapter
+
+      # CrawlRunsClient needs the raw adapter for the event-driven long-poll wait
+      # (the generated item builder can't attach ?wait/&timeout_s query params).
+      @crawl_runs  = CrawlRunsClient.new(@v1, @adapter)
+      @sessions    = SessionsClient.new(@v1)
+      # Sub-clients with void-DELETE endpoints (204 No Content) also need the raw
+      # adapter so ResponseHelper.void_request! can work around the broken generated
+      # delete builders. See ResponseHelper.void_request! for the two Kiota defects.
+      @profiles    = ProfilesClient.new(@v1, @adapter)
+      @webhooks    = WebhooksClient.new(@v1, @adapter)
+      @schedules   = SchedulesClient.new(@v1, @adapter)
+      @datasets    = DatasetsClient.new(@v1)
+      @recordings  = RecordingsClient.new(@v1, @adapter)
+      @kv          = KVClient.new(@v1)
+    end
+
+    # @return [CrawlRunsClient]
+    attr_reader :crawl_runs
+    # @return [SessionsClient]
+    attr_reader :sessions
+    # @return [ProfilesClient]
+    attr_reader :profiles
+    # @return [WebhooksClient]
+    attr_reader :webhooks
+    # @return [SchedulesClient]
+    attr_reader :schedules
+    # @return [DatasetsClient]
+    attr_reader :datasets
+    # @return [RecordingsClient]
+    attr_reader :recordings
+    # @return [KVClient]
+    attr_reader :kv
+
+    # ---------------------------------------------------------------------------
+    # Top-level facade methods — delegate to generated builders
+    # ---------------------------------------------------------------------------
+
+    # Scrape a single URL and return the rendered content.
+    # Delegates to POST /v1/scrape via the generated ScrapeRequestBuilder.
+    # @param url [String] target URL
+    # @param format [String] output format: "markdown" (default), "html", "text"
+    # @param engine [String] browser engine: "auto" (default), "chrome", "firefox", "webkit"
+    # @param javascript [Boolean] enable JavaScript rendering (default true)
+    # @param extract_schema [Hash, nil] JSON Schema for structured extraction
+    # @param raise_on_result_error [Boolean] raise {GhostCrawl::ScrapeError} on a
+    #   target-side (HTTP-200) failure instead of returning the raw hash (default true)
+    # @return [Hash] response with +content+, +markdown+, +status+, and other fields
+    def scrape(url:, format: "markdown", engine: "auto", javascript: true, extract_schema: nil,
+               raise_on_result_error: true, **opts)
+      # Use AdditionalDataBody to send only the fields we specify — the generated
+      # ScrapeRequest model would serialize typed defaults (nulls + empty enums) that
+      # cause 422 validation errors on the server.
+      data = { "url" => url, "format" => format, "engine" => engine,
+               "javascript_enabled" => javascript }.merge(opts.transform_keys(&:to_s))
+      data["extract_schema"] = extract_schema unless extract_schema.nil?
+      hash = ResponseHelper.to_hash(@v1.scrape.post(AdditionalDataBody.new(data)))
+      normalize_scrape_content(hash)
+      raise_on_result_error ? ResponseHelper.raise_on_result_error!(hash) : hash
+    end
+
+    # Search the web and return results.
+    # Delegates to POST /v1/search via the generated SearchRequestBuilder.
+    #
+    # /v1/search requires your own search-backend API key (BYO; GhostCrawl
+    # charges no markup). Pass it as +provider_key+ — it is sent as the
+    # +X-Provider-Authorization: Bearer <provider_key>+ header the backend
+    # requires. Without it the API replies 401 search_backend_key_missing.
+    # @param query [String] search query
+    # @param engine [String] search engine: "google" (default), "bing", "duckduckgo"
+    # @param limit [Integer] maximum results (default 10)
+    # @param provider_key [String, nil] BYO search-backend key (sent as X-Provider-Authorization)
+    # @return [Hash] response with +results+ list
+    def search(query:, engine: "google", limit: 10, provider_key: nil, **opts)
+      data = { "query" => query, "engine" => engine,
+               "limit" => limit }.merge(opts.transform_keys(&:to_s))
+      config = nil
+      unless provider_key.nil?
+        config = MicrosoftKiotaAbstractions::RequestConfiguration.new
+        headers = MicrosoftKiotaAbstractions::RequestHeaders.new
+        headers.add("X-Provider-Authorization", "Bearer #{provider_key}")
+        config.headers = headers
+      end
+      ResponseHelper.to_hash(@v1.search.post(AdditionalDataBody.new(data), config))
+    end
+
+    # Extract structured data from a URL using a JSON Schema.
+    # Delegates to POST /v1/extract via the generated ExtractRequestBuilder.
+    # @param url [String] target URL
+    # @param schema [Hash] JSON Schema describing the shape to extract
+    # @param raise_on_result_error [Boolean] raise {GhostCrawl::ScrapeError} on a
+    #   target-side (HTTP-200) failure instead of returning the raw hash (default true)
+    # @return [Hash] extracted data
+    def extract(url:, schema:, raise_on_result_error: true, **opts)
+      data = { "url" => url, "schema" => schema }.merge(opts.transform_keys(&:to_s))
+      hash = ResponseHelper.to_hash(@v1.extract.post(AdditionalDataBody.new(data)))
+      raise_on_result_error ? ResponseHelper.raise_on_result_error!(hash) : hash
+    end
+
+    # Start a deep crawl from a seed URL.
+    # Delegates to POST /v1/crawl/deep via the generated CrawlDeepRequestBuilder.
+    #
+    # Pass +wait: true+ to block until the run is terminal. The deep-crawl start
+    # returns a +run_id+, then the wait is handled by the same event-driven
+    # server-blocking long-poll as {CrawlRunsClient#wait_for_completion} — no
+    # client-side +sleep+ loop.
+    #
+    # @param url [String] seed URL
+    # @param max_depth [Integer] maximum crawl depth (default 2)
+    # @param max_pages [Integer] maximum pages (default 100)
+    # @param wait [Boolean] block until the run is terminal (default false)
+    # @param timeout [Integer] total seconds to wait when +wait: true+ (default 300)
+    # @param raise_on_result_error [Boolean] raise {GhostCrawl::ScrapeError} on a
+    #   target-side (HTTP-200) failure instead of returning the raw hash (default true).
+    #   Ignored on the +wait: true+ path, which returns the terminal run.
+    # @return [Hash] crawl run record (terminal when +wait: true+ and it finished in time)
+    def crawl(url:, max_depth: 2, max_pages: 100, wait: false, timeout: 300,
+              raise_on_result_error: true, **opts)
+      data = { "seed_urls" => [url], "max_depth" => max_depth,
+               "max_urls" => max_pages }.merge(opts.transform_keys(&:to_s))
+      hash = ResponseHelper.to_hash(@v1.crawl.deep.post(AdditionalDataBody.new(data)))
+
+      unless wait
+        return raise_on_result_error ? ResponseHelper.raise_on_result_error!(hash) : hash
+      end
+
+      run_id = hash["run_id"]
+      return hash if run_id.nil? || run_id.to_s.empty?
+      @crawl_runs.wait_for_completion(run_id, timeout: timeout)
+    end
+
+    # Map all URLs reachable from a seed URL.
+    # Delegates to POST /v1/map via the generated MapRequestBuilder.
+    # @param url [String] seed URL
+    # @return [Hash] response with +urls+ list
+    def map(url:, **opts)
+      data = { "url" => url }.merge(opts.transform_keys(&:to_s))
+      ResponseHelper.to_hash(@v1.map.post(AdditionalDataBody.new(data)))
+    end
+
+    # Render a URL to a PDF document and return the raw +application/pdf+ bytes.
+    # Delegates to POST /v1/pdf.
+    #
+    # +/v1/pdf+ responds with binary, not a JSON envelope, so this returns the
+    # bytes verbatim (an ASCII-8BIT String) — write them straight to a file:
+    #
+    #   data = client.pdf(url: "https://example.com")
+    #   File.binwrite("page.pdf", data)
+    #
+    # PDF output is Chrome-only; a request that resolves to a Firefox or WebKit
+    # identity is rejected with 400 +pdf_engine_unsupported+ (a {GhostCrawlError}).
+    #
+    # +/v1/pdf+ has no generated request builder, so this hand-builds a
+    # {MicrosoftKiotaAbstractions::RequestInformation} and routes it through the
+    # SAME adapter (base URL + bearer auth + transport) as the modeled calls.
+    #
+    # @param url [String] target URL to render
+    # @param paper_format [String] page size: "a4" (default), "letter", "legal", "tabloid"
+    # @param landscape [Boolean] render in landscape orientation (default false)
+    # @param engine [String] browser engine (PDF is Chrome-only; default "auto")
+    # @return [String] the raw PDF bytes (ASCII-8BIT)
+    def pdf(url:, paper_format: "a4", landscape: false, engine: "auto", **opts)
+      data = { "url" => url, "paper_format" => paper_format,
+               "landscape" => landscape, "engine" => engine }.merge(opts.transform_keys(&:to_s))
+      request_info = MicrosoftKiotaAbstractions::RequestInformation.new
+      request_info.http_method = :POST
+      request_info.url_template = "{+baseurl}/v1/pdf"
+      request_info.headers.try_add("Accept", "application/pdf")
+      request_info.set_stream_content(JSON.generate(data), "application/json")
+      ResponseHelper.binary_request!(@adapter, request_info)
+    end
+
+    # Capture a screenshot of a URL and return the raw +image/png+ bytes.
+    # Delegates to POST /v1/screenshot.
+    #
+    # +/v1/screenshot+ responds with a binary image, not a JSON envelope, so this
+    # returns the bytes verbatim (an ASCII-8BIT String) — write them straight to a
+    # file:
+    #
+    #   data = client.screenshot(url: "https://example.com")
+    #   File.binwrite("page.png", data)
+    #
+    # +/v1/screenshot+ has no generated request builder, so this hand-builds a
+    # {MicrosoftKiotaAbstractions::RequestInformation} and routes it through the
+    # SAME adapter (base URL + bearer auth + transport) as the modeled calls,
+    # mirroring {#pdf} exactly (the raw-bytes dispatch idiom).
+    #
+    # @param url [String] target URL to capture
+    # @param format [String] image format: "png" (default), "jpeg", "webp"
+    # @param full_page [Boolean] capture the full scrollable page (default false)
+    # @param screenshot_selector [String, nil] CSS selector to clip to (optional)
+    # @param engine [String] browser engine (default "auto")
+    # @return [String] the raw image bytes (ASCII-8BIT)
+    def screenshot(url:, format: "png", full_page: false, screenshot_selector: nil,
+                   engine: "auto", **opts)
+      data = { "url" => url, "format" => format, "full_page" => full_page,
+               "engine" => engine }
+      data["screenshot_selector"] = screenshot_selector unless screenshot_selector.nil?
+      data.merge!(opts.transform_keys(&:to_s))
+      request_info = MicrosoftKiotaAbstractions::RequestInformation.new
+      request_info.http_method = :POST
+      request_info.url_template = "{+baseurl}/v1/screenshot"
+      request_info.headers.try_add("Accept", "image/png")
+      request_info.set_stream_content(JSON.generate(data), "application/json")
+      ResponseHelper.binary_request!(@adapter, request_info)
+    end
+
+    # Render a URL and return the rendered-content JSON envelope as a Hash.
+    # Delegates to POST /v1/content.
+    #
+    # +/v1/content+ responds with +application/json+ ({+content+, +url+, +status+,
+    # +format+, +status_code+, +bytes+}); this returns the decoded Hash. It uses
+    # the same hand-built {MicrosoftKiotaAbstractions::RequestInformation} +
+    # shared adapter dispatch as {#pdf}, parsing the JSON body at the end.
+    #
+    # @param url [String] target URL to render
+    # @param engine [String] browser engine (default "auto")
+    # @return [Hash] the rendered-content envelope (top-level +content+ = HTML)
+    def content(url:, engine: "auto", **opts)
+      data = { "url" => url, "engine" => engine }.merge(opts.transform_keys(&:to_s))
+      request_info = MicrosoftKiotaAbstractions::RequestInformation.new
+      request_info.http_method = :POST
+      request_info.url_template = "{+baseurl}/v1/content"
+      request_info.headers.try_add("Accept", "application/json")
+      request_info.set_stream_content(JSON.generate(data), "application/json")
+      bytes = ResponseHelper.binary_request!(@adapter, request_info)
+      parsed = JSON.parse(bytes)
+      parsed.is_a?(Hash) ? parsed : { "content" => parsed }
+    end
+
+    # Execute an agent task via POST /v1/agent.
+    #
+    # The agent capability is gated per account; when it is not enabled the API
+    # replies +404 not_found+ — this returns that problem+json body as a Hash
+    # (carrying +"detail"+) rather than raising, so callers can branch on
+    # +result.key?("detail")+. The agent brings its own LLM (BYO) — this method
+    # handles the client serialize→POST→parse plumbing against the real route.
+    #
+    # +/v1/agent+ has no generated request builder, so this hand-builds a
+    # {MicrosoftKiotaAbstractions::RequestInformation} and routes it through the
+    # SAME adapter (base URL + bearer auth + transport) as the modeled calls.
+    #
+    # @param url [String, nil] optional starting URL (folded into +task+ as +start_url+)
+    # @param instruction [String, nil] natural-language task instruction
+    # @param task [Hash, nil] explicit structured task object (overrides url/instruction)
+    # @return [Hash] the agent result, or the capability-gated 404 problem+json body (has +"detail"+)
+    def agent(url: nil, instruction: nil, task: nil, **opts)
+      payload = task || { "instruction" => instruction, "start_url" => url }
+      data = { "task" => payload }.merge(opts.transform_keys(&:to_s))
+      request_info = MicrosoftKiotaAbstractions::RequestInformation.new
+      request_info.http_method = :POST
+      request_info.url_template = "{+baseurl}/v1/agent"
+      request_info.headers.try_add("Accept", "application/json")
+      request_info.set_stream_content(JSON.generate(data), "application/json")
+      bytes = ResponseHelper.binary_request!(@adapter, request_info)
+      parsed = JSON.parse(bytes)
+      parsed.is_a?(Hash) ? parsed : { "detail" => parsed }
+    rescue GhostCrawl::GhostCrawlError => e
+      # Agent is account-gated: a 404 not_found is the documented "capability
+      # disabled" answer, not a transport failure. Surface the problem+json body
+      # as a Hash (carrying "detail") rather than raising. Any other status is a
+      # real error and is re-raised.
+      raise unless e.status_code == 404
+      body = e.body.to_s
+      brace = body.index("{")
+      decoded =
+        if brace
+          begin
+            JSON.parse(body[brace..])
+          rescue StandardError
+            nil
+          end
+        end
+      decoded.is_a?(Hash) ? decoded : { "detail" => e.message, "code" => (e.code || "not_found") }
+    end
+
+    # List the account's persisted session storage-states.
+    # Delegates to GET /v1/storage-states.
+    # @return [Hash] the storage-states envelope
+    def storage_states
+      json_request(:GET, "/v1/storage-states", nil)
+    end
+
+    private
+
+    # Route an ad-hoc JSON request through the SAME adapter (base URL + bearer
+    # auth + transport) as the modeled calls, for routes the generated core has
+    # no builder for. Returns the decoded Hash. Mirrors {#content}'s dispatch.
+    # @api private
+    def json_request(method, path, body)
+      request_info = MicrosoftKiotaAbstractions::RequestInformation.new
+      request_info.http_method = method
+      request_info.url_template = "{+baseurl}#{path}"
+      request_info.headers.try_add("Accept", "application/json")
+      if !body.nil? && method != :GET
+        request_info.set_stream_content(JSON.generate(body), "application/json")
+      end
+      bytes = ResponseHelper.binary_request!(@adapter, request_info)
+      parsed = (JSON.parse(bytes) if bytes && !bytes.empty?)
+      parsed.is_a?(Hash) ? parsed : {}
+    end
+
+    # Normalizes a +"content"+ key onto a decoded scrape response.
+    #
+    # The API has two scrape response shapes:
+    #   * the FLAT +format="markdown"+ shape, which carries the rendered page at
+    #     the top level under the format-specific key (+"markdown"+/+"html"+/
+    #     +"text"+) but omits +identity_id+; and
+    #   * the standard ENVELOPE ({ +status+, +results+, +routing_mode+,
+    #     +request_class+, +identity_id+ }), where the rendered page lives under
+    #     +results[0]+ and the top-level +identity_id+ is present.
+    # The documented quickstart reads +result["content"]+, so this mirrors the
+    # rendered page onto +"content"+ in place from WHICHEVER shape is present,
+    # KEEPING the original keys intact (backward compatible).
+    #
+    # No-op unless +result+ is a Hash that does not already carry +"content"+.
+    # The value chosen is: the field named by +result["format"]+ when that field
+    # is a String, else the first String among +"markdown"+, +"html"+, +"text"+ —
+    # looked up first at the top level (flat shape), then on +results[0]+
+    # (envelope shape).
+    #
+    # @param result [Object] the decoded response (only mutated when a Hash)
+    # @return [Object] the same +result+, unchanged reference
+    # @api private
+    def normalize_scrape_content(result)
+      return result unless result.is_a?(Hash) && !result.key?("content")
+
+      fmt = result["format"]
+      value = scrape_content_from(result, fmt)
+
+      # Envelope shape: the page is under results[0], not at the top level.
+      unless value.is_a?(String)
+        first = result["results"]
+        first = first.first if first.is_a?(Array)
+        value = scrape_content_from(first, fmt) if first.is_a?(Hash)
+      end
+
+      result["content"] = value if value.is_a?(String)
+      result
+    end
+
+    # Picks the rendered-page String out of a scrape result Hash: the field named
+    # by +fmt+ when present, else the first String among +"markdown"+, +"html"+,
+    # +"text"+. Returns +nil+ when +src+ carries no such String.
+    # @api private
+    def scrape_content_from(src, fmt)
+      return nil unless src.is_a?(Hash)
+
+      value = src[fmt] if fmt.is_a?(String)
+      return value if value.is_a?(String)
+
+      %w[markdown html text].each do |key|
+        candidate = src[key]
+        return candidate if candidate.is_a?(String)
+      end
+      nil
+    end
+  end
+end