ghostcrawl 2.2.3 → 2.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e499bc41b516f5cea6e37d2ec06c17e8fe770a8d928e8415e470be0874d56786
-  data.tar.gz: ed4d7fa29dcdc4f1316d0f870555bf32cb61063b3bc7efc07fe46e5682626a50
+  metadata.gz: 9b83f5c591c7a3ffd859db07f96c9725f84db05826b65e9227eadce1ff3eded3
+  data.tar.gz: 0aa598b5c6d776b5477c0709aa26dd3789d5f79084421b1fae802e90892c4dc0
 SHA512:
-  metadata.gz: 349f33fe5785ed3458e5e749b7a45e56bf6e9aa8fdcc4b68625792a174676733ac1d0bed6f6148bed07716059498f5f5e565bf4d013357af1d8d4632f5d9ffb3
-  data.tar.gz: 7939615bd575ac30e78455194a2979f204d82523b548ede855da34ff06f1422f8e8f676f44fe7ad47262ad63f03b6ef68c39acdc190f9dcb981c06e39717191f
+  metadata.gz: 3458805483c931bbd9adca163e74d0610c7d02abdb6ca260459fbdbe162c573f29752846d92bf45189a8fcea7957f770588364c9c483082729386c8b2d7c281a
+  data.tar.gz: 8d56532b83ea407f5a8e9190832fe1fd8ae3327cd427cb3f26be0fd727c20567a5bc1331998caa403d622edd7456956b5a373ae1961d1281f336b741ea607b32

data/LICENSE

@@ -1,46 +1,21 @@
-GhostCrawl Software License
-
-Copyright (c) 2026 GhostCrawl, LLC. All rights reserved.
-
-This software — including the GhostCrawl SDKs, command-line interface, example
-code, documentation, and the GhostCrawl self-host image (collectively, the
-"Software") — is the proprietary property of GhostCrawl, LLC ("GhostCrawl") and
-its licensors. It is licensed, not sold.
-
-1. LICENSE GRANT. Subject to your continuous compliance with this License and the
-   GhostCrawl Terms of Service (https://docs.ghostcrawl.io/legal/terms),
-   GhostCrawl grants you a limited, revocable, non-exclusive, non-transferable,
-   and non-sublicensable license to install and use the Software, including
-   within your own applications, solely to access and use the GhostCrawl service
-   under a valid GhostCrawl account.
-
-2. RESTRICTIONS. Except as expressly permitted in Section 1, you may not, in
-   whole or in part: (a) copy the Software except as strictly necessary to use it
-   as permitted; (b) modify, adapt, translate, or create derivative works of the
-   Software; (c) distribute, publish, sublicense, sell, resell, rent, lease,
-   lend, or otherwise transfer the Software, or make it available to any third
-   party, including as a hosted or managed service; (d) reverse engineer,
-   decompile, disassemble, or otherwise attempt to derive the source code,
-   structure, or internal design of any compiled component of the Software;
-   (e) circumvent, disable, or tamper with any license, authentication,
-   usage-metering, watermarking, or security mechanism; or (f) remove, alter, or
-   obscure any proprietary, copyright, or attribution notice.
-
-3. OWNERSHIP. GhostCrawl and its licensors retain all right, title, and interest
-   in and to the Software and all intellectual property rights therein. No rights
-   are granted to you except as expressly set out in this License. All rights
-   reserved.
-
-4. TERMINATION. This License terminates automatically, without notice, if you
-   breach any of its terms or cease to maintain a valid GhostCrawl account where
-   one is required. Upon termination you must cease all use of the Software and
-   destroy all copies in your possession or control.
-
-5. NO WARRANTY; LIMITATION OF LIABILITY. THE SOFTWARE IS PROVIDED "AS IS" WITHOUT
-   WARRANTY OF ANY KIND, EXPRESS OR IMPLIED. TO THE MAXIMUM EXTENT PERMITTED BY
-   LAW, GHOSTCRAWL AND ITS LICENSORS WILL NOT BE LIABLE FOR ANY CLAIM, DAMAGES, OR
-   OTHER LIABILITY ARISING FROM OR IN CONNECTION WITH THE SOFTWARE OR ITS USE. The
-   GhostCrawl Terms of Service and Acceptable Use Policy
-   (https://docs.ghostcrawl.io/legal/terms) are incorporated by reference.
-
-For licensing inquiries, contact legal@ghostcrawl.io.
+MIT License
+
+Copyright (c) 2026 GhostCrawl, LLC
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -29,9 +29,11 @@ client = Ghostcrawl::Client.new(token: "gck_live_YOUR_KEY")
 result = client.scrape(url: "https://example.com", format: "markdown")
 puts result["content"]
 
-# Start a crawl run
-run = client.crawl_runs.start(url: "https://example.com", max_depth: 2, max_pages: 50)
-puts "run_id: #{run["run_id"]}"
+# Start a crawl run and wait for it to finish — the server blocks until the run
+# is terminal (no client-side poll-with-sleep loop).
+run = client.crawl_runs.start(url: "https://example.com", max_depth: 2, max_pages: 50,
+                              wait: true, timeout: 300)
+puts "#{run["run_id"]} → #{run["status"]}"
 
 # Web search
 results = client.search(query: "latest AI research", engine: "google", limit: 10)
@@ -72,6 +74,92 @@ data = client.extract(
 puts "#{data["name"]} — $#{data["price"]}"
 ```
 
+## Browser utilities — content, screenshot, PDF
+
+```ruby
+require "ghostcrawl"
+
+client = Ghostcrawl::Client.new(token: "gck_live_YOUR_KEY")
+
+# Rendered content as a JSON envelope: {url, status, format, status_code, content, bytes}
+page = client.content(url: "https://example.com")
+puts "#{page["status_code"]} — #{page["bytes"]} bytes"
+
+# Screenshot — returns raw PNG bytes
+png = client.screenshot(url: "https://example.com", full_page: true)
+File.binwrite("page.png", png)
+
+# PDF — returns raw application/pdf bytes (Chrome-only; a Firefox/WebKit identity
+# is rejected with a 400 pdf_engine_unsupported)
+pdf = client.pdf(url: "https://example.com", paper_format: "a4")
+File.binwrite("page.pdf", pdf)
+```
+
+## Agent (BYO model, account-gated)
+
+The agent lane runs a natural-language browser task. It is **bring-your-own-model** (supply your
+own `provider_config` in the request body) and **account-gated**: the API returns `404 not_found`
+unless the capability is enabled for your account. `agent` does **not** raise on that 404 — it
+returns the `problem+json` body as a Hash, so branch on `result.key?("detail")`.
+
+```ruby
+require "ghostcrawl"
+
+client = Ghostcrawl::Client.new(token: "gck_live_YOUR_KEY")
+
+# provider_config is BYO — reference your provider key by env-var name, never a literal.
+result = client.agent(
+  task: {
+    "instruction" => "click the 'Books to Scrape' link",
+    "start_url"   => "https://books.toscrape.com"
+  },
+  provider_config: {
+    "provider" => "openai",
+    "api_key"  => "OPENAI_API_KEY",
+    "model"    => "gpt-4o"
+  }
+)
+if result.key?("detail")
+  puts "agent lane not enabled for this account (BYO/gated)"
+else
+  puts result
+end
+```
+
+## Crawl runs — wait for completion
+
+Crawl runs are asynchronous. Instead of hand-writing a poll-with-sleep loop, use
+the event-driven wait API: the **server** blocks until the run reaches a terminal
+state (`completed` / `failed` / `cancelled`) and returns the finished run.
+
+```ruby
+# Start and wait in one call — sends wait_until: "completed"
+run = client.crawl_runs.start(
+  url: "https://example.com",
+  max_depth: 2,
+  max_pages: 50,
+  wait: true,      # block until terminal (no client sleep loop)
+  timeout: 300     # give up after 300s, returning the still-running run
+)
+puts run["status"]        # "completed" | "failed" | "cancelled"
+
+# Or wait on a run you already started (e.g. from a webhook or another process):
+run = client.crawl_runs.wait_for_completion(run_id, timeout: 300)
+
+# Deep crawl supports the same wait: keyword:
+run = client.crawl(url: "https://example.com", wait: true, timeout: 300)
+```
+
+Under the hood `wait_for_completion` chains server-blocking
+`GET /v1/crawl-runs/{run_id}?wait=true&timeout_s=N` requests until the run is
+terminal or your `timeout` deadline — there is no `sleep` between checks. On
+timeout the current non-terminal run is returned, so you can simply call
+`wait_for_completion` again to keep waiting.
+
+**Prefer webhooks for fire-and-forget crawls.** Register an endpoint with
+`client.webhooks.create(url:, event_types: ["crawl.completed"])` and let GhostCrawl
+POST the result to you instead of holding a connection open.
+
 ## Error handling
 
 ```ruby
@@ -112,8 +200,12 @@ client = Ghostcrawl::Client.new(
 | Data extraction | `client.extract(url:, schema:)` | Structured JSON from any page |
 | Deep crawl | `client.crawl(url:)` | Crawl a site depth-first |
 | URL map | `client.map(url:)` | Discover all reachable URLs |
+| Content | `client.content(url:)` | Rendered content JSON envelope |
+| Screenshot | `client.screenshot(url:)` | Capture a URL to PNG bytes |
+| PDF | `client.pdf(url:)` | Render a URL to PDF bytes (Chrome-only) |
+| Agent (BYO) | `client.agent(task:)` | NL browser task — account-gated, BYO model |
 | Account | `client.me` | Get account info and usage |
-| Crawl runs | `client.crawl_runs` | start, list, get\_run, cancel |
+| Crawl runs | `client.crawl_runs` | start (with `wait:`), wait\_for\_completion, list, get, cancel |
 | Sessions | `client.sessions` | create, list, extend, release |
 | Profiles | `client.profiles` | list, get\_profile, create, update, delete |
 | Webhooks | `client.webhooks` | list, get\_webhook, create, delete, rotate\_secret |

data/_generated/v1/binary/binary.rb

@@ -0,0 +1,9 @@
+module Ghostcrawl
+    module V1
+        ##
+        # Namespace retained as the raw-response sentinel used by binary-returning
+        # operations (datasets/page/har/discovery) and the long-poll wait factory.
+        module Binary
+        end
+    end
+end

data/_generated/v1/v1_request_builder.rb

@@ -2,7 +2,7 @@ require 'microsoft_kiota_abstractions'
 require_relative '../ghostcrawl'
 require_relative './audit/audit_request_builder'
 require_relative './billing/billing_request_builder'
-require_relative './binary/binary_request_builder'
+require_relative './binary/binary'
 require_relative './budgets/budgets_request_builder'
 require_relative './cdp/cdp_request_builder'
 require_relative './contact/contact_request_builder'
@@ -50,11 +50,6 @@ module Ghostcrawl
                 return Ghostcrawl::V1::Billing::BillingRequestBuilder.new(@path_parameters, @request_adapter)
             end
             ## 
-            # The binary property
-            def binary()
-                return Ghostcrawl::V1::Binary::BinaryRequestBuilder.new(@path_parameters, @request_adapter)
-            end
-            ## 
             # The budgets property
             def budgets()
                 return Ghostcrawl::V1::Budgets::BudgetsRequestBuilder.new(@path_parameters, @request_adapter)

data/lib/ghostcrawl/client.rb

@@ -381,6 +381,47 @@ module Ghostcrawl
       Ghostcrawl.raise_translated(e)
     end
 
+    # Executes a request whose success response is RAW BINARY (e.g. the
+    # +application/pdf+ body from +POST /v1/pdf+) and returns the bytes verbatim.
+    #
+    # Mirrors {void_request!}: it reuses the adapter's own request pipeline
+    # (base URL + bearer auth via +authenticate_request+, then +run_request+) but
+    # deliberately skips Kiota body deserialization — the response is not JSON, so
+    # there is no parse node to build. A 2xx returns the raw response body (an
+    # ASCII-8BIT String of bytes, ready to write to a file); a >=400 is translated
+    # into the documented typed {GhostcrawlError} hierarchy (the problem+json
+    # +detail+ rides along in the message).
+    #
+    # @param adapter [MicrosoftKiotaFaraday::FaradayRequestAdapter]
+    # @param request_info [MicrosoftKiotaAbstractions::RequestInformation]
+    # @return [String] the raw response bytes (ASCII-8BIT)
+    # @api private
+    def self.binary_request!(adapter, request_info)
+      request_info.path_parameters["baseurl"] = adapter.get_base_url
+      auth_fiber = adapter.authentication_provider.authenticate_request(request_info)
+      auth_fiber.resume if auth_fiber.respond_to?(:resume)
+      request = adapter.get_request_from_request_info(request_info)
+      response = adapter.client.run_request(
+        request.http_method, request.path, request.body, request.headers
+      )
+
+      status = response.status
+      if status >= 400
+        body = response.body.to_s
+        err = MicrosoftKiotaAbstractions::ApiError.new(
+          "The server returned an unexpected status code:#{status}" \
+          "#{body.empty? ? '' : " #{body}"}"
+        )
+        Ghostcrawl.raise_translated(err)
+      end
+
+      response.body.to_s.b
+    rescue Ghostcrawl::GhostcrawlError
+      raise
+    rescue StandardError => e
+      Ghostcrawl.raise_translated(e)
+    end
+
     # Inspects a decoded HTTP-200 response hash for a RESULT-channel failure (the
     # target page could not be scraped) and raises {Ghostcrawl::ScrapeError} when
     # one is present. This is the reliable, highest-value error path: the body is
@@ -463,26 +504,101 @@ module Ghostcrawl
 
   # Manage crawl runs — /v1/crawl-runs.
   class CrawlRunsClient
-    def initialize(v1)
+    # Terminal run states — a crawl run in any of these will never change again,
+    # so a wait can stop. Both spellings of "cancelled" are accepted.
+    TERMINAL_STATUSES = %w[completed failed cancelled canceled].freeze
+
+    # Per-request server-block window (seconds) for the long-poll wait. Each
+    # +GET ...?wait=true&timeout_s=WAIT_WINDOW_S+ makes the SERVER block for up
+    # to this long, so the client never sleeps between checks. Kept well under
+    # the client read timeout ({Client::DEFAULT_TIMEOUT}) so a single blocking
+    # request can never trip the transport timeout.
+    WAIT_WINDOW_S = 30
+
+    def initialize(v1, adapter = nil)
       @v1 = v1
+      @adapter = adapter
     end
 
     # Start a new crawl run from a seed URL.
     # Delegates to POST /v1/crawl-runs via the generated CrawlRunsRequestBuilder.
     # The endpoint is a tagged union: a start request requires +action: "start"+
     # and a +seed_urls+ array (not a bare +url+).
+    #
+    # Pass +wait: true+ to block until the run reaches a terminal state
+    # (completed | failed | cancelled) or +timeout+ seconds elapse. This sends
+    # +wait_until: "completed"+ so the SERVER blocks — no client-side poll loop,
+    # no +sleep+. When the run is still running at +timeout+ the current
+    # (non-terminal) record is returned; call {#wait_for_completion} again to
+    # keep waiting.
+    #
     # @param url [String] seed URL
     # @param max_depth [Integer] maximum crawl depth (default 2)
     # @param max_pages [Integer] maximum pages (default 100)
+    # @param wait [Boolean] block until the run is terminal (default false)
+    # @param timeout [Integer] total seconds to wait when +wait: true+ (default 300)
     # @param raise_on_result_error [Boolean] raise {Ghostcrawl::ScrapeError} on a
-    #   target-side (HTTP-200) failure instead of returning the raw hash (default true)
-    # @return [Hash] crawl run record with +run_id+ and +status+
-    def start(url:, max_depth: 2, max_pages: 100, raise_on_result_error: true, **opts)
-      body = AdditionalDataBody.new({ "action" => "start", "seed_urls" => [url],
-                                      "max_depth" => max_depth, "max_pages" => max_pages }
-                                      .merge(opts.transform_keys(&:to_s)))
-      hash = ResponseHelper.to_hash(@v1.crawl_runs.post(body))
-      raise_on_result_error ? ResponseHelper.raise_on_result_error!(hash) : hash
+    #   target-side (HTTP-200) failure instead of returning the raw hash (default true).
+    #   Ignored on the +wait: true+ path, which always returns the terminal run so
+    #   the caller can inspect a +failed+ status.
+    # @return [Hash] crawl run record with +run_id+ and +status+ (results present
+    #   when it completed while waiting)
+    def start(url:, max_depth: 2, max_pages: 100, wait: false, timeout: 300,
+              raise_on_result_error: true, **opts)
+      data = { "action" => "start", "seed_urls" => [url],
+               "max_depth" => max_depth, "max_pages" => max_pages }
+             .merge(opts.transform_keys(&:to_s))
+
+      unless wait
+        hash = ResponseHelper.to_hash(@v1.crawl_runs.post(AdditionalDataBody.new(data)))
+        return raise_on_result_error ? ResponseHelper.raise_on_result_error!(hash) : hash
+      end
+
+      # Start-and-wait: ask the server to block until terminal. Bound the first
+      # server block to a safe window, then long-poll the rest via GET so a low
+      # per-request read timeout can never abort the whole wait.
+      window = [timeout, WAIT_WINDOW_S].min
+      data["wait_until"] = "completed"
+      data["timeout_s"]  = window
+      hash = ResponseHelper.to_hash(@v1.crawl_runs.post(AdditionalDataBody.new(data)))
+      return hash if terminal?(hash)
+
+      run_id = hash["run_id"]
+      remaining = timeout - window
+      return hash if run_id.nil? || run_id.to_s.empty? || remaining <= 0
+      wait_for_completion(run_id, timeout: remaining)
+    end
+
+    # Block until an existing crawl run reaches a terminal state, or +timeout+
+    # seconds elapse. Event-driven: each iteration issues a SERVER-blocking
+    # +GET /v1/crawl-runs/{run_id}?wait=true&timeout_s=N+ that returns the moment
+    # the run goes terminal (or after its own window). There is no client +sleep+
+    # — the wait cost lives on the server, and successive blocking windows are
+    # chained until the caller's +timeout+ deadline.
+    #
+    # On timeout the current non-terminal run is returned (never raises for a
+    # slow run); a terminal run is returned as soon as it is observed.
+    #
+    # @param run_id [String] the run to wait on
+    # @param timeout [Integer] total seconds to wait (default 300)
+    # @return [Hash] the run record (terminal when it finished in time)
+    def wait_for_completion(run_id, timeout: 300)
+      if run_id.nil? || run_id.to_s.empty?
+        raise ArgumentError, "run_id is required"
+      end
+
+      deadline = monotonic + timeout
+      loop do
+        remaining = deadline - monotonic
+        # Deadline reached — one final non-blocking read of the current state.
+        return get(run_id) if remaining <= 0
+
+        window = [remaining, WAIT_WINDOW_S].min
+        run = fetch_waiting(run_id, window)
+        return run if terminal?(run)
+        # The server just blocked for ~window seconds; loop straight into the
+        # next blocking window. No client-side delay.
+      end
     end
 
     # List crawl runs.
@@ -502,6 +618,33 @@ module Ghostcrawl
     def cancel(run_id)
       ResponseHelper.to_hash(@v1.crawl_runs.by_run_id(run_id).cancel.post)
     end
+
+    private
+
+    # True when +run+ is a Hash carrying a terminal +status+.
+    def terminal?(run)
+      run.is_a?(Hash) && TERMINAL_STATUSES.include?(run["status"].to_s)
+    end
+
+    # Issues one SERVER-blocking GET on the run with +?wait=true&timeout_s=N+.
+    #
+    # The generated item builder's +to_get_request_information+ does not wire
+    # query parameters (its URL template has no query placeholder), so we take
+    # the request it builds, extend the template with +{?wait,timeout_s}+, and
+    # set the params before sending through the raw adapter — mirroring how
+    # {Client#me} re-issues a request with the Binary (raw-JSON) response factory.
+    def fetch_waiting(run_id, timeout_s)
+      request_info = @v1.crawl_runs.by_run_id(run_id).to_get_request_information(nil)
+      request_info.url_template = "{+baseurl}/v1/crawl-runs/{run_id}{?wait,timeout_s}"
+      request_info.query_parameters["wait"] = "true"
+      request_info.query_parameters["timeout_s"] = timeout_s.to_i.to_s
+      ResponseHelper.to_hash(@adapter.send_async(request_info, Ghostcrawl::V1::Binary, {}))
+    end
+
+    # Monotonic clock (seconds) — immune to wall-clock adjustments during a wait.
+    def monotonic
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
   end
 
   # Manage browser sessions — /v1/sessions.
@@ -848,7 +991,9 @@ module Ghostcrawl
       # (Binary) response factory instead of the typed MeResponse model.
       @adapter = adapter
 
-      @crawl_runs  = CrawlRunsClient.new(@v1)
+      # CrawlRunsClient needs the raw adapter for the event-driven long-poll wait
+      # (the generated item builder can't attach ?wait/&timeout_s query params).
+      @crawl_runs  = CrawlRunsClient.new(@v1, @adapter)
       @sessions    = SessionsClient.new(@v1)
       # Sub-clients with void-DELETE endpoints (204 No Content) also need the raw
       # adapter so ResponseHelper.void_request! can work around the broken generated
@@ -945,17 +1090,34 @@ module Ghostcrawl
 
     # Start a deep crawl from a seed URL.
     # Delegates to POST /v1/crawl/deep via the generated CrawlDeepRequestBuilder.
+    #
+    # Pass +wait: true+ to block until the run is terminal. The deep-crawl start
+    # returns a +run_id+, then the wait is handled by the same event-driven
+    # server-blocking long-poll as {CrawlRunsClient#wait_for_completion} — no
+    # client-side +sleep+ loop.
+    #
     # @param url [String] seed URL
     # @param max_depth [Integer] maximum crawl depth (default 2)
     # @param max_pages [Integer] maximum pages (default 100)
+    # @param wait [Boolean] block until the run is terminal (default false)
+    # @param timeout [Integer] total seconds to wait when +wait: true+ (default 300)
     # @param raise_on_result_error [Boolean] raise {Ghostcrawl::ScrapeError} on a
-    #   target-side (HTTP-200) failure instead of returning the raw hash (default true)
-    # @return [Hash] crawl run record
-    def crawl(url:, max_depth: 2, max_pages: 100, raise_on_result_error: true, **opts)
+    #   target-side (HTTP-200) failure instead of returning the raw hash (default true).
+    #   Ignored on the +wait: true+ path, which returns the terminal run.
+    # @return [Hash] crawl run record (terminal when +wait: true+ and it finished in time)
+    def crawl(url:, max_depth: 2, max_pages: 100, wait: false, timeout: 300,
+              raise_on_result_error: true, **opts)
       data = { "seed_urls" => [url], "max_depth" => max_depth,
                "max_urls" => max_pages }.merge(opts.transform_keys(&:to_s))
       hash = ResponseHelper.to_hash(@v1.crawl.deep.post(AdditionalDataBody.new(data)))
-      raise_on_result_error ? ResponseHelper.raise_on_result_error!(hash) : hash
+
+      unless wait
+        return raise_on_result_error ? ResponseHelper.raise_on_result_error!(hash) : hash
+      end
+
+      run_id = hash["run_id"]
+      return hash if run_id.nil? || run_id.to_s.empty?
+      @crawl_runs.wait_for_completion(run_id, timeout: timeout)
     end
 
     # Map all URLs reachable from a seed URL.
@@ -982,18 +1144,211 @@ module Ghostcrawl
       ResponseHelper.to_hash(@adapter.send_async(request_info, Ghostcrawl::V1::Binary, {}))
     end
 
+    # Render a URL to a PDF document and return the raw +application/pdf+ bytes.
+    # Delegates to POST /v1/pdf.
+    #
+    # +/v1/pdf+ responds with binary, not a JSON envelope, so this returns the
+    # bytes verbatim (an ASCII-8BIT String) — write them straight to a file:
+    #
+    #   data = client.pdf(url: "https://example.com")
+    #   File.binwrite("page.pdf", data)
+    #
+    # PDF output is Chrome-only; a request that resolves to a Firefox or WebKit
+    # identity is rejected with 400 +pdf_engine_unsupported+ (a {GhostcrawlError}).
+    #
+    # +/v1/pdf+ has no generated request builder, so this hand-builds a
+    # {MicrosoftKiotaAbstractions::RequestInformation} and routes it through the
+    # SAME adapter (base URL + bearer auth + transport) as the modeled calls.
+    #
+    # @param url [String] target URL to render
+    # @param paper_format [String] page size: "a4" (default), "letter", "legal", "tabloid"
+    # @param landscape [Boolean] render in landscape orientation (default false)
+    # @param engine [String] browser engine (PDF is Chrome-only; default "auto")
+    # @return [String] the raw PDF bytes (ASCII-8BIT)
+    def pdf(url:, paper_format: "a4", landscape: false, engine: "auto", **opts)
+      data = { "url" => url, "paper_format" => paper_format,
+               "landscape" => landscape, "engine" => engine }.merge(opts.transform_keys(&:to_s))
+      request_info = MicrosoftKiotaAbstractions::RequestInformation.new
+      request_info.http_method = :POST
+      request_info.url_template = "{+baseurl}/v1/pdf"
+      request_info.headers.try_add("Accept", "application/pdf")
+      request_info.set_stream_content(JSON.generate(data), "application/json")
+      ResponseHelper.binary_request!(@adapter, request_info)
+    end
+
+    # Capture a screenshot of a URL and return the raw +image/png+ bytes.
+    # Delegates to POST /v1/screenshot.
+    #
+    # +/v1/screenshot+ responds with a binary image, not a JSON envelope, so this
+    # returns the bytes verbatim (an ASCII-8BIT String) — write them straight to a
+    # file:
+    #
+    #   data = client.screenshot(url: "https://example.com")
+    #   File.binwrite("page.png", data)
+    #
+    # +/v1/screenshot+ has no generated request builder, so this hand-builds a
+    # {MicrosoftKiotaAbstractions::RequestInformation} and routes it through the
+    # SAME adapter (base URL + bearer auth + transport) as the modeled calls,
+    # mirroring {#pdf} exactly (the raw-bytes dispatch idiom).
+    #
+    # @param url [String] target URL to capture
+    # @param format [String] image format: "png" (default), "jpeg", "webp"
+    # @param full_page [Boolean] capture the full scrollable page (default false)
+    # @param screenshot_selector [String, nil] CSS selector to clip to (optional)
+    # @param engine [String] browser engine (default "auto")
+    # @return [String] the raw image bytes (ASCII-8BIT)
+    def screenshot(url:, format: "png", full_page: false, screenshot_selector: nil,
+                   engine: "auto", **opts)
+      data = { "url" => url, "format" => format, "full_page" => full_page,
+               "engine" => engine }
+      data["screenshot_selector"] = screenshot_selector unless screenshot_selector.nil?
+      data.merge!(opts.transform_keys(&:to_s))
+      request_info = MicrosoftKiotaAbstractions::RequestInformation.new
+      request_info.http_method = :POST
+      request_info.url_template = "{+baseurl}/v1/screenshot"
+      request_info.headers.try_add("Accept", "image/png")
+      request_info.set_stream_content(JSON.generate(data), "application/json")
+      ResponseHelper.binary_request!(@adapter, request_info)
+    end
+
+    # Render a URL and return the rendered-content JSON envelope as a Hash.
+    # Delegates to POST /v1/content.
+    #
+    # +/v1/content+ responds with +application/json+ ({+content+, +url+, +status+,
+    # +format+, +status_code+, +bytes+}); this returns the decoded Hash. It uses
+    # the same hand-built {MicrosoftKiotaAbstractions::RequestInformation} +
+    # shared adapter dispatch as {#pdf}, parsing the JSON body at the end.
+    #
+    # @param url [String] target URL to render
+    # @param engine [String] browser engine (default "auto")
+    # @return [Hash] the rendered-content envelope (top-level +content+ = HTML)
+    def content(url:, engine: "auto", **opts)
+      data = { "url" => url, "engine" => engine }.merge(opts.transform_keys(&:to_s))
+      request_info = MicrosoftKiotaAbstractions::RequestInformation.new
+      request_info.http_method = :POST
+      request_info.url_template = "{+baseurl}/v1/content"
+      request_info.headers.try_add("Accept", "application/json")
+      request_info.set_stream_content(JSON.generate(data), "application/json")
+      bytes = ResponseHelper.binary_request!(@adapter, request_info)
+      parsed = JSON.parse(bytes)
+      parsed.is_a?(Hash) ? parsed : { "content" => parsed }
+    end
+
+    # Execute an agent task via POST /v1/agent.
+    #
+    # The agent capability is gated per account; when it is not enabled the API
+    # replies +404 not_found+ — this returns that problem+json body as a Hash
+    # (carrying +"detail"+) rather than raising, so callers can branch on
+    # +result.key?("detail")+. The LLM step is BYO / not a hosted agent loop
+    # (RESEARCH §2) — this method only proves the client serialize→POST→parse
+    # plumbing round-trips against the real route.
+    #
+    # +/v1/agent+ has no generated request builder, so this hand-builds a
+    # {MicrosoftKiotaAbstractions::RequestInformation} and routes it through the
+    # SAME adapter (base URL + bearer auth + transport) as the modeled calls.
+    #
+    # @param url [String, nil] optional starting URL (folded into +task+ as +start_url+)
+    # @param instruction [String, nil] natural-language task instruction
+    # @param task [Hash, nil] explicit structured task object (overrides url/instruction)
+    # @return [Hash] the agent result, or the gated-404 problem+json body (has +"detail"+)
+    def agent(url: nil, instruction: nil, task: nil, **opts)
+      payload = task || { "instruction" => instruction, "start_url" => url }
+      data = { "task" => payload }.merge(opts.transform_keys(&:to_s))
+      request_info = MicrosoftKiotaAbstractions::RequestInformation.new
+      request_info.http_method = :POST
+      request_info.url_template = "{+baseurl}/v1/agent"
+      request_info.headers.try_add("Accept", "application/json")
+      request_info.set_stream_content(JSON.generate(data), "application/json")
+      bytes = ResponseHelper.binary_request!(@adapter, request_info)
+      parsed = JSON.parse(bytes)
+      parsed.is_a?(Hash) ? parsed : { "detail" => parsed }
+    rescue Ghostcrawl::GhostcrawlError => e
+      # Agent is account-gated: a 404 not_found is the documented "capability
+      # disabled" answer, not a transport failure. Surface the problem+json body
+      # as a Hash (carrying "detail") rather than raising. Any other status is a
+      # real error and is re-raised.
+      raise unless e.status_code == 404
+      body = e.body.to_s
+      brace = body.index("{")
+      decoded =
+        if brace
+          begin
+            JSON.parse(body[brace..])
+          rescue StandardError
+            nil
+          end
+        end
+      decoded.is_a?(Hash) ? decoded : { "detail" => e.message, "code" => (e.code || "not_found") }
+    end
+
+    # Create a new identity envelope.
+    # Delegates to POST /v1/identity. Returns the opaque encrypted identity
+    # envelope; the SDK never decodes or inspects the ciphertext.
+    #
+    # @param claim_os [String] OS to emulate (ios/android/macos/windows/linux)
+    # @param claim_browser [String] browser to emulate (chrome/firefox/safari)
+    # @param device_model [String, nil] device model (required for mobile claim_os)
+    # @param viewport [Object, nil] "fullscreen"/"any" or {"width"=>Integer,"height"=>Integer}
+    # @return [Hash] the identity envelope
+    def identity(claim_os:, claim_browser:, device_model: nil, viewport: nil, **opts)
+      data = { "claim_os" => claim_os, "claim_browser" => claim_browser }
+      data["device_model"] = device_model unless device_model.nil?
+      data["viewport"] = viewport unless viewport.nil?
+      data.merge!(opts.transform_keys(&:to_s))
+      json_request(:POST, "/v1/identity", data)
+    end
+
+    # Get the current account's cost/usage report.
+    # Delegates to GET /v1/me/usage (the canonical account-scoped usage endpoint).
+    # @return [Hash] usage envelope (+wallet+, +total_usd+, +by_engine+, …)
+    def usage
+      json_request(:GET, "/v1/me/usage", nil)
+    end
+
+    # List the account's persisted session storage-states.
+    # Delegates to GET /v1/storage-states.
+    # @return [Hash] the storage-states envelope
+    def storage_states
+      json_request(:GET, "/v1/storage-states", nil)
+    end
+
     private
 
+    # Route an ad-hoc JSON request through the SAME adapter (base URL + bearer
+    # auth + transport) as the modeled calls, for routes the generated core has
+    # no builder for. Returns the decoded Hash. Mirrors {#content}'s dispatch.
+    # @api private
+    def json_request(method, path, body)
+      request_info = MicrosoftKiotaAbstractions::RequestInformation.new
+      request_info.http_method = method
+      request_info.url_template = "{+baseurl}#{path}"
+      request_info.headers.try_add("Accept", "application/json")
+      if !body.nil? && method != :GET
+        request_info.set_stream_content(JSON.generate(body), "application/json")
+      end
+      bytes = ResponseHelper.binary_request!(@adapter, request_info)
+      parsed = (JSON.parse(bytes) if bytes && !bytes.empty?)
+      parsed.is_a?(Hash) ? parsed : {}
+    end
+
     # Normalizes a +"content"+ key onto a decoded scrape response.
     #
-    # The API returns the rendered page under a FORMAT-SPECIFIC key
-    # (+"markdown"+, +"html"+, or +"text"+), but the documented quickstart reads
-    # +result["content"]+. This mirrors that value onto +"content"+ in place,
-    # KEEPING the format-specific key intact (backward compatible).
+    # The API has two scrape response shapes:
+    #   * the FLAT +format="markdown"+ shape, which carries the rendered page at
+    #     the top level under the format-specific key (+"markdown"+/+"html"+/
+    #     +"text"+) but omits +identity_id+; and
+    #   * the standard ENVELOPE ({ +status+, +results+, +routing_mode+,
+    #     +request_class+, +identity_id+ }), where the rendered page lives under
+    #     +results[0]+ and the top-level +identity_id+ is present.
+    # The documented quickstart reads +result["content"]+, so this mirrors the
+    # rendered page onto +"content"+ in place from WHICHEVER shape is present,
+    # KEEPING the original keys intact (backward compatible).
     #
     # No-op unless +result+ is a Hash that does not already carry +"content"+.
     # The value chosen is: the field named by +result["format"]+ when that field
-    # is a String, else the first String among +"markdown"+, +"html"+, +"text"+.
+    # is a String, else the first String among +"markdown"+, +"html"+, +"text"+ —
+    # looked up first at the top level (flat shape), then on +results[0]+
+    # (envelope shape).
     #
     # @param result [Object] the decoded response (only mutated when a Hash)
     # @return [Object] the same +result+, unchanged reference
@@ -1002,21 +1357,34 @@ module Ghostcrawl
       return result unless result.is_a?(Hash) && !result.key?("content")
 
       fmt = result["format"]
-      value = result[fmt] if fmt.is_a?(String)
-      value = nil unless value.is_a?(String)
+      value = scrape_content_from(result, fmt)
 
+      # Envelope shape: the page is under results[0], not at the top level.
       unless value.is_a?(String)
-        %w[markdown html text].each do |key|
-          candidate = result[key]
-          if candidate.is_a?(String)
-            value = candidate
-            break
-          end
-        end
+        first = result["results"]
+        first = first.first if first.is_a?(Array)
+        value = scrape_content_from(first, fmt) if first.is_a?(Hash)
       end
 
       result["content"] = value if value.is_a?(String)
       result
     end
+
+    # Picks the rendered-page String out of a scrape result Hash: the field named
+    # by +fmt+ when present, else the first String among +"markdown"+, +"html"+,
+    # +"text"+. Returns +nil+ when +src+ carries no such String.
+    # @api private
+    def scrape_content_from(src, fmt)
+      return nil unless src.is_a?(Hash)
+
+      value = src[fmt] if fmt.is_a?(String)
+      return value if value.is_a?(String)
+
+      %w[markdown html text].each do |key|
+        candidate = src[key]
+        return candidate if candidate.is_a?(String)
+      end
+      nil
+    end
   end
 end

data/lib/ghostcrawl/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ghostcrawl
-  VERSION = "2.2.3"
+  VERSION = "2.3.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ghostcrawl
 version: !ruby/object:Gem::Version
-  version: 2.2.3
+  version: 2.3.1
 platform: ruby
 authors:
 - GhostCrawl
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-07-01 00:00:00.000000000 Z
+date: 2026-07-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: microsoft_kiota_abstractions
@@ -225,9 +225,6 @@ files:
 - _generated/v1/billing/usage/usage.rb
 - _generated/v1/billing/usage/usage_request_builder.rb
 - _generated/v1/binary/binary.rb
-- _generated/v1/binary/binary_request_builder.rb
-- _generated/v1/binary/download/download.rb
-- _generated/v1/binary/download/download_request_builder.rb
 - _generated/v1/budgets/budgets.rb
 - _generated/v1/budgets/budgets_request_builder.rb
 - _generated/v1/budgets/policy/item/item.rb
@@ -431,7 +428,7 @@ files:
 - lib/ghostcrawl/version.rb
 homepage: https://ghostcrawl.io
 licenses:
-- Nonstandard
+- MIT
 metadata:
   homepage_uri: https://ghostcrawl.io
   documentation_uri: https://docs.ghostcrawl.io

data/_generated/v1/binary/binary_request_builder.rb

@@ -1,31 +0,0 @@
-require 'microsoft_kiota_abstractions'
-require_relative '../../ghostcrawl'
-require_relative '../v1'
-require_relative './binary'
-require_relative './download/download_request_builder'
-
-module Ghostcrawl
-    module V1
-        module Binary
-            ## 
-            # Builds and executes requests for operations under #v1#binary
-            class BinaryRequestBuilder < MicrosoftKiotaAbstractions::BaseRequestBuilder
-                
-                ## 
-                # The download property
-                def download()
-                    return Ghostcrawl::V1::Binary::Download::DownloadRequestBuilder.new(@path_parameters, @request_adapter)
-                end
-                ## 
-                ## Instantiates a new BinaryRequestBuilder and sets the default values.
-                ## @param path_parameters Path parameters for the request
-                ## @param request_adapter The request adapter to use to execute the requests.
-                ## @return a void
-                ## 
-                def initialize(path_parameters, request_adapter)
-                    super(path_parameters, request_adapter, "{+baseurl}/v1/binary")
-                end
-            end
-        end
-    end
-end

data/_generated/v1/binary/download/download_request_builder.rb

@@ -1,77 +0,0 @@
-require 'microsoft_kiota_abstractions'
-require_relative '../../../ghostcrawl'
-require_relative '../../../models/h_t_t_p_validation_error'
-require_relative '../../v1'
-require_relative '../binary'
-require_relative './download'
-
-module Ghostcrawl
-    module V1
-        module Binary
-            module Download
-                ## 
-                # Builds and executes requests for operations under #v1#binary#download
-                class DownloadRequestBuilder < MicrosoftKiotaAbstractions::BaseRequestBuilder
-                    
-                    ## 
-                    ## Instantiates a new DownloadRequestBuilder and sets the default values.
-                    ## @param path_parameters Path parameters for the request
-                    ## @param request_adapter The request adapter to use to execute the requests.
-                    ## @return a void
-                    ## 
-                    def initialize(path_parameters, request_adapter)
-                        super(path_parameters, request_adapter, "{+baseurl}/v1/binary/download?engine={engine}{&release_id*}")
-                    end
-                    ## 
-                    ## Download your licensed engine binary (Chrome, Firefox, or WebKit) for self-hosting.Select the engine via the ``engine`` query parameter. The binary is streamedto the client and bound to your account.
-                    ## @param request_configuration Configuration for the request such as headers, query parameters, and middleware options.
-                    ## @return a Fiber of binary
-                    ## 
-                    def get(request_configuration=nil)
-                        request_info = self.to_get_request_information(
-                            request_configuration
-                        )
-                        error_mapping = Hash.new
-                        error_mapping["422"] = lambda {|pn| Ghostcrawl::Models::HTTPValidationError.create_from_discriminator_value(pn) }
-                        return @request_adapter.send_async(request_info, Binary, error_mapping)
-                    end
-                    ## 
-                    ## Download your licensed engine binary (Chrome, Firefox, or WebKit) for self-hosting.Select the engine via the ``engine`` query parameter. The binary is streamedto the client and bound to your account.
-                    ## @param request_configuration Configuration for the request such as headers, query parameters, and middleware options.
-                    ## @return a request_information
-                    ## 
-                    def to_get_request_information(request_configuration=nil)
-                        request_info = MicrosoftKiotaAbstractions::RequestInformation.new()
-                        unless request_configuration.nil?
-                            request_info.add_headers_from_raw_object(request_configuration.headers)
-                            request_info.set_query_string_parameters_from_raw_object(request_configuration.query_parameters)
-                            request_info.add_request_options(request_configuration.options)
-                        end
-                        request_info.url_template = @url_template
-                        request_info.path_parameters = @path_parameters
-                        request_info.http_method = :GET
-                        request_info.headers.try_add('Accept', 'application/json')
-                        return request_info
-                    end
-                    ## 
-                    ## Returns a request builder with the provided arbitrary URL. Using this method means any other path or query parameters are ignored.
-                    ## @param raw_url The raw URL to use for the request builder.
-                    ## @return a download_request_builder
-                    ## 
-                    def with_url(raw_url)
-                        raise StandardError, 'raw_url cannot be null' if raw_url.nil?
-                        return DownloadRequestBuilder.new(raw_url, @request_adapter)
-                    end
-
-                    ## 
-                    # Download your licensed engine binary (Chrome, Firefox, or WebKit) for self-hosting.Select the engine via the ``engine`` query parameter. The binary is streamedto the client and bound to your account.
-                    class DownloadRequestBuilderGetQueryParameters
-                        
-                        attr_accessor :engine
-                        attr_accessor :release_id
-                    end
-                end
-            end
-        end
-    end
-end