ghostcrawl 2.3.0 → 2.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 482db5d9b31ab1b7293fc19b6834be23062a37d6ef8632fae462e369e5dfc244
-  data.tar.gz: 01eababfa0e5cd0ecc5ca39386c4b7a8cdbe2d8f0ff7f079382c89fed5859c9c
+  metadata.gz: 9b83f5c591c7a3ffd859db07f96c9725f84db05826b65e9227eadce1ff3eded3
+  data.tar.gz: 0aa598b5c6d776b5477c0709aa26dd3789d5f79084421b1fae802e90892c4dc0
 SHA512:
-  metadata.gz: 866336d0c4c835f6ccaafd9a444c03eb4931b97cbf4b65ca003b01ff5d374a68b06500542805951ee25c3276adf64b0d16c433a956f0dbbe8bebd91ef10e0211
-  data.tar.gz: 250b128f21670a470961468a6da62c1647495d7b4065a6d1fbe4f13328ac680d7b40ebc6acfd68338e88fa9f8bbfb99ec88ea0d5f6f6ebb56cff56261800ea41
+  metadata.gz: 3458805483c931bbd9adca163e74d0610c7d02abdb6ca260459fbdbe162c573f29752846d92bf45189a8fcea7957f770588364c9c483082729386c8b2d7c281a
+  data.tar.gz: 8d56532b83ea407f5a8e9190832fe1fd8ae3327cd427cb3f26be0fd727c20567a5bc1331998caa403d622edd7456956b5a373ae1961d1281f336b741ea607b32

data/README.md

@@ -74,6 +74,58 @@ data = client.extract(
 puts "#{data["name"]} — $#{data["price"]}"
 ```
 
+## Browser utilities — content, screenshot, PDF
+
+```ruby
+require "ghostcrawl"
+
+client = Ghostcrawl::Client.new(token: "gck_live_YOUR_KEY")
+
+# Rendered content as a JSON envelope: {url, status, format, status_code, content, bytes}
+page = client.content(url: "https://example.com")
+puts "#{page["status_code"]} — #{page["bytes"]} bytes"
+
+# Screenshot — returns raw PNG bytes
+png = client.screenshot(url: "https://example.com", full_page: true)
+File.binwrite("page.png", png)
+
+# PDF — returns raw application/pdf bytes (Chrome-only; a Firefox/WebKit identity
+# is rejected with a 400 pdf_engine_unsupported)
+pdf = client.pdf(url: "https://example.com", paper_format: "a4")
+File.binwrite("page.pdf", pdf)
+```
+
+## Agent (BYO model, account-gated)
+
+The agent lane runs a natural-language browser task. It is **bring-your-own-model** (supply your
+own `provider_config` in the request body) and **account-gated**: the API returns `404 not_found`
+unless the capability is enabled for your account. `agent` does **not** raise on that 404 — it
+returns the `problem+json` body as a Hash, so branch on `result.key?("detail")`.
+
+```ruby
+require "ghostcrawl"
+
+client = Ghostcrawl::Client.new(token: "gck_live_YOUR_KEY")
+
+# provider_config is BYO — reference your provider key by env-var name, never a literal.
+result = client.agent(
+  task: {
+    "instruction" => "click the 'Books to Scrape' link",
+    "start_url"   => "https://books.toscrape.com"
+  },
+  provider_config: {
+    "provider" => "openai",
+    "api_key"  => "OPENAI_API_KEY",
+    "model"    => "gpt-4o"
+  }
+)
+if result.key?("detail")
+  puts "agent lane not enabled for this account (BYO/gated)"
+else
+  puts result
+end
+```
+
 ## Crawl runs — wait for completion
 
 Crawl runs are asynchronous. Instead of hand-writing a poll-with-sleep loop, use
@@ -148,6 +200,10 @@ client = Ghostcrawl::Client.new(
 | Data extraction | `client.extract(url:, schema:)` | Structured JSON from any page |
 | Deep crawl | `client.crawl(url:)` | Crawl a site depth-first |
 | URL map | `client.map(url:)` | Discover all reachable URLs |
+| Content | `client.content(url:)` | Rendered content JSON envelope |
+| Screenshot | `client.screenshot(url:)` | Capture a URL to PNG bytes |
+| PDF | `client.pdf(url:)` | Render a URL to PDF bytes (Chrome-only) |
+| Agent (BYO) | `client.agent(task:)` | NL browser task — account-gated, BYO model |
 | Account | `client.me` | Get account info and usage |
 | Crawl runs | `client.crawl_runs` | start (with `wait:`), wait\_for\_completion, list, get, cancel |
 | Sessions | `client.sessions` | create, list, extend, release |

data/lib/ghostcrawl/client.rb

@@ -381,6 +381,47 @@ module Ghostcrawl
       Ghostcrawl.raise_translated(e)
     end
 
+    # Executes a request whose success response is RAW BINARY (e.g. the
+    # +application/pdf+ body from +POST /v1/pdf+) and returns the bytes verbatim.
+    #
+    # Mirrors {void_request!}: it reuses the adapter's own request pipeline
+    # (base URL + bearer auth via +authenticate_request+, then +run_request+) but
+    # deliberately skips Kiota body deserialization — the response is not JSON, so
+    # there is no parse node to build. A 2xx returns the raw response body (an
+    # ASCII-8BIT String of bytes, ready to write to a file); a >=400 is translated
+    # into the documented typed {GhostcrawlError} hierarchy (the problem+json
+    # +detail+ rides along in the message).
+    #
+    # @param adapter [MicrosoftKiotaFaraday::FaradayRequestAdapter]
+    # @param request_info [MicrosoftKiotaAbstractions::RequestInformation]
+    # @return [String] the raw response bytes (ASCII-8BIT)
+    # @api private
+    def self.binary_request!(adapter, request_info)
+      request_info.path_parameters["baseurl"] = adapter.get_base_url
+      auth_fiber = adapter.authentication_provider.authenticate_request(request_info)
+      auth_fiber.resume if auth_fiber.respond_to?(:resume)
+      request = adapter.get_request_from_request_info(request_info)
+      response = adapter.client.run_request(
+        request.http_method, request.path, request.body, request.headers
+      )
+
+      status = response.status
+      if status >= 400
+        body = response.body.to_s
+        err = MicrosoftKiotaAbstractions::ApiError.new(
+          "The server returned an unexpected status code:#{status}" \
+          "#{body.empty? ? '' : " #{body}"}"
+        )
+        Ghostcrawl.raise_translated(err)
+      end
+
+      response.body.to_s.b
+    rescue Ghostcrawl::GhostcrawlError
+      raise
+    rescue StandardError => e
+      Ghostcrawl.raise_translated(e)
+    end
+
     # Inspects a decoded HTTP-200 response hash for a RESULT-channel failure (the
     # target page could not be scraped) and raises {Ghostcrawl::ScrapeError} when
     # one is present. This is the reliable, highest-value error path: the body is
@@ -1103,8 +1144,193 @@ module Ghostcrawl
       ResponseHelper.to_hash(@adapter.send_async(request_info, Ghostcrawl::V1::Binary, {}))
     end
 
+    # Render a URL to a PDF document and return the raw +application/pdf+ bytes.
+    # Delegates to POST /v1/pdf.
+    #
+    # +/v1/pdf+ responds with binary, not a JSON envelope, so this returns the
+    # bytes verbatim (an ASCII-8BIT String) — write them straight to a file:
+    #
+    #   data = client.pdf(url: "https://example.com")
+    #   File.binwrite("page.pdf", data)
+    #
+    # PDF output is Chrome-only; a request that resolves to a Firefox or WebKit
+    # identity is rejected with 400 +pdf_engine_unsupported+ (a {GhostcrawlError}).
+    #
+    # +/v1/pdf+ has no generated request builder, so this hand-builds a
+    # {MicrosoftKiotaAbstractions::RequestInformation} and routes it through the
+    # SAME adapter (base URL + bearer auth + transport) as the modeled calls.
+    #
+    # @param url [String] target URL to render
+    # @param paper_format [String] page size: "a4" (default), "letter", "legal", "tabloid"
+    # @param landscape [Boolean] render in landscape orientation (default false)
+    # @param engine [String] browser engine (PDF is Chrome-only; default "auto")
+    # @return [String] the raw PDF bytes (ASCII-8BIT)
+    def pdf(url:, paper_format: "a4", landscape: false, engine: "auto", **opts)
+      data = { "url" => url, "paper_format" => paper_format,
+               "landscape" => landscape, "engine" => engine }.merge(opts.transform_keys(&:to_s))
+      request_info = MicrosoftKiotaAbstractions::RequestInformation.new
+      request_info.http_method = :POST
+      request_info.url_template = "{+baseurl}/v1/pdf"
+      request_info.headers.try_add("Accept", "application/pdf")
+      request_info.set_stream_content(JSON.generate(data), "application/json")
+      ResponseHelper.binary_request!(@adapter, request_info)
+    end
+
+    # Capture a screenshot of a URL and return the raw +image/png+ bytes.
+    # Delegates to POST /v1/screenshot.
+    #
+    # +/v1/screenshot+ responds with a binary image, not a JSON envelope, so this
+    # returns the bytes verbatim (an ASCII-8BIT String) — write them straight to a
+    # file:
+    #
+    #   data = client.screenshot(url: "https://example.com")
+    #   File.binwrite("page.png", data)
+    #
+    # +/v1/screenshot+ has no generated request builder, so this hand-builds a
+    # {MicrosoftKiotaAbstractions::RequestInformation} and routes it through the
+    # SAME adapter (base URL + bearer auth + transport) as the modeled calls,
+    # mirroring {#pdf} exactly (the raw-bytes dispatch idiom).
+    #
+    # @param url [String] target URL to capture
+    # @param format [String] image format: "png" (default), "jpeg", "webp"
+    # @param full_page [Boolean] capture the full scrollable page (default false)
+    # @param screenshot_selector [String, nil] CSS selector to clip to (optional)
+    # @param engine [String] browser engine (default "auto")
+    # @return [String] the raw image bytes (ASCII-8BIT)
+    def screenshot(url:, format: "png", full_page: false, screenshot_selector: nil,
+                   engine: "auto", **opts)
+      data = { "url" => url, "format" => format, "full_page" => full_page,
+               "engine" => engine }
+      data["screenshot_selector"] = screenshot_selector unless screenshot_selector.nil?
+      data.merge!(opts.transform_keys(&:to_s))
+      request_info = MicrosoftKiotaAbstractions::RequestInformation.new
+      request_info.http_method = :POST
+      request_info.url_template = "{+baseurl}/v1/screenshot"
+      request_info.headers.try_add("Accept", "image/png")
+      request_info.set_stream_content(JSON.generate(data), "application/json")
+      ResponseHelper.binary_request!(@adapter, request_info)
+    end
+
+    # Render a URL and return the rendered-content JSON envelope as a Hash.
+    # Delegates to POST /v1/content.
+    #
+    # +/v1/content+ responds with +application/json+ ({+content+, +url+, +status+,
+    # +format+, +status_code+, +bytes+}); this returns the decoded Hash. It uses
+    # the same hand-built {MicrosoftKiotaAbstractions::RequestInformation} +
+    # shared adapter dispatch as {#pdf}, parsing the JSON body at the end.
+    #
+    # @param url [String] target URL to render
+    # @param engine [String] browser engine (default "auto")
+    # @return [Hash] the rendered-content envelope (top-level +content+ = HTML)
+    def content(url:, engine: "auto", **opts)
+      data = { "url" => url, "engine" => engine }.merge(opts.transform_keys(&:to_s))
+      request_info = MicrosoftKiotaAbstractions::RequestInformation.new
+      request_info.http_method = :POST
+      request_info.url_template = "{+baseurl}/v1/content"
+      request_info.headers.try_add("Accept", "application/json")
+      request_info.set_stream_content(JSON.generate(data), "application/json")
+      bytes = ResponseHelper.binary_request!(@adapter, request_info)
+      parsed = JSON.parse(bytes)
+      parsed.is_a?(Hash) ? parsed : { "content" => parsed }
+    end
+
+    # Execute an agent task via POST /v1/agent.
+    #
+    # The agent capability is gated per account; when it is not enabled the API
+    # replies +404 not_found+ — this returns that problem+json body as a Hash
+    # (carrying +"detail"+) rather than raising, so callers can branch on
+    # +result.key?("detail")+. The LLM step is BYO / not a hosted agent loop
+    # (RESEARCH §2) — this method only proves the client serialize→POST→parse
+    # plumbing round-trips against the real route.
+    #
+    # +/v1/agent+ has no generated request builder, so this hand-builds a
+    # {MicrosoftKiotaAbstractions::RequestInformation} and routes it through the
+    # SAME adapter (base URL + bearer auth + transport) as the modeled calls.
+    #
+    # @param url [String, nil] optional starting URL (folded into +task+ as +start_url+)
+    # @param instruction [String, nil] natural-language task instruction
+    # @param task [Hash, nil] explicit structured task object (overrides url/instruction)
+    # @return [Hash] the agent result, or the gated-404 problem+json body (has +"detail"+)
+    def agent(url: nil, instruction: nil, task: nil, **opts)
+      payload = task || { "instruction" => instruction, "start_url" => url }
+      data = { "task" => payload }.merge(opts.transform_keys(&:to_s))
+      request_info = MicrosoftKiotaAbstractions::RequestInformation.new
+      request_info.http_method = :POST
+      request_info.url_template = "{+baseurl}/v1/agent"
+      request_info.headers.try_add("Accept", "application/json")
+      request_info.set_stream_content(JSON.generate(data), "application/json")
+      bytes = ResponseHelper.binary_request!(@adapter, request_info)
+      parsed = JSON.parse(bytes)
+      parsed.is_a?(Hash) ? parsed : { "detail" => parsed }
+    rescue Ghostcrawl::GhostcrawlError => e
+      # Agent is account-gated: a 404 not_found is the documented "capability
+      # disabled" answer, not a transport failure. Surface the problem+json body
+      # as a Hash (carrying "detail") rather than raising. Any other status is a
+      # real error and is re-raised.
+      raise unless e.status_code == 404
+      body = e.body.to_s
+      brace = body.index("{")
+      decoded =
+        if brace
+          begin
+            JSON.parse(body[brace..])
+          rescue StandardError
+            nil
+          end
+        end
+      decoded.is_a?(Hash) ? decoded : { "detail" => e.message, "code" => (e.code || "not_found") }
+    end
+
+    # Create a new identity envelope.
+    # Delegates to POST /v1/identity. Returns the opaque encrypted identity
+    # envelope; the SDK never decodes or inspects the ciphertext.
+    #
+    # @param claim_os [String] OS to emulate (ios/android/macos/windows/linux)
+    # @param claim_browser [String] browser to emulate (chrome/firefox/safari)
+    # @param device_model [String, nil] device model (required for mobile claim_os)
+    # @param viewport [Object, nil] "fullscreen"/"any" or {"width"=>Integer,"height"=>Integer}
+    # @return [Hash] the identity envelope
+    def identity(claim_os:, claim_browser:, device_model: nil, viewport: nil, **opts)
+      data = { "claim_os" => claim_os, "claim_browser" => claim_browser }
+      data["device_model"] = device_model unless device_model.nil?
+      data["viewport"] = viewport unless viewport.nil?
+      data.merge!(opts.transform_keys(&:to_s))
+      json_request(:POST, "/v1/identity", data)
+    end
+
+    # Get the current account's cost/usage report.
+    # Delegates to GET /v1/me/usage (the canonical account-scoped usage endpoint).
+    # @return [Hash] usage envelope (+wallet+, +total_usd+, +by_engine+, …)
+    def usage
+      json_request(:GET, "/v1/me/usage", nil)
+    end
+
+    # List the account's persisted session storage-states.
+    # Delegates to GET /v1/storage-states.
+    # @return [Hash] the storage-states envelope
+    def storage_states
+      json_request(:GET, "/v1/storage-states", nil)
+    end
+
     private
 
+    # Route an ad-hoc JSON request through the SAME adapter (base URL + bearer
+    # auth + transport) as the modeled calls, for routes the generated core has
+    # no builder for. Returns the decoded Hash. Mirrors {#content}'s dispatch.
+    # @api private
+    def json_request(method, path, body)
+      request_info = MicrosoftKiotaAbstractions::RequestInformation.new
+      request_info.http_method = method
+      request_info.url_template = "{+baseurl}#{path}"
+      request_info.headers.try_add("Accept", "application/json")
+      if !body.nil? && method != :GET
+        request_info.set_stream_content(JSON.generate(body), "application/json")
+      end
+      bytes = ResponseHelper.binary_request!(@adapter, request_info)
+      parsed = (JSON.parse(bytes) if bytes && !bytes.empty?)
+      parsed.is_a?(Hash) ? parsed : {}
+    end
+
     # Normalizes a +"content"+ key onto a decoded scrape response.
     #
     # The API has two scrape response shapes:

data/lib/ghostcrawl/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ghostcrawl
-  VERSION = "2.3.0"
+  VERSION = "2.3.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ghostcrawl
 version: !ruby/object:Gem::Version
-  version: 2.3.0
+  version: 2.3.1
 platform: ruby
 authors:
 - GhostCrawl
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-07-19 00:00:00.000000000 Z
+date: 2026-07-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: microsoft_kiota_abstractions