ghostcrawl 2.3.4

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: e50d69d157331b61abadd2c88956ed9944a981c4cf33d2dcc5cbec5f771b00f9
+  data.tar.gz: de3c4e08089c1a7b4dcc623ef196163cb4cede19be5690d0ac3daaa243a96a8d
+SHA512:
+  metadata.gz: 7354be27df2e4eb86457c477a09059e2f8d2445355f1646437517a697df7de9f7de9439b85fe77601135724f88e3dd0189214f3f507250f153860d3cd0c86dfd
+  data.tar.gz: 32daf0d816b38242ce52f211c48295d3ea2af66665fafa79ed1e51352cb4ea5f9cbdf8a354716bad59fb3baa889b31781c700ca50ef2c767e2b5c8534dfbfb80

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 GhostCrawl, LLC
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,218 @@
+# ghostcrawl — Ruby SDK
+
+The official Ruby client for the [GhostCrawl](https://ghostcrawl.io) API.
+Collect web data at scale — scrape, crawl, search, and extract structured data.
+
+## Install
+
+```bash
+gem install ghostcrawl
+```
+
+Or in your `Gemfile`:
+
+```ruby
+gem "ghostcrawl", "~> 2.1"
+```
+
+Requires Ruby 3.0+. No runtime dependencies — uses stdlib `net/http` only.
+
+## Quickstart
+
+```ruby
+require "ghostcrawl"
+
+# Token from constructor or GHOSTCRAWL_API_KEY env var
+client = GhostCrawl::Client.new(token: "gck_live_YOUR_KEY")
+
+# Scrape a URL
+result = client.scrape(url: "https://example.com", format: "markdown")
+puts result["content"]
+
+# Start a crawl run and wait for it to finish — the server blocks until the run
+# is terminal (no client-side poll-with-sleep loop).
+run = client.crawl_runs.start(url: "https://example.com", max_depth: 2, max_pages: 50,
+                              wait: true, timeout: 300)
+puts "#{run["run_id"]} → #{run["status"]}"
+
+# Web search
+results = client.search(query: "latest AI research", engine: "google", limit: 10)
+puts results["results"].length
+```
+
+## Authentication
+
+```ruby
+# Option 1: pass token directly
+client = GhostCrawl::Client.new(token: "gck_live_YOUR_KEY")
+
+# Option 2: set environment variable (recommended for production)
+# export GHOSTCRAWL_API_KEY=gck_live_YOUR_KEY
+client = GhostCrawl::Client.new
+```
+
+Every request sends `Authorization: Bearer <token>`. This is the only auth scheme the API accepts.
+
+## Extract structured data
+
+```ruby
+require "ghostcrawl"
+
+client = GhostCrawl::Client.new(token: "gck_live_YOUR_KEY")
+
+data = client.extract(
+  url: "https://example.com/product",
+  schema: {
+    type: "object",
+    properties: {
+      name:        { type: "string" },
+      price:       { type: "number" },
+      description: { type: "string" }
+    }
+  }
+)
+puts "#{data["name"]} — $#{data["price"]}"
+```
+
+## Browser utilities — content, screenshot, PDF
+
+```ruby
+require "ghostcrawl"
+
+client = GhostCrawl::Client.new(token: "gck_live_YOUR_KEY")
+
+# Rendered content as a JSON envelope: {url, status, format, status_code, content, bytes}
+page = client.content(url: "https://example.com")
+puts "#{page["status_code"]} — #{page["bytes"]} bytes"
+
+# Screenshot — returns raw PNG bytes
+png = client.screenshot(url: "https://example.com", full_page: true)
+File.binwrite("page.png", png)
+
+# PDF — returns raw application/pdf bytes (Chrome-only; a Firefox/WebKit identity
+# is rejected with a 400 pdf_engine_unsupported)
+pdf = client.pdf(url: "https://example.com", paper_format: "a4")
+File.binwrite("page.pdf", pdf)
+```
+
+## Agent (BYO model, account-gated)
+
+The agent lane runs a natural-language browser task. It is **bring-your-own-model** (supply your
+own `provider_config` in the request body) and **account-gated**: the API returns `404 not_found`
+unless the capability is enabled for your account. `agent` does **not** raise on that 404 — it
+returns the `problem+json` body as a Hash, so branch on `result.key?("detail")`.
+
+```ruby
+require "ghostcrawl"
+
+client = GhostCrawl::Client.new(token: "gck_live_YOUR_KEY")
+
+# provider_config is BYO — reference your provider key by env-var name, never a literal.
+result = client.agent(
+  task: {
+    "instruction" => "click the 'Books to Scrape' link",
+    "start_url"   => "https://books.toscrape.com"
+  },
+  provider_config: {
+    "provider" => "openai",
+    "api_key"  => "OPENAI_API_KEY",
+    "model"    => "gpt-4o"
+  }
+)
+if result.key?("detail")
+  puts "agent lane not enabled for this account (BYO/gated)"
+else
+  puts result
+end
+```
+
+## Crawl runs — wait for completion
+
+Crawl runs are asynchronous. Instead of hand-writing a poll-with-sleep loop, use
+the event-driven wait API: the **server** blocks until the run reaches a terminal
+state (`completed` / `failed` / `cancelled`) and returns the finished run.
+
+```ruby
+# Start and wait in one call — sends wait_until: "completed"
+run = client.crawl_runs.start(
+  url: "https://example.com",
+  max_depth: 2,
+  max_pages: 50,
+  wait: true,      # block until terminal (no client sleep loop)
+  timeout: 300     # give up after 300s, returning the still-running run
+)
+puts run["status"]        # "completed" | "failed" | "cancelled"
+
+# Or wait on a run you already started (e.g. from a webhook or another process):
+run = client.crawl_runs.wait_for_completion(run_id, timeout: 300)
+
+# Deep crawl supports the same wait: keyword:
+run = client.crawl(url: "https://example.com", wait: true, timeout: 300)
+```
+
+Under the hood `wait_for_completion` chains server-blocking
+`GET /v1/crawl-runs/{run_id}?wait=true&timeout_s=N` requests until the run is
+terminal or your `timeout` deadline — there is no `sleep` between checks. On
+timeout the current non-terminal run is returned, so you can simply call
+`wait_for_completion` again to keep waiting.
+
+**Prefer webhooks for fire-and-forget crawls.** Register an endpoint with
+`client.webhooks.create(url:, event_types: ["crawl.completed"])` and let GhostCrawl
+POST the result to you instead of holding a connection open.
+
+## Error handling
+
+```ruby
+require "ghostcrawl"
+
+client = GhostCrawl::Client.new(token: "gck_live_YOUR_KEY")
+
+begin
+  result = client.scrape(url: "https://example.com")
+rescue GhostCrawl::AuthenticationError
+  puts "Invalid API key — check your token"
+rescue GhostCrawl::PaymentRequiredError
+  puts "Usage limit reached — upgrade your plan"
+rescue GhostCrawl::RateLimitError
+  puts "Rate limit reached — retry after a short delay"
+rescue GhostCrawl::APIError => e
+  puts "Server error: #{e.status_code}"
+rescue GhostCrawl::GhostCrawlError => e
+  puts "API error: #{e.message} (#{e.status_code})"
+end
+```
+
+## Self-hosted
+
+```ruby
+client = GhostCrawl::Client.new(
+  token: "gck_live_YOUR_KEY",
+  base_url: "http://localhost:8080"
+)
+```
+
+## All resources
+
+| Resource | Method / accessor | Key operations |
+|----------|------------------|----------------|
+| Scraping | `client.scrape(url:)` | Render and return page content |
+| Web search | `client.search(query:)` | Search Google, Bing, DuckDuckGo |
+| Data extraction | `client.extract(url:, schema:)` | Structured JSON from any page |
+| Deep crawl | `client.crawl(url:)` | Crawl a site depth-first |
+| URL map | `client.map(url:)` | Discover all reachable URLs |
+| Content | `client.content(url:)` | Rendered content JSON envelope |
+| Screenshot | `client.screenshot(url:)` | Capture a URL to PNG bytes |
+| PDF | `client.pdf(url:)` | Render a URL to PDF bytes (Chrome-only) |
+| Agent (BYO) | `client.agent(task:)` | NL browser task — account-gated, BYO model |
+| Crawl runs | `client.crawl_runs` | start (with `wait:`), wait\_for\_completion, list, get, cancel |
+| Sessions | `client.sessions` | create, list, extend, release |
+| Profiles | `client.profiles` | list, get\_profile, create, update, delete |
+| Webhooks | `client.webhooks` | list, get\_webhook, create, delete, rotate\_secret |
+| Schedules | `client.schedules` | list, get\_schedule, create, delete |
+| Datasets | `client.datasets` | list, get\_dataset, create, delete, rows, append |
+| Recordings | `client.recordings` | list, get\_recording, delete |
+| Key-Value Store | `client.kv` | get, set, delete |
+
+## License
+
+Proprietary — GhostCrawl Software License. See [LICENSE](LICENSE).

data/_generated/ghostcrawl_client.rb

@@ -0,0 +1,27 @@
+require 'microsoft_kiota_abstractions'
+require 'microsoft_kiota_serialization_json'
+require_relative './ghostcrawl'
+require_relative './v1/v1_request_builder'
+
+module GhostCrawl
+    ## 
+    # The main entry point of the SDK, exposes the configuration and the fluent API.
+    class GhostCrawlClient < MicrosoftKiotaAbstractions::BaseRequestBuilder
+        
+        ## 
+        # The v1 property
+        def v1()
+            return GhostCrawl::V1::V1RequestBuilder.new(@path_parameters, @request_adapter)
+        end
+        ## 
+        ## Instantiates a new GhostCrawlClient and sets the default values.
+        ## @param request_adapter The request adapter to use to execute the requests.
+        ## @return a void
+        ## 
+        def initialize(request_adapter)
+            super(Hash.new, request_adapter, "{+baseurl}")
+            MicrosoftKiotaAbstractions::ApiClientBuilder.register_default_serializer(MicrosoftKiotaSerializationJson::JsonSerializationWriterFactory)
+            MicrosoftKiotaAbstractions::ApiClientBuilder.register_default_deserializer(MicrosoftKiotaSerializationJson::JsonParseNodeFactory)
+        end
+    end
+end

data/_generated/models/agent_request.rb

@@ -0,0 +1,203 @@
+require 'microsoft_kiota_abstractions'
+require_relative '../ghostcrawl'
+require_relative './models'
+
+module GhostCrawl
+    module Models
+        class AgentRequest
+            include MicrosoftKiotaAbstractions::Parsable
+            ## 
+            # The api_key property
+            @api_key
+            ## 
+            # The callback_url property
+            @callback_url
+            ## 
+            # The mode property
+            @mode
+            ## 
+            # Optional stable profile identifier. When supplied on a successful sync `/v1/agent` call, the executed step trace is recorded into the in-process action cache (keyed by `(profile_id, task)`) and can be replayed via `POST /v1/agent/action-cache/replay`. Response payload includes `action_cache_recorded: true|false` so callers can confirm cache durability.
+            @profile_id
+            ## 
+            # The proxy property
+            @proxy
+            ## 
+            # Agent task payload; URL fields are policy-validated when present.
+            @task
+            ## 
+            # The tenant_id property
+            @tenant_id
+            ## 
+            # The workspace_id property
+            @workspace_id
+            ## 
+            ## Gets the api_key property value. The api_key property
+            ## @return a string
+            ## 
+            def api_key
+                return @api_key
+            end
+            ## 
+            ## Sets the api_key property value. The api_key property
+            ## @param value Value to set for the api_key property.
+            ## @return a void
+            ## 
+            def api_key=(value)
+                @api_key = value
+            end
+            ## 
+            ## Gets the callback_url property value. The callback_url property
+            ## @return a string
+            ## 
+            def callback_url
+                return @callback_url
+            end
+            ## 
+            ## Sets the callback_url property value. The callback_url property
+            ## @param value Value to set for the callback_url property.
+            ## @return a void
+            ## 
+            def callback_url=(value)
+                @callback_url = value
+            end
+            ## 
+            ## Instantiates a new AgentRequest and sets the default values.
+            ## @return a void
+            ## 
+            def initialize()
+                @mode = GhostCrawl::Models::AgentRequestMode[:Sync]
+            end
+            ## 
+            ## Creates a new instance of the appropriate class based on discriminator value
+            ## @param parse_node The parse node to use to read the discriminator value and create the object
+            ## @return a agent_request
+            ## 
+            def self.create_from_discriminator_value(parse_node)
+                raise StandardError, 'parse_node cannot be null' if parse_node.nil?
+                return AgentRequest.new
+            end
+            ## 
+            ## The deserialization information for the current model
+            ## @return a i_dictionary
+            ## 
+            def get_field_deserializers()
+                return {
+                    "api_key" => lambda {|n| @api_key = n.get_string_value() },
+                    "callback_url" => lambda {|n| @callback_url = n.get_string_value() },
+                    "mode" => lambda {|n| @mode = n.get_enum_value(GhostCrawl::Models::AgentRequestMode) },
+                    "profile_id" => lambda {|n| @profile_id = n.get_string_value() },
+                    "proxy" => lambda {|n| @proxy = n.get_object_value(lambda {|pn| GhostCrawl::Models::ProxyConfig.create_from_discriminator_value(pn) }) },
+                    "task" => lambda {|n| @task = n.get_object_value(lambda {|pn| GhostCrawl::Models::AgentRequestTask.create_from_discriminator_value(pn) }) },
+                    "tenant_id" => lambda {|n| @tenant_id = n.get_string_value() },
+                    "workspace_id" => lambda {|n| @workspace_id = n.get_string_value() },
+                }
+            end
+            ## 
+            ## Gets the mode property value. The mode property
+            ## @return a agent_request_mode
+            ## 
+            def mode
+                return @mode
+            end
+            ## 
+            ## Sets the mode property value. The mode property
+            ## @param value Value to set for the mode property.
+            ## @return a void
+            ## 
+            def mode=(value)
+                @mode = value
+            end
+            ## 
+            ## Gets the profile_id property value. Optional stable profile identifier. When supplied on a successful sync `/v1/agent` call, the executed step trace is recorded into the in-process action cache (keyed by `(profile_id, task)`) and can be replayed via `POST /v1/agent/action-cache/replay`. Response payload includes `action_cache_recorded: true|false` so callers can confirm cache durability.
+            ## @return a string
+            ## 
+            def profile_id
+                return @profile_id
+            end
+            ## 
+            ## Sets the profile_id property value. Optional stable profile identifier. When supplied on a successful sync `/v1/agent` call, the executed step trace is recorded into the in-process action cache (keyed by `(profile_id, task)`) and can be replayed via `POST /v1/agent/action-cache/replay`. Response payload includes `action_cache_recorded: true|false` so callers can confirm cache durability.
+            ## @param value Value to set for the profile_id property.
+            ## @return a void
+            ## 
+            def profile_id=(value)
+                @profile_id = value
+            end
+            ## 
+            ## Gets the proxy property value. The proxy property
+            ## @return a proxy_config
+            ## 
+            def proxy
+                return @proxy
+            end
+            ## 
+            ## Sets the proxy property value. The proxy property
+            ## @param value Value to set for the proxy property.
+            ## @return a void
+            ## 
+            def proxy=(value)
+                @proxy = value
+            end
+            ## 
+            ## Serializes information the current object
+            ## @param writer Serialization writer to use to serialize this model
+            ## @return a void
+            ## 
+            def serialize(writer)
+                raise StandardError, 'writer cannot be null' if writer.nil?
+                writer.write_string_value("api_key", @api_key)
+                writer.write_string_value("callback_url", @callback_url)
+                writer.write_enum_value("mode", @mode)
+                writer.write_string_value("profile_id", @profile_id)
+                writer.write_object_value("proxy", @proxy)
+                writer.write_object_value("task", @task)
+                writer.write_string_value("tenant_id", @tenant_id)
+                writer.write_string_value("workspace_id", @workspace_id)
+            end
+            ## 
+            ## Gets the task property value. Agent task payload; URL fields are policy-validated when present.
+            ## @return a agent_request_task
+            ## 
+            def task
+                return @task
+            end
+            ## 
+            ## Sets the task property value. Agent task payload; URL fields are policy-validated when present.
+            ## @param value Value to set for the task property.
+            ## @return a void
+            ## 
+            def task=(value)
+                @task = value
+            end
+            ## 
+            ## Gets the tenant_id property value. The tenant_id property
+            ## @return a string
+            ## 
+            def tenant_id
+                return @tenant_id
+            end
+            ## 
+            ## Sets the tenant_id property value. The tenant_id property
+            ## @param value Value to set for the tenant_id property.
+            ## @return a void
+            ## 
+            def tenant_id=(value)
+                @tenant_id = value
+            end
+            ## 
+            ## Gets the workspace_id property value. The workspace_id property
+            ## @return a string
+            ## 
+            def workspace_id
+                return @workspace_id
+            end
+            ## 
+            ## Sets the workspace_id property value. The workspace_id property
+            ## @param value Value to set for the workspace_id property.
+            ## @return a void
+            ## 
+            def workspace_id=(value)
+                @workspace_id = value
+            end
+        end
+    end
+end

data/_generated/models/agent_request_mode.rb

@@ -0,0 +1,8 @@
+module GhostCrawl
+    module Models
+        AgentRequestMode = {
+            Sync: :Sync,
+            Async: :Async,
+        }
+    end
+end

data/_generated/models/agent_request_task.rb

@@ -0,0 +1,164 @@
+require 'microsoft_kiota_abstractions'
+require_relative '../ghostcrawl'
+require_relative './models'
+
+module GhostCrawl
+    module Models
+        ## 
+        # Agent task payload; URL fields are policy-validated when present.
+        class AgentRequestTask
+            include MicrosoftKiotaAbstractions::AdditionalDataHolder, MicrosoftKiotaAbstractions::Parsable
+            ## 
+            # Stores additional data not described in the OpenAPI description found when deserializing. Can be used for serialization as well.
+            @additional_data
+            ## 
+            # The goal property
+            @goal
+            ## 
+            # The start_url property
+            @start_url
+            ## 
+            # Typed agent-step baseline contract (observe/act/extract).
+            @steps
+            ## 
+            # The target_url property
+            @target_url
+            ## 
+            # The url property
+            @url
+            ## 
+            ## Gets the AdditionalData property value. Stores additional data not described in the OpenAPI description found when deserializing. Can be used for serialization as well.
+            ## @return a i_dictionary
+            ## 
+            def additional_data
+                return @additional_data
+            end
+            ## 
+            ## Sets the AdditionalData property value. Stores additional data not described in the OpenAPI description found when deserializing. Can be used for serialization as well.
+            ## @param value Value to set for the AdditionalData property.
+            ## @return a void
+            ## 
+            def additional_data=(value)
+                @additional_data = value
+            end
+            ## 
+            ## Instantiates a new AgentRequestTask and sets the default values.
+            ## @return a void
+            ## 
+            def initialize()
+                @additional_data = Hash.new
+            end
+            ## 
+            ## Creates a new instance of the appropriate class based on discriminator value
+            ## @param parse_node The parse node to use to read the discriminator value and create the object
+            ## @return a agent_request_task
+            ## 
+            def self.create_from_discriminator_value(parse_node)
+                raise StandardError, 'parse_node cannot be null' if parse_node.nil?
+                return AgentRequestTask.new
+            end
+            ## 
+            ## The deserialization information for the current model
+            ## @return a i_dictionary
+            ## 
+            def get_field_deserializers()
+                return {
+                    "goal" => lambda {|n| @goal = n.get_string_value() },
+                    "start_url" => lambda {|n| @start_url = n.get_string_value() },
+                    "steps" => lambda {|n| @steps = n.get_collection_of_object_values(lambda {|pn| GhostCrawl::Models::AgentRequestTaskSteps.create_from_discriminator_value(pn) }) },
+                    "target_url" => lambda {|n| @target_url = n.get_string_value() },
+                    "url" => lambda {|n| @url = n.get_string_value() },
+                }
+            end
+            ## 
+            ## Gets the goal property value. The goal property
+            ## @return a string
+            ## 
+            def goal
+                return @goal
+            end
+            ## 
+            ## Sets the goal property value. The goal property
+            ## @param value Value to set for the goal property.
+            ## @return a void
+            ## 
+            def goal=(value)
+                @goal = value
+            end
+            ## 
+            ## Serializes information the current object
+            ## @param writer Serialization writer to use to serialize this model
+            ## @return a void
+            ## 
+            def serialize(writer)
+                raise StandardError, 'writer cannot be null' if writer.nil?
+                writer.write_string_value("goal", @goal)
+                writer.write_string_value("start_url", @start_url)
+                writer.write_collection_of_object_values("steps", @steps)
+                writer.write_string_value("target_url", @target_url)
+                writer.write_string_value("url", @url)
+                writer.write_additional_data(@additional_data)
+            end
+            ## 
+            ## Gets the start_url property value. The start_url property
+            ## @return a string
+            ## 
+            def start_url
+                return @start_url
+            end
+            ## 
+            ## Sets the start_url property value. The start_url property
+            ## @param value Value to set for the start_url property.
+            ## @return a void
+            ## 
+            def start_url=(value)
+                @start_url = value
+            end
+            ## 
+            ## Gets the steps property value. Typed agent-step baseline contract (observe/act/extract).
+            ## @return a agent_request_task_steps
+            ## 
+            def steps
+                return @steps
+            end
+            ## 
+            ## Sets the steps property value. Typed agent-step baseline contract (observe/act/extract).
+            ## @param value Value to set for the steps property.
+            ## @return a void
+            ## 
+            def steps=(value)
+                @steps = value
+            end
+            ## 
+            ## Gets the target_url property value. The target_url property
+            ## @return a string
+            ## 
+            def target_url
+                return @target_url
+            end
+            ## 
+            ## Sets the target_url property value. The target_url property
+            ## @param value Value to set for the target_url property.
+            ## @return a void
+            ## 
+            def target_url=(value)
+                @target_url = value
+            end
+            ## 
+            ## Gets the url property value. The url property
+            ## @return a string
+            ## 
+            def url
+                return @url
+            end
+            ## 
+            ## Sets the url property value. The url property
+            ## @param value Value to set for the url property.
+            ## @return a void
+            ## 
+            def url=(value)
+                @url = value
+            end
+        end
+    end
+end