ghostcrawl 2.2.2 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ff09081da25b9c78b261437d5bc26c2a4b25f3e727a53e8393f1adb00aa6536c
-  data.tar.gz: ad0869fe147e1c33a49fadd364bc93ce5857fd73eb03b584d4c1ce618e1092a9
+  metadata.gz: 482db5d9b31ab1b7293fc19b6834be23062a37d6ef8632fae462e369e5dfc244
+  data.tar.gz: 01eababfa0e5cd0ecc5ca39386c4b7a8cdbe2d8f0ff7f079382c89fed5859c9c
 SHA512:
-  metadata.gz: '085820b153fb6b70ea26f6e866787388835cfec7e4cce86498f83551abe6b4c8800b5eb14f86203c59ca77b7e70cffbd459d0d84f7a75fc69f5337914bcf60e6'
-  data.tar.gz: 11a00207fc7469d02bec976018c8e627ea11c0da0ce78d4dc5a6d5b76fced17a5f3a37053159e0bde94dd0c44e4d3e273e7cf57cdac1d652a410bcba4da814ff
+  metadata.gz: 866336d0c4c835f6ccaafd9a444c03eb4931b97cbf4b65ca003b01ff5d374a68b06500542805951ee25c3276adf64b0d16c433a956f0dbbe8bebd91ef10e0211
+  data.tar.gz: 250b128f21670a470961468a6da62c1647495d7b4065a6d1fbe4f13328ac680d7b40ebc6acfd68338e88fa9f8bbfb99ec88ea0d5f6f6ebb56cff56261800ea41

data/LICENSE

@@ -1,46 +1,21 @@
-GhostCrawl Software License
-
-Copyright (c) 2026 GhostCrawl, LLC. All rights reserved.
-
-This software — including the GhostCrawl SDKs, command-line interface, example
-code, documentation, and the GhostCrawl self-host image (collectively, the
-"Software") — is the proprietary property of GhostCrawl, LLC ("GhostCrawl") and
-its licensors. It is licensed, not sold.
-
-1. LICENSE GRANT. Subject to your continuous compliance with this License and the
-   GhostCrawl Terms of Service (https://docs.ghostcrawl.io/legal/terms),
-   GhostCrawl grants you a limited, revocable, non-exclusive, non-transferable,
-   and non-sublicensable license to install and use the Software, including
-   within your own applications, solely to access and use the GhostCrawl service
-   under a valid GhostCrawl account.
-
-2. RESTRICTIONS. Except as expressly permitted in Section 1, you may not, in
-   whole or in part: (a) copy the Software except as strictly necessary to use it
-   as permitted; (b) modify, adapt, translate, or create derivative works of the
-   Software; (c) distribute, publish, sublicense, sell, resell, rent, lease,
-   lend, or otherwise transfer the Software, or make it available to any third
-   party, including as a hosted or managed service; (d) reverse engineer,
-   decompile, disassemble, or otherwise attempt to derive the source code,
-   structure, or internal design of any compiled component of the Software;
-   (e) circumvent, disable, or tamper with any license, authentication,
-   usage-metering, watermarking, or security mechanism; or (f) remove, alter, or
-   obscure any proprietary, copyright, or attribution notice.
-
-3. OWNERSHIP. GhostCrawl and its licensors retain all right, title, and interest
-   in and to the Software and all intellectual property rights therein. No rights
-   are granted to you except as expressly set out in this License. All rights
-   reserved.
-
-4. TERMINATION. This License terminates automatically, without notice, if you
-   breach any of its terms or cease to maintain a valid GhostCrawl account where
-   one is required. Upon termination you must cease all use of the Software and
-   destroy all copies in your possession or control.
-
-5. NO WARRANTY; LIMITATION OF LIABILITY. THE SOFTWARE IS PROVIDED "AS IS" WITHOUT
-   WARRANTY OF ANY KIND, EXPRESS OR IMPLIED. TO THE MAXIMUM EXTENT PERMITTED BY
-   LAW, GHOSTCRAWL AND ITS LICENSORS WILL NOT BE LIABLE FOR ANY CLAIM, DAMAGES, OR
-   OTHER LIABILITY ARISING FROM OR IN CONNECTION WITH THE SOFTWARE OR ITS USE. The
-   GhostCrawl Terms of Service and Acceptable Use Policy
-   (https://docs.ghostcrawl.io/legal/terms) are incorporated by reference.
-
-For licensing inquiries, contact legal@ghostcrawl.io.
+MIT License
+
+Copyright (c) 2026 GhostCrawl, LLC
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -29,9 +29,11 @@ client = Ghostcrawl::Client.new(token: "gck_live_YOUR_KEY")
 result = client.scrape(url: "https://example.com", format: "markdown")
 puts result["content"]
 
-# Start a crawl run
-run = client.crawl_runs.start(url: "https://example.com", max_depth: 2, max_pages: 50)
-puts "run_id: #{run["run_id"]}"
+# Start a crawl run and wait for it to finish — the server blocks until the run
+# is terminal (no client-side poll-with-sleep loop).
+run = client.crawl_runs.start(url: "https://example.com", max_depth: 2, max_pages: 50,
+                              wait: true, timeout: 300)
+puts "#{run["run_id"]} → #{run["status"]}"
 
 # Web search
 results = client.search(query: "latest AI research", engine: "google", limit: 10)
@@ -72,6 +74,40 @@ data = client.extract(
 puts "#{data["name"]} — $#{data["price"]}"
 ```
 
+## Crawl runs — wait for completion
+
+Crawl runs are asynchronous. Instead of hand-writing a poll-with-sleep loop, use
+the event-driven wait API: the **server** blocks until the run reaches a terminal
+state (`completed` / `failed` / `cancelled`) and returns the finished run.
+
+```ruby
+# Start and wait in one call — sends wait_until: "completed"
+run = client.crawl_runs.start(
+  url: "https://example.com",
+  max_depth: 2,
+  max_pages: 50,
+  wait: true,      # block until terminal (no client sleep loop)
+  timeout: 300     # give up after 300s, returning the still-running run
+)
+puts run["status"]        # "completed" | "failed" | "cancelled"
+
+# Or wait on a run you already started (e.g. from a webhook or another process):
+run = client.crawl_runs.wait_for_completion(run_id, timeout: 300)
+
+# Deep crawl supports the same wait: keyword:
+run = client.crawl(url: "https://example.com", wait: true, timeout: 300)
+```
+
+Under the hood `wait_for_completion` chains server-blocking
+`GET /v1/crawl-runs/{run_id}?wait=true&timeout_s=N` requests until the run is
+terminal or your `timeout` deadline — there is no `sleep` between checks. On
+timeout the current non-terminal run is returned, so you can simply call
+`wait_for_completion` again to keep waiting.
+
+**Prefer webhooks for fire-and-forget crawls.** Register an endpoint with
+`client.webhooks.create(url:, event_types: ["crawl.completed"])` and let GhostCrawl
+POST the result to you instead of holding a connection open.
+
 ## Error handling
 
 ```ruby
@@ -113,7 +149,7 @@ client = Ghostcrawl::Client.new(
 | Deep crawl | `client.crawl(url:)` | Crawl a site depth-first |
 | URL map | `client.map(url:)` | Discover all reachable URLs |
 | Account | `client.me` | Get account info and usage |
-| Crawl runs | `client.crawl_runs` | start, list, get\_run, cancel |
+| Crawl runs | `client.crawl_runs` | start (with `wait:`), wait\_for\_completion, list, get, cancel |
 | Sessions | `client.sessions` | create, list, extend, release |
 | Profiles | `client.profiles` | list, get\_profile, create, update, delete |
 | Webhooks | `client.webhooks` | list, get\_webhook, create, delete, rotate\_secret |

data/_generated/v1/binary/binary.rb

@@ -0,0 +1,9 @@
+module Ghostcrawl
+    module V1
+        ##
+        # Namespace retained as the raw-response sentinel used by binary-returning
+        # operations (datasets/page/har/discovery) and the long-poll wait factory.
+        module Binary
+        end
+    end
+end

data/_generated/v1/v1_request_builder.rb

@@ -2,7 +2,7 @@ require 'microsoft_kiota_abstractions'
 require_relative '../ghostcrawl'
 require_relative './audit/audit_request_builder'
 require_relative './billing/billing_request_builder'
-require_relative './binary/binary_request_builder'
+require_relative './binary/binary'
 require_relative './budgets/budgets_request_builder'
 require_relative './cdp/cdp_request_builder'
 require_relative './contact/contact_request_builder'
@@ -50,11 +50,6 @@ module Ghostcrawl
                 return Ghostcrawl::V1::Billing::BillingRequestBuilder.new(@path_parameters, @request_adapter)
             end
             ## 
-            # The binary property
-            def binary()
-                return Ghostcrawl::V1::Binary::BinaryRequestBuilder.new(@path_parameters, @request_adapter)
-            end
-            ## 
             # The budgets property
             def budgets()
                 return Ghostcrawl::V1::Budgets::BudgetsRequestBuilder.new(@path_parameters, @request_adapter)

data/lib/ghostcrawl/client.rb

@@ -16,6 +16,7 @@
 #   client = Ghostcrawl::Client.new(token: "gck_live_YOUR_KEY")
 #   result = client.scrape(url: "https://example.com")
 
+require "json"
 require "microsoft_kiota_abstractions"
 require "microsoft_kiota_faraday"
 require "microsoft_kiota_serialization_json"
@@ -57,6 +58,72 @@ module Ghostcrawl
         super
       end
     end
+
+    # Fix 1b: the pinned Kiota writer's write_collection_of_object_values calls
+    # `self.write_object_value(nil, v).writer` on each element — it relies on
+    # write_object_value(nil, ...) returning a *writer object* that responds to
+    # `.writer`. Our write_object_value override above (Fix 1) changes that
+    # nil-key contract to serialize-into-self and return the serialize result
+    # (a Hash / nil), so the upstream `.writer` call raises
+    # `NoMethodError: undefined method 'writer' for {}:Hash`. That aborts
+    # serialize for ANY response model carrying a collection of typed objects
+    # (WebhookListResponse#items, etc.), which ResponseHelper.serialize_parsable
+    # then swallows -> the facade returns {} and silently drops the whole list.
+    #
+    # Fix: serialize each element into its OWN fresh writer and collect the raw
+    # per-element hashes directly, never depending on the write_object_value
+    # return contract. Mirrors the non-nil upstream branch (line 154) without the
+    # broken `.writer` deref.
+    def write_collection_of_object_values(key, values)
+      return unless values
+      hashes = values.map do |v|
+        temp = MicrosoftKiotaSerializationJson::JsonSerializationWriter.new
+        v.serialize(temp)
+        temp.writer
+      end
+      if key.nil?
+        hashes
+      else
+        writer[key] = hashes
+      end
+    end
+
+    # Fix 1c: the pinned Kiota writer's write_any_value (used by
+    # write_additional_data, i.e. EVERY field AdditionalDataBody sends) has no
+    # case for a plain Hash and mis-handles an Array of Hashes:
+    #   * a Hash hits the `value.is_a? Object` branch -> `return value.to_s`,
+    #     which returns a string but NEVER assigns @writer[key] -> the whole
+    #     nested object is SILENTLY DROPPED from the request body.
+    #   * an Array is sent to write_collection_of_primitive_values, which mangles
+    #     any non-primitive (Hash) element.
+    # This drops/mangles every nested field the facade sends: scrape(extract_schema),
+    # extract(schema), crawl/crawl_runs opts, datasets.append(rows: [ {..} ]),
+    # schedules.create(job_params: {..}) — the last of which the API REQUIRES,
+    # so the request 422s ("job_params required") even though the caller supplied it.
+    #
+    # Fix: intercept Hash and Array here and write a fully-recursed, JSON-native
+    # structure straight into @writer[key]. The writer emits @writer.to_json at the
+    # end, so plain nested Hash/Array values serialize correctly. Everything else
+    # falls through to the upstream primitive handling.
+    def write_any_value(key, value)
+      if value.is_a?(Hash) || value.is_a?(Array)
+        return super unless key # nil-key arrays keep upstream behavior
+        writer[key] = KiotaWriterFix.jsonable(value)
+        return
+      end
+      super
+    end
+
+    # Recursively converts a value into a JSON-native structure (Hash/Array of
+    # plain scalars), so it round-trips through @writer.to_json intact.
+    # @api private
+    def self.jsonable(value)
+      case value
+      when Hash  then value.each_with_object({}) { |(k, v), h| h[k.to_s] = jsonable(v) }
+      when Array then value.map { |v| jsonable(v) }
+      else value
+      end
+    end
   end
 
   # Patch the writer class once, idempotently
@@ -190,17 +257,130 @@ module Ghostcrawl
       when NilClass
         {}
       else
-        # Typed Parsable: try serializing to JSON then parsing back
-        if value.respond_to?(:additional_data) && value.additional_data
-          value.additional_data.transform_values { |v| to_hash(v) }
-        elsif value.respond_to?(:to_h)
-          value.to_h.transform_values { |v| to_hash(v) }
-        else
-          value
-        end
+        # Typed Parsable (Kiota-generated model). Kiota models keep their spec
+        # fields in typed instance variables, NOT in +additional_data+ (which is
+        # only the overflow bucket for unmapped keys). Reading +additional_data+
+        # alone therefore returns +{}+ for a fully-typed response (e.g.
+        # MapResponse{@links,@success}, WebhookListResponse{@items,@total}),
+        # silently dropping the real payload.
+        #
+        # Recover the typed fields by round-tripping the model through its own
+        # +serialize+ (the Parsable contract) into a JSON writer and parsing the
+        # result back to a plain Hash. Then overlay any +additional_data+ (unmapped
+        # keys the server sent that the model didn't declare). Degrade gracefully:
+        # a model whose +serialize+ raises (e.g. an unresolved composed-type member)
+        # falls back to the prior additional_data/to_h behavior so nothing regresses.
+        parsable_to_hash(value)
+      end
+    end
+
+    # Converts a typed Kiota Parsable into a plain Hash by serializing it via its
+    # own +serialize+ and parsing the emitted JSON, then merging +additional_data+.
+    # Falls back to +additional_data+ / +to_h+ / the value itself on any failure.
+    # @api private
+    def self.parsable_to_hash(value)
+      serialized = serialize_parsable(value)
+      if serialized.is_a?(Hash)
+        extra = (value.additional_data if value.respond_to?(:additional_data)) || {}
+        merged = serialized.merge(extra.transform_keys(&:to_s))
+        return merged.transform_values { |v| to_hash(v) }
+      end
+
+      if value.respond_to?(:additional_data) && value.additional_data && !value.additional_data.empty?
+        value.additional_data.transform_values { |v| to_hash(v) }
+      elsif value.respond_to?(:to_h)
+        value.to_h.transform_values { |v| to_hash(v) }
+      elsif value.respond_to?(:additional_data) && value.additional_data
+        value.additional_data.transform_values { |v| to_hash(v) }
+      else
+        value
       end
     end
 
+    # Serializes a Parsable to a Ruby Hash using the pinned Kiota JSON writer.
+    # Returns the parsed Hash, or +nil+ when the model isn't a serializable
+    # Parsable or serialization/parse fails (caller then falls back).
+    # @api private
+    def self.serialize_parsable(value)
+      return nil unless value.respond_to?(:serialize)
+
+      writer = MicrosoftKiotaSerializationJson::JsonSerializationWriter.new
+      value.serialize(writer)
+      content = writer.get_serialized_content
+      json = content.is_a?(String) ? content : content.read
+      return nil if json.nil? || json.empty?
+
+      parsed = JSON.parse(json)
+      parsed.is_a?(Hash) ? parsed : nil
+    rescue StandardError
+      nil
+    end
+
+    # Executes a request that returns NO response body (an HTTP 204, or any DELETE
+    # whose spec maps the success response to +void+) and returns a plain Hash.
+    #
+    # Why this exists — two upstream defects in the pinned Kiota Ruby runtime that
+    # make the generated void-DELETE builders unusable:
+    #
+    #   1. The generated +delete+ for a void response calls
+    #      +send_async(request_info, nil, ...)+, and the Faraday adapter hard-raises
+    #      +"factory cannot be null"+ BEFORE the request is ever sent — so the DELETE
+    #      never reaches the server and the caller gets a bare +StandardError+
+    #      (not a {GhostcrawlError}).
+    #   2. For void DELETEs the adapter DOES send maps to +Binary+, the response is an
+    #      empty 204 body, and +get_root_parse_node+ then feeds +""+ to the JSON parser,
+    #      raising +JSON::ParserError+ on a perfectly successful delete.
+    #
+    # This helper reuses the adapter's OWN public request pipeline
+    # (+convert_to_native_request_async+ applies base-url + auth, +run_request+
+    # sends it) but skips body deserialization entirely: a 2xx returns +{}+ (or the
+    # decoded JSON when the server did send a body, e.g. +{"deleted":true}+), and a
+    # >=400 is translated into the documented typed {GhostcrawlError} hierarchy.
+    #
+    # @param adapter [MicrosoftKiotaFaraday::FaradayRequestAdapter]
+    # @param request_info [MicrosoftKiotaAbstractions::RequestInformation]
+    # @return [Hash] +{}+ on an empty 2xx, or the decoded body when one is present
+    # @api private
+    def self.void_request!(adapter, request_info)
+      # Replicate the first half of the adapter's send_async by hand — apply the
+      # base URL, run the bearer-auth Fiber (which mutates request_info's headers),
+      # then build the native Faraday request. We deliberately do NOT call the
+      # adapter's convert_to_native_request_async helper: its body uses a bare
+      # +return+ inside a Fiber, which raises LocalJumpError("unexpected return")
+      # when resumed under Ruby 3.x.
+      request_info.path_parameters["baseurl"] = adapter.get_base_url
+      # authenticate_request returns a Fiber only when the Authorization header is
+      # not already present; it returns nil otherwise. Guard the resume.
+      auth_fiber = adapter.authentication_provider.authenticate_request(request_info)
+      auth_fiber.resume if auth_fiber.respond_to?(:resume)
+      request = adapter.get_request_from_request_info(request_info)
+      response = adapter.client.run_request(
+        request.http_method, request.path, request.body, request.headers
+      )
+
+      status = response.status
+      if status >= 400
+        # Reuse the typed-error translation: synthesize a message carrying the
+        # ":<status>" token raise_translated keys off, plus any body text.
+        body = response.body.to_s
+        err = MicrosoftKiotaAbstractions::ApiError.new(
+          "The server returned an unexpected status code:#{status}" \
+          "#{body.empty? ? '' : " #{body}"}"
+        )
+        Ghostcrawl.raise_translated(err)
+      end
+
+      body = response.body.to_s
+      return {} if body.strip.empty?
+
+      parsed = (JSON.parse(body) rescue nil)
+      parsed.is_a?(Hash) ? parsed.transform_values { |v| to_hash(v) } : {}
+    rescue Ghostcrawl::GhostcrawlError
+      raise
+    rescue StandardError => e
+      Ghostcrawl.raise_translated(e)
+    end
+
     # Inspects a decoded HTTP-200 response hash for a RESULT-channel failure (the
     # target page could not be scraped) and raises {Ghostcrawl::ScrapeError} when
     # one is present. This is the reliable, highest-value error path: the body is
@@ -283,26 +463,101 @@ module Ghostcrawl
 
   # Manage crawl runs — /v1/crawl-runs.
   class CrawlRunsClient
-    def initialize(v1)
+    # Terminal run states — a crawl run in any of these will never change again,
+    # so a wait can stop. Both spellings of "cancelled" are accepted.
+    TERMINAL_STATUSES = %w[completed failed cancelled canceled].freeze
+
+    # Per-request server-block window (seconds) for the long-poll wait. Each
+    # +GET ...?wait=true&timeout_s=WAIT_WINDOW_S+ makes the SERVER block for up
+    # to this long, so the client never sleeps between checks. Kept well under
+    # the client read timeout ({Client::DEFAULT_TIMEOUT}) so a single blocking
+    # request can never trip the transport timeout.
+    WAIT_WINDOW_S = 30
+
+    def initialize(v1, adapter = nil)
       @v1 = v1
+      @adapter = adapter
     end
 
     # Start a new crawl run from a seed URL.
     # Delegates to POST /v1/crawl-runs via the generated CrawlRunsRequestBuilder.
     # The endpoint is a tagged union: a start request requires +action: "start"+
     # and a +seed_urls+ array (not a bare +url+).
+    #
+    # Pass +wait: true+ to block until the run reaches a terminal state
+    # (completed | failed | cancelled) or +timeout+ seconds elapse. This sends
+    # +wait_until: "completed"+ so the SERVER blocks — no client-side poll loop,
+    # no +sleep+. When the run is still running at +timeout+ the current
+    # (non-terminal) record is returned; call {#wait_for_completion} again to
+    # keep waiting.
+    #
     # @param url [String] seed URL
     # @param max_depth [Integer] maximum crawl depth (default 2)
     # @param max_pages [Integer] maximum pages (default 100)
+    # @param wait [Boolean] block until the run is terminal (default false)
+    # @param timeout [Integer] total seconds to wait when +wait: true+ (default 300)
     # @param raise_on_result_error [Boolean] raise {Ghostcrawl::ScrapeError} on a
-    #   target-side (HTTP-200) failure instead of returning the raw hash (default true)
-    # @return [Hash] crawl run record with +run_id+ and +status+
-    def start(url:, max_depth: 2, max_pages: 100, raise_on_result_error: true, **opts)
-      body = AdditionalDataBody.new({ "action" => "start", "seed_urls" => [url],
-                                      "max_depth" => max_depth, "max_pages" => max_pages }
-                                      .merge(opts.transform_keys(&:to_s)))
-      hash = ResponseHelper.to_hash(@v1.crawl_runs.post(body))
-      raise_on_result_error ? ResponseHelper.raise_on_result_error!(hash) : hash
+    #   target-side (HTTP-200) failure instead of returning the raw hash (default true).
+    #   Ignored on the +wait: true+ path, which always returns the terminal run so
+    #   the caller can inspect a +failed+ status.
+    # @return [Hash] crawl run record with +run_id+ and +status+ (results present
+    #   when it completed while waiting)
+    def start(url:, max_depth: 2, max_pages: 100, wait: false, timeout: 300,
+              raise_on_result_error: true, **opts)
+      data = { "action" => "start", "seed_urls" => [url],
+               "max_depth" => max_depth, "max_pages" => max_pages }
+             .merge(opts.transform_keys(&:to_s))
+
+      unless wait
+        hash = ResponseHelper.to_hash(@v1.crawl_runs.post(AdditionalDataBody.new(data)))
+        return raise_on_result_error ? ResponseHelper.raise_on_result_error!(hash) : hash
+      end
+
+      # Start-and-wait: ask the server to block until terminal. Bound the first
+      # server block to a safe window, then long-poll the rest via GET so a low
+      # per-request read timeout can never abort the whole wait.
+      window = [timeout, WAIT_WINDOW_S].min
+      data["wait_until"] = "completed"
+      data["timeout_s"]  = window
+      hash = ResponseHelper.to_hash(@v1.crawl_runs.post(AdditionalDataBody.new(data)))
+      return hash if terminal?(hash)
+
+      run_id = hash["run_id"]
+      remaining = timeout - window
+      return hash if run_id.nil? || run_id.to_s.empty? || remaining <= 0
+      wait_for_completion(run_id, timeout: remaining)
+    end
+
+    # Block until an existing crawl run reaches a terminal state, or +timeout+
+    # seconds elapse. Event-driven: each iteration issues a SERVER-blocking
+    # +GET /v1/crawl-runs/{run_id}?wait=true&timeout_s=N+ that returns the moment
+    # the run goes terminal (or after its own window). There is no client +sleep+
+    # — the wait cost lives on the server, and successive blocking windows are
+    # chained until the caller's +timeout+ deadline.
+    #
+    # On timeout the current non-terminal run is returned (never raises for a
+    # slow run); a terminal run is returned as soon as it is observed.
+    #
+    # @param run_id [String] the run to wait on
+    # @param timeout [Integer] total seconds to wait (default 300)
+    # @return [Hash] the run record (terminal when it finished in time)
+    def wait_for_completion(run_id, timeout: 300)
+      if run_id.nil? || run_id.to_s.empty?
+        raise ArgumentError, "run_id is required"
+      end
+
+      deadline = monotonic + timeout
+      loop do
+        remaining = deadline - monotonic
+        # Deadline reached — one final non-blocking read of the current state.
+        return get(run_id) if remaining <= 0
+
+        window = [remaining, WAIT_WINDOW_S].min
+        run = fetch_waiting(run_id, window)
+        return run if terminal?(run)
+        # The server just blocked for ~window seconds; loop straight into the
+        # next blocking window. No client-side delay.
+      end
     end
 
     # List crawl runs.
@@ -322,6 +577,33 @@ module Ghostcrawl
     def cancel(run_id)
       ResponseHelper.to_hash(@v1.crawl_runs.by_run_id(run_id).cancel.post)
     end
+
+    private
+
+    # True when +run+ is a Hash carrying a terminal +status+.
+    def terminal?(run)
+      run.is_a?(Hash) && TERMINAL_STATUSES.include?(run["status"].to_s)
+    end
+
+    # Issues one SERVER-blocking GET on the run with +?wait=true&timeout_s=N+.
+    #
+    # The generated item builder's +to_get_request_information+ does not wire
+    # query parameters (its URL template has no query placeholder), so we take
+    # the request it builds, extend the template with +{?wait,timeout_s}+, and
+    # set the params before sending through the raw adapter — mirroring how
+    # {Client#me} re-issues a request with the Binary (raw-JSON) response factory.
+    def fetch_waiting(run_id, timeout_s)
+      request_info = @v1.crawl_runs.by_run_id(run_id).to_get_request_information(nil)
+      request_info.url_template = "{+baseurl}/v1/crawl-runs/{run_id}{?wait,timeout_s}"
+      request_info.query_parameters["wait"] = "true"
+      request_info.query_parameters["timeout_s"] = timeout_s.to_i.to_s
+      ResponseHelper.to_hash(@adapter.send_async(request_info, Ghostcrawl::V1::Binary, {}))
+    end
+
+    # Monotonic clock (seconds) — immune to wall-clock adjustments during a wait.
+    def monotonic
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
   end
 
   # Manage browser sessions — /v1/sessions.
@@ -347,7 +629,7 @@ module Ghostcrawl
     # Extend a session's TTL.
     # Delegates to POST /v1/sessions/{id}/extend via the generated builder.
     def extend(session_id, duration_seconds: 300)
-      ResponseHelper.to_hash(@v1.sessions.by_profile__id(session_id).extend.post(
+      ResponseHelper.to_hash(@v1.sessions.by_profile_id(session_id).extend.post(
         AdditionalDataBody.new({ "ttl_seconds" => duration_seconds })
       ))
     end
@@ -355,14 +637,15 @@ module Ghostcrawl
     # Release a session back to the pool.
     # Delegates to POST /v1/sessions/{id}/release via the generated builder.
     def release(session_id)
-      ResponseHelper.to_hash(@v1.sessions.by_profile__id(session_id).release.post)
+      ResponseHelper.to_hash(@v1.sessions.by_profile_id(session_id).release.post)
     end
   end
 
   # Manage identity profiles — /v1/profiles.
   class ProfilesClient
-    def initialize(v1)
+    def initialize(v1, adapter = nil)
       @v1 = v1
+      @adapter = adapter
     end
 
     # List all profiles.
@@ -392,15 +675,18 @@ module Ghostcrawl
 
     # Delete a profile.
     # Delegates to DELETE /v1/profiles/{name} via the generated builder.
+    # The endpoint answers 204 No Content; routing through {ResponseHelper.void_request!}
+    # avoids the Kiota JSON parser choking on the empty body.
     def delete(name)
-      ResponseHelper.to_hash(@v1.profiles.by_name(name).delete)
+      ResponseHelper.void_request!(@adapter, @v1.profiles.by_name(name).to_delete_request_information(nil))
     end
   end
 
   # Manage webhooks — /v1/webhooks.
   class WebhooksClient
-    def initialize(v1)
+    def initialize(v1, adapter = nil)
       @v1 = v1
+      @adapter = adapter
     end
 
     # List all webhooks.
@@ -426,10 +712,13 @@ module Ghostcrawl
     end
 
     # Delete a webhook.
-    # Delegates to DELETE /v1/webhooks/{id} via the generated builder.
+    # Delegates to DELETE /v1/webhooks/{id}. The generated builder passes a +nil+
+    # response factory (204 void), which the Kiota adapter rejects with a bare
+    # StandardError BEFORE sending the request — so we build the request info and
+    # run it through {ResponseHelper.void_request!} instead, which actually fires
+    # the DELETE and returns +{}+ on success.
     def delete(webhook_id)
-      @v1.webhooks.by_webhook_id(webhook_id).delete
-      {}
+      ResponseHelper.void_request!(@adapter, @v1.webhooks.by_webhook_id(webhook_id).to_delete_request_information(nil))
     end
 
     # Rotate the signing secret for a webhook.
@@ -441,8 +730,9 @@ module Ghostcrawl
 
   # Manage schedules — /v1/schedules.
   class SchedulesClient
-    def initialize(v1)
+    def initialize(v1, adapter = nil)
       @v1 = v1
+      @adapter = adapter
     end
 
     # List all schedules.
@@ -459,16 +749,45 @@ module Ghostcrawl
 
     # Create a new schedule.
     # Delegates to POST /v1/schedules via the generated SchedulesRequestBuilder.
-    def create(cron:, task:, **opts)
-      data = { "cron_expr" => cron, "task" => task }.merge(opts.transform_keys(&:to_s))
+    #
+    # The API's ScheduleCreateRequest requires +name+, +job_type+
+    # ("scrape" | "crawl" | "change_monitor"), +cron_expr+, and +job_params+
+    # (the full scrape/crawl request body). It rejects any other top-level field
+    # with a 422 — so a legacy +task+ key must be TRANSLATED, never forwarded.
+    #
+    # @param cron [String] cron expression (sent as +cron_expr+)
+    # @param name [String] schedule name (required by the API)
+    # @param job_type [String, nil] "scrape" | "crawl" | "change_monitor"
+    # @param job_params [Hash, nil] full job request body (e.g. { url: ... })
+    # @param task [Hash, nil] DEPRECATED legacy shape { "action" => ..., ...rest };
+    #   when given (and job_type/job_params are absent) it is split into
+    #   +job_type+ (from +task["action"]+) and +job_params+ (the remaining keys).
+    def create(cron:, name: nil, job_type: nil, job_params: nil, task: nil, **opts)
+      # Back-compat: derive job_type/job_params from a legacy `task` hash.
+      if task.is_a?(Hash)
+        t = task.transform_keys(&:to_s)
+        job_type ||= t["action"] || t["job_type"] || t["type"]
+        job_params ||= t.reject { |k, _| %w[action job_type type].include?(k) }
+      end
+
+      data = { "cron_expr" => cron }
+      data["name"]       = name       unless name.nil?
+      data["job_type"]   = job_type   unless job_type.nil?
+      data["job_params"] = job_params unless job_params.nil?
+      # opts may override/supply name/job_type/job_params/notify_webhook/monitor_mode.
+      # Never forward a raw `task` field — the API 422s on it.
+      data.merge!(opts.transform_keys(&:to_s).reject { |k, _| k == "task" })
+
       ResponseHelper.to_hash(@v1.schedules.post(AdditionalDataBody.new(data)))
     end
 
     # Delete a schedule.
-    # Delegates to DELETE /v1/schedules/{id} via the generated builder.
+    # Delegates to DELETE /v1/schedules/{id}. The generated builder returns a Fiber
+    # that must be resumed to actually send the request (the old +; {}+ discarded it,
+    # so the DELETE never fired). {ResponseHelper.void_request!} runs it and tolerates
+    # the empty 204 body.
     def delete(schedule_id)
-      @v1.schedules.by_schedule_id(schedule_id).delete
-      {}
+      ResponseHelper.void_request!(@adapter, @v1.schedules.by_schedule_id(schedule_id).to_delete_request_information(nil))
     end
   end
 
@@ -519,8 +838,9 @@ module Ghostcrawl
 
   # Manage session recordings — /v1/recordings.
   class RecordingsClient
-    def initialize(v1)
+    def initialize(v1, adapter = nil)
       @v1 = v1
+      @adapter = adapter
     end
 
     # List all recordings.
@@ -531,15 +851,17 @@ module Ghostcrawl
 
     # Get a recording by ID.
     # Delegates to GET /v1/recordings/{id} via the generated builder.
+    # (The generated accessor is +by_recording_id+, single underscore.)
     def get(recording_id)
-      ResponseHelper.to_hash(@v1.recordings.by_recording__id(recording_id).get)
+      ResponseHelper.to_hash(@v1.recordings.by_recording_id(recording_id).get)
     end
 
     # Delete a recording.
-    # Delegates to DELETE /v1/recordings/{id} via the generated builder.
+    # Delegates to DELETE /v1/recordings/{id}. Same void-DELETE defect as webhooks
+    # (generated builder passes a +nil+ factory → adapter raises before sending), so
+    # we route through {ResponseHelper.void_request!}.
     def delete(recording_id)
-      @v1.recordings.by_recording__id(recording_id).delete
-      {}
+      ResponseHelper.void_request!(@adapter, @v1.recordings.by_recording_id(recording_id).to_delete_request_information(nil))
     end
   end
 
@@ -628,13 +950,18 @@ module Ghostcrawl
       # (Binary) response factory instead of the typed MeResponse model.
       @adapter = adapter
 
-      @crawl_runs  = CrawlRunsClient.new(@v1)
+      # CrawlRunsClient needs the raw adapter for the event-driven long-poll wait
+      # (the generated item builder can't attach ?wait/&timeout_s query params).
+      @crawl_runs  = CrawlRunsClient.new(@v1, @adapter)
       @sessions    = SessionsClient.new(@v1)
-      @profiles    = ProfilesClient.new(@v1)
-      @webhooks    = WebhooksClient.new(@v1)
-      @schedules   = SchedulesClient.new(@v1)
+      # Sub-clients with void-DELETE endpoints (204 No Content) also need the raw
+      # adapter so ResponseHelper.void_request! can work around the broken generated
+      # delete builders. See ResponseHelper.void_request! for the two Kiota defects.
+      @profiles    = ProfilesClient.new(@v1, @adapter)
+      @webhooks    = WebhooksClient.new(@v1, @adapter)
+      @schedules   = SchedulesClient.new(@v1, @adapter)
       @datasets    = DatasetsClient.new(@v1)
-      @recordings  = RecordingsClient.new(@v1)
+      @recordings  = RecordingsClient.new(@v1, @adapter)
       @kv          = KVClient.new(@v1)
     end
 
@@ -722,17 +1049,34 @@ module Ghostcrawl
 
     # Start a deep crawl from a seed URL.
     # Delegates to POST /v1/crawl/deep via the generated CrawlDeepRequestBuilder.
+    #
+    # Pass +wait: true+ to block until the run is terminal. The deep-crawl start
+    # returns a +run_id+, then the wait is handled by the same event-driven
+    # server-blocking long-poll as {CrawlRunsClient#wait_for_completion} — no
+    # client-side +sleep+ loop.
+    #
     # @param url [String] seed URL
     # @param max_depth [Integer] maximum crawl depth (default 2)
     # @param max_pages [Integer] maximum pages (default 100)
+    # @param wait [Boolean] block until the run is terminal (default false)
+    # @param timeout [Integer] total seconds to wait when +wait: true+ (default 300)
     # @param raise_on_result_error [Boolean] raise {Ghostcrawl::ScrapeError} on a
-    #   target-side (HTTP-200) failure instead of returning the raw hash (default true)
-    # @return [Hash] crawl run record
-    def crawl(url:, max_depth: 2, max_pages: 100, raise_on_result_error: true, **opts)
+    #   target-side (HTTP-200) failure instead of returning the raw hash (default true).
+    #   Ignored on the +wait: true+ path, which returns the terminal run.
+    # @return [Hash] crawl run record (terminal when +wait: true+ and it finished in time)
+    def crawl(url:, max_depth: 2, max_pages: 100, wait: false, timeout: 300,
+              raise_on_result_error: true, **opts)
       data = { "seed_urls" => [url], "max_depth" => max_depth,
                "max_urls" => max_pages }.merge(opts.transform_keys(&:to_s))
       hash = ResponseHelper.to_hash(@v1.crawl.deep.post(AdditionalDataBody.new(data)))
-      raise_on_result_error ? ResponseHelper.raise_on_result_error!(hash) : hash
+
+      unless wait
+        return raise_on_result_error ? ResponseHelper.raise_on_result_error!(hash) : hash
+      end
+
+      run_id = hash["run_id"]
+      return hash if run_id.nil? || run_id.to_s.empty?
+      @crawl_runs.wait_for_completion(run_id, timeout: timeout)
     end
 
     # Map all URLs reachable from a seed URL.
@@ -763,14 +1107,22 @@ module Ghostcrawl
 
     # Normalizes a +"content"+ key onto a decoded scrape response.
     #
-    # The API returns the rendered page under a FORMAT-SPECIFIC key
-    # (+"markdown"+, +"html"+, or +"text"+), but the documented quickstart reads
-    # +result["content"]+. This mirrors that value onto +"content"+ in place,
-    # KEEPING the format-specific key intact (backward compatible).
+    # The API has two scrape response shapes:
+    #   * the FLAT +format="markdown"+ shape, which carries the rendered page at
+    #     the top level under the format-specific key (+"markdown"+/+"html"+/
+    #     +"text"+) but omits +identity_id+; and
+    #   * the standard ENVELOPE ({ +status+, +results+, +routing_mode+,
+    #     +request_class+, +identity_id+ }), where the rendered page lives under
+    #     +results[0]+ and the top-level +identity_id+ is present.
+    # The documented quickstart reads +result["content"]+, so this mirrors the
+    # rendered page onto +"content"+ in place from WHICHEVER shape is present,
+    # KEEPING the original keys intact (backward compatible).
     #
     # No-op unless +result+ is a Hash that does not already carry +"content"+.
     # The value chosen is: the field named by +result["format"]+ when that field
-    # is a String, else the first String among +"markdown"+, +"html"+, +"text"+.
+    # is a String, else the first String among +"markdown"+, +"html"+, +"text"+ —
+    # looked up first at the top level (flat shape), then on +results[0]+
+    # (envelope shape).
     #
     # @param result [Object] the decoded response (only mutated when a Hash)
     # @return [Object] the same +result+, unchanged reference
@@ -779,21 +1131,34 @@ module Ghostcrawl
       return result unless result.is_a?(Hash) && !result.key?("content")
 
       fmt = result["format"]
-      value = result[fmt] if fmt.is_a?(String)
-      value = nil unless value.is_a?(String)
+      value = scrape_content_from(result, fmt)
 
+      # Envelope shape: the page is under results[0], not at the top level.
       unless value.is_a?(String)
-        %w[markdown html text].each do |key|
-          candidate = result[key]
-          if candidate.is_a?(String)
-            value = candidate
-            break
-          end
-        end
+        first = result["results"]
+        first = first.first if first.is_a?(Array)
+        value = scrape_content_from(first, fmt) if first.is_a?(Hash)
       end
 
       result["content"] = value if value.is_a?(String)
       result
     end
+
+    # Picks the rendered-page String out of a scrape result Hash: the field named
+    # by +fmt+ when present, else the first String among +"markdown"+, +"html"+,
+    # +"text"+. Returns +nil+ when +src+ carries no such String.
+    # @api private
+    def scrape_content_from(src, fmt)
+      return nil unless src.is_a?(Hash)
+
+      value = src[fmt] if fmt.is_a?(String)
+      return value if value.is_a?(String)
+
+      %w[markdown html text].each do |key|
+        candidate = src[key]
+        return candidate if candidate.is_a?(String)
+      end
+      nil
+    end
   end
 end

data/lib/ghostcrawl/error_codes.rb

@@ -92,7 +92,14 @@ module Ghostcrawl
       when 403 then FORBIDDEN
       when 404 then NOT_FOUND
       when 409 then CONFLICT
-      when 422 then BYO_PROXY_INVALID
+      # 422 is the generic validation status the API returns for ANY malformed
+      # request (its canonical code is "bad_request": missing field, failed
+      # extraction validation, etc.). byo_proxy_invalid is only ONE 422 sub-case
+      # and, when it genuinely occurs, arrives WITH a problem+json body carrying
+      # code:"byo_proxy_invalid" (which raise_translated reads first). Using the
+      # BYO-specific code as the blanket 422 fallback mislabels every unrelated
+      # validation 422. Fall back to the generic bad_request instead.
+      when 422 then BAD_REQUEST
       when 429 then RATE_LIMITED
       when 503 then SERVICE_UNAVAILABLE
       when 504 then FLEET_TIMEOUT

data/lib/ghostcrawl/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ghostcrawl
-  VERSION = "2.2.2"
+  VERSION = "2.3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ghostcrawl
 version: !ruby/object:Gem::Version
-  version: 2.2.2
+  version: 2.3.0
 platform: ruby
 authors:
 - GhostCrawl
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-07-01 00:00:00.000000000 Z
+date: 2026-07-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: microsoft_kiota_abstractions
@@ -225,9 +225,6 @@ files:
 - _generated/v1/billing/usage/usage.rb
 - _generated/v1/billing/usage/usage_request_builder.rb
 - _generated/v1/binary/binary.rb
-- _generated/v1/binary/binary_request_builder.rb
-- _generated/v1/binary/download/download.rb
-- _generated/v1/binary/download/download_request_builder.rb
 - _generated/v1/budgets/budgets.rb
 - _generated/v1/budgets/budgets_request_builder.rb
 - _generated/v1/budgets/policy/item/item.rb
@@ -431,7 +428,7 @@ files:
 - lib/ghostcrawl/version.rb
 homepage: https://ghostcrawl.io
 licenses:
-- Nonstandard
+- MIT
 metadata:
   homepage_uri: https://ghostcrawl.io
   documentation_uri: https://docs.ghostcrawl.io

data/_generated/v1/binary/binary_request_builder.rb

@@ -1,31 +0,0 @@
-require 'microsoft_kiota_abstractions'
-require_relative '../../ghostcrawl'
-require_relative '../v1'
-require_relative './binary'
-require_relative './download/download_request_builder'
-
-module Ghostcrawl
-    module V1
-        module Binary
-            ## 
-            # Builds and executes requests for operations under #v1#binary
-            class BinaryRequestBuilder < MicrosoftKiotaAbstractions::BaseRequestBuilder
-                
-                ## 
-                # The download property
-                def download()
-                    return Ghostcrawl::V1::Binary::Download::DownloadRequestBuilder.new(@path_parameters, @request_adapter)
-                end
-                ## 
-                ## Instantiates a new BinaryRequestBuilder and sets the default values.
-                ## @param path_parameters Path parameters for the request
-                ## @param request_adapter The request adapter to use to execute the requests.
-                ## @return a void
-                ## 
-                def initialize(path_parameters, request_adapter)
-                    super(path_parameters, request_adapter, "{+baseurl}/v1/binary")
-                end
-            end
-        end
-    end
-end

data/_generated/v1/binary/download/download_request_builder.rb

@@ -1,77 +0,0 @@
-require 'microsoft_kiota_abstractions'
-require_relative '../../../ghostcrawl'
-require_relative '../../../models/h_t_t_p_validation_error'
-require_relative '../../v1'
-require_relative '../binary'
-require_relative './download'
-
-module Ghostcrawl
-    module V1
-        module Binary
-            module Download
-                ## 
-                # Builds and executes requests for operations under #v1#binary#download
-                class DownloadRequestBuilder < MicrosoftKiotaAbstractions::BaseRequestBuilder
-                    
-                    ## 
-                    ## Instantiates a new DownloadRequestBuilder and sets the default values.
-                    ## @param path_parameters Path parameters for the request
-                    ## @param request_adapter The request adapter to use to execute the requests.
-                    ## @return a void
-                    ## 
-                    def initialize(path_parameters, request_adapter)
-                        super(path_parameters, request_adapter, "{+baseurl}/v1/binary/download?engine={engine}{&release_id*}")
-                    end
-                    ## 
-                    ## Download your licensed engine binary (Chrome, Firefox, or WebKit) for self-hosting.Select the engine via the ``engine`` query parameter. The binary is streamedto the client and bound to your account.
-                    ## @param request_configuration Configuration for the request such as headers, query parameters, and middleware options.
-                    ## @return a Fiber of binary
-                    ## 
-                    def get(request_configuration=nil)
-                        request_info = self.to_get_request_information(
-                            request_configuration
-                        )
-                        error_mapping = Hash.new
-                        error_mapping["422"] = lambda {|pn| Ghostcrawl::Models::HTTPValidationError.create_from_discriminator_value(pn) }
-                        return @request_adapter.send_async(request_info, Binary, error_mapping)
-                    end
-                    ## 
-                    ## Download your licensed engine binary (Chrome, Firefox, or WebKit) for self-hosting.Select the engine via the ``engine`` query parameter. The binary is streamedto the client and bound to your account.
-                    ## @param request_configuration Configuration for the request such as headers, query parameters, and middleware options.
-                    ## @return a request_information
-                    ## 
-                    def to_get_request_information(request_configuration=nil)
-                        request_info = MicrosoftKiotaAbstractions::RequestInformation.new()
-                        unless request_configuration.nil?
-                            request_info.add_headers_from_raw_object(request_configuration.headers)
-                            request_info.set_query_string_parameters_from_raw_object(request_configuration.query_parameters)
-                            request_info.add_request_options(request_configuration.options)
-                        end
-                        request_info.url_template = @url_template
-                        request_info.path_parameters = @path_parameters
-                        request_info.http_method = :GET
-                        request_info.headers.try_add('Accept', 'application/json')
-                        return request_info
-                    end
-                    ## 
-                    ## Returns a request builder with the provided arbitrary URL. Using this method means any other path or query parameters are ignored.
-                    ## @param raw_url The raw URL to use for the request builder.
-                    ## @return a download_request_builder
-                    ## 
-                    def with_url(raw_url)
-                        raise StandardError, 'raw_url cannot be null' if raw_url.nil?
-                        return DownloadRequestBuilder.new(raw_url, @request_adapter)
-                    end
-
-                    ## 
-                    # Download your licensed engine binary (Chrome, Firefox, or WebKit) for self-hosting.Select the engine via the ``engine`` query parameter. The binary is streamedto the client and bound to your account.
-                    class DownloadRequestBuilderGetQueryParameters
-                        
-                        attr_accessor :engine
-                        attr_accessor :release_id
-                    end
-                end
-            end
-        end
-    end
-end