ghostcrawl 2.2.2 → 2.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ff09081da25b9c78b261437d5bc26c2a4b25f3e727a53e8393f1adb00aa6536c
-  data.tar.gz: ad0869fe147e1c33a49fadd364bc93ce5857fd73eb03b584d4c1ce618e1092a9
+  metadata.gz: e499bc41b516f5cea6e37d2ec06c17e8fe770a8d928e8415e470be0874d56786
+  data.tar.gz: ed4d7fa29dcdc4f1316d0f870555bf32cb61063b3bc7efc07fe46e5682626a50
 SHA512:
-  metadata.gz: '085820b153fb6b70ea26f6e866787388835cfec7e4cce86498f83551abe6b4c8800b5eb14f86203c59ca77b7e70cffbd459d0d84f7a75fc69f5337914bcf60e6'
-  data.tar.gz: 11a00207fc7469d02bec976018c8e627ea11c0da0ce78d4dc5a6d5b76fced17a5f3a37053159e0bde94dd0c44e4d3e273e7cf57cdac1d652a410bcba4da814ff
+  metadata.gz: 349f33fe5785ed3458e5e749b7a45e56bf6e9aa8fdcc4b68625792a174676733ac1d0bed6f6148bed07716059498f5f5e565bf4d013357af1d8d4632f5d9ffb3
+  data.tar.gz: 7939615bd575ac30e78455194a2979f204d82523b548ede855da34ff06f1422f8e8f676f44fe7ad47262ad63f03b6ef68c39acdc190f9dcb981c06e39717191f

data/lib/ghostcrawl/client.rb

@@ -16,6 +16,7 @@
 #   client = Ghostcrawl::Client.new(token: "gck_live_YOUR_KEY")
 #   result = client.scrape(url: "https://example.com")
 
+require "json"
 require "microsoft_kiota_abstractions"
 require "microsoft_kiota_faraday"
 require "microsoft_kiota_serialization_json"
@@ -57,6 +58,72 @@ module Ghostcrawl
         super
       end
     end
+
+    # Fix 1b: the pinned Kiota writer's write_collection_of_object_values calls
+    # `self.write_object_value(nil, v).writer` on each element — it relies on
+    # write_object_value(nil, ...) returning a *writer object* that responds to
+    # `.writer`. Our write_object_value override above (Fix 1) changes that
+    # nil-key contract to serialize-into-self and return the serialize result
+    # (a Hash / nil), so the upstream `.writer` call raises
+    # `NoMethodError: undefined method 'writer' for {}:Hash`. That aborts
+    # serialize for ANY response model carrying a collection of typed objects
+    # (WebhookListResponse#items, etc.), which ResponseHelper.serialize_parsable
+    # then swallows -> the facade returns {} and silently drops the whole list.
+    #
+    # Fix: serialize each element into its OWN fresh writer and collect the raw
+    # per-element hashes directly, never depending on the write_object_value
+    # return contract. Mirrors the non-nil upstream branch (line 154) without the
+    # broken `.writer` deref.
+    def write_collection_of_object_values(key, values)
+      return unless values
+      hashes = values.map do |v|
+        temp = MicrosoftKiotaSerializationJson::JsonSerializationWriter.new
+        v.serialize(temp)
+        temp.writer
+      end
+      if key.nil?
+        hashes
+      else
+        writer[key] = hashes
+      end
+    end
+
+    # Fix 1c: the pinned Kiota writer's write_any_value (used by
+    # write_additional_data, i.e. EVERY field AdditionalDataBody sends) has no
+    # case for a plain Hash and mis-handles an Array of Hashes:
+    #   * a Hash hits the `value.is_a? Object` branch -> `return value.to_s`,
+    #     which returns a string but NEVER assigns @writer[key] -> the whole
+    #     nested object is SILENTLY DROPPED from the request body.
+    #   * an Array is sent to write_collection_of_primitive_values, which mangles
+    #     any non-primitive (Hash) element.
+    # This drops/mangles every nested field the facade sends: scrape(extract_schema),
+    # extract(schema), crawl/crawl_runs opts, datasets.append(rows: [ {..} ]),
+    # schedules.create(job_params: {..}) — the last of which the API REQUIRES,
+    # so the request 422s ("job_params required") even though the caller supplied it.
+    #
+    # Fix: intercept Hash and Array here and write a fully-recursed, JSON-native
+    # structure straight into @writer[key]. The writer emits @writer.to_json at the
+    # end, so plain nested Hash/Array values serialize correctly. Everything else
+    # falls through to the upstream primitive handling.
+    def write_any_value(key, value)
+      if value.is_a?(Hash) || value.is_a?(Array)
+        return super unless key # nil-key arrays keep upstream behavior
+        writer[key] = KiotaWriterFix.jsonable(value)
+        return
+      end
+      super
+    end
+
+    # Recursively converts a value into a JSON-native structure (Hash/Array of
+    # plain scalars), so it round-trips through @writer.to_json intact.
+    # @api private
+    def self.jsonable(value)
+      case value
+      when Hash  then value.each_with_object({}) { |(k, v), h| h[k.to_s] = jsonable(v) }
+      when Array then value.map { |v| jsonable(v) }
+      else value
+      end
+    end
   end
 
   # Patch the writer class once, idempotently
@@ -190,15 +257,128 @@ module Ghostcrawl
       when NilClass
         {}
       else
-        # Typed Parsable: try serializing to JSON then parsing back
-        if value.respond_to?(:additional_data) && value.additional_data
-          value.additional_data.transform_values { |v| to_hash(v) }
-        elsif value.respond_to?(:to_h)
-          value.to_h.transform_values { |v| to_hash(v) }
-        else
-          value
-        end
+        # Typed Parsable (Kiota-generated model). Kiota models keep their spec
+        # fields in typed instance variables, NOT in +additional_data+ (which is
+        # only the overflow bucket for unmapped keys). Reading +additional_data+
+        # alone therefore returns +{}+ for a fully-typed response (e.g.
+        # MapResponse{@links,@success}, WebhookListResponse{@items,@total}),
+        # silently dropping the real payload.
+        #
+        # Recover the typed fields by round-tripping the model through its own
+        # +serialize+ (the Parsable contract) into a JSON writer and parsing the
+        # result back to a plain Hash. Then overlay any +additional_data+ (unmapped
+        # keys the server sent that the model didn't declare). Degrade gracefully:
+        # a model whose +serialize+ raises (e.g. an unresolved composed-type member)
+        # falls back to the prior additional_data/to_h behavior so nothing regresses.
+        parsable_to_hash(value)
+      end
+    end
+
+    # Converts a typed Kiota Parsable into a plain Hash by serializing it via its
+    # own +serialize+ and parsing the emitted JSON, then merging +additional_data+.
+    # Falls back to +additional_data+ / +to_h+ / the value itself on any failure.
+    # @api private
+    def self.parsable_to_hash(value)
+      serialized = serialize_parsable(value)
+      if serialized.is_a?(Hash)
+        extra = (value.additional_data if value.respond_to?(:additional_data)) || {}
+        merged = serialized.merge(extra.transform_keys(&:to_s))
+        return merged.transform_values { |v| to_hash(v) }
+      end
+
+      if value.respond_to?(:additional_data) && value.additional_data && !value.additional_data.empty?
+        value.additional_data.transform_values { |v| to_hash(v) }
+      elsif value.respond_to?(:to_h)
+        value.to_h.transform_values { |v| to_hash(v) }
+      elsif value.respond_to?(:additional_data) && value.additional_data
+        value.additional_data.transform_values { |v| to_hash(v) }
+      else
+        value
+      end
+    end
+
+    # Serializes a Parsable to a Ruby Hash using the pinned Kiota JSON writer.
+    # Returns the parsed Hash, or +nil+ when the model isn't a serializable
+    # Parsable or serialization/parse fails (caller then falls back).
+    # @api private
+    def self.serialize_parsable(value)
+      return nil unless value.respond_to?(:serialize)
+
+      writer = MicrosoftKiotaSerializationJson::JsonSerializationWriter.new
+      value.serialize(writer)
+      content = writer.get_serialized_content
+      json = content.is_a?(String) ? content : content.read
+      return nil if json.nil? || json.empty?
+
+      parsed = JSON.parse(json)
+      parsed.is_a?(Hash) ? parsed : nil
+    rescue StandardError
+      nil
+    end
+
+    # Executes a request that returns NO response body (an HTTP 204, or any DELETE
+    # whose spec maps the success response to +void+) and returns a plain Hash.
+    #
+    # Why this exists — two upstream defects in the pinned Kiota Ruby runtime that
+    # make the generated void-DELETE builders unusable:
+    #
+    #   1. The generated +delete+ for a void response calls
+    #      +send_async(request_info, nil, ...)+, and the Faraday adapter hard-raises
+    #      +"factory cannot be null"+ BEFORE the request is ever sent — so the DELETE
+    #      never reaches the server and the caller gets a bare +StandardError+
+    #      (not a {GhostcrawlError}).
+    #   2. For void DELETEs the adapter DOES send maps to +Binary+, the response is an
+    #      empty 204 body, and +get_root_parse_node+ then feeds +""+ to the JSON parser,
+    #      raising +JSON::ParserError+ on a perfectly successful delete.
+    #
+    # This helper reuses the adapter's OWN public request pipeline
+    # (+convert_to_native_request_async+ applies base-url + auth, +run_request+
+    # sends it) but skips body deserialization entirely: a 2xx returns +{}+ (or the
+    # decoded JSON when the server did send a body, e.g. +{"deleted":true}+), and a
+    # >=400 is translated into the documented typed {GhostcrawlError} hierarchy.
+    #
+    # @param adapter [MicrosoftKiotaFaraday::FaradayRequestAdapter]
+    # @param request_info [MicrosoftKiotaAbstractions::RequestInformation]
+    # @return [Hash] +{}+ on an empty 2xx, or the decoded body when one is present
+    # @api private
+    def self.void_request!(adapter, request_info)
+      # Replicate the first half of the adapter's send_async by hand — apply the
+      # base URL, run the bearer-auth Fiber (which mutates request_info's headers),
+      # then build the native Faraday request. We deliberately do NOT call the
+      # adapter's convert_to_native_request_async helper: its body uses a bare
+      # +return+ inside a Fiber, which raises LocalJumpError("unexpected return")
+      # when resumed under Ruby 3.x.
+      request_info.path_parameters["baseurl"] = adapter.get_base_url
+      # authenticate_request returns a Fiber only when the Authorization header is
+      # not already present; it returns nil otherwise. Guard the resume.
+      auth_fiber = adapter.authentication_provider.authenticate_request(request_info)
+      auth_fiber.resume if auth_fiber.respond_to?(:resume)
+      request = adapter.get_request_from_request_info(request_info)
+      response = adapter.client.run_request(
+        request.http_method, request.path, request.body, request.headers
+      )
+
+      status = response.status
+      if status >= 400
+        # Reuse the typed-error translation: synthesize a message carrying the
+        # ":<status>" token raise_translated keys off, plus any body text.
+        body = response.body.to_s
+        err = MicrosoftKiotaAbstractions::ApiError.new(
+          "The server returned an unexpected status code:#{status}" \
+          "#{body.empty? ? '' : " #{body}"}"
+        )
+        Ghostcrawl.raise_translated(err)
       end
+
+      body = response.body.to_s
+      return {} if body.strip.empty?
+
+      parsed = (JSON.parse(body) rescue nil)
+      parsed.is_a?(Hash) ? parsed.transform_values { |v| to_hash(v) } : {}
+    rescue Ghostcrawl::GhostcrawlError
+      raise
+    rescue StandardError => e
+      Ghostcrawl.raise_translated(e)
     end
 
     # Inspects a decoded HTTP-200 response hash for a RESULT-channel failure (the
@@ -347,7 +527,7 @@ module Ghostcrawl
     # Extend a session's TTL.
     # Delegates to POST /v1/sessions/{id}/extend via the generated builder.
     def extend(session_id, duration_seconds: 300)
-      ResponseHelper.to_hash(@v1.sessions.by_profile__id(session_id).extend.post(
+      ResponseHelper.to_hash(@v1.sessions.by_profile_id(session_id).extend.post(
         AdditionalDataBody.new({ "ttl_seconds" => duration_seconds })
       ))
     end
@@ -355,14 +535,15 @@ module Ghostcrawl
     # Release a session back to the pool.
     # Delegates to POST /v1/sessions/{id}/release via the generated builder.
     def release(session_id)
-      ResponseHelper.to_hash(@v1.sessions.by_profile__id(session_id).release.post)
+      ResponseHelper.to_hash(@v1.sessions.by_profile_id(session_id).release.post)
     end
   end
 
   # Manage identity profiles — /v1/profiles.
   class ProfilesClient
-    def initialize(v1)
+    def initialize(v1, adapter = nil)
       @v1 = v1
+      @adapter = adapter
     end
 
     # List all profiles.
@@ -392,15 +573,18 @@ module Ghostcrawl
 
     # Delete a profile.
     # Delegates to DELETE /v1/profiles/{name} via the generated builder.
+    # The endpoint answers 204 No Content; routing through {ResponseHelper.void_request!}
+    # avoids the Kiota JSON parser choking on the empty body.
     def delete(name)
-      ResponseHelper.to_hash(@v1.profiles.by_name(name).delete)
+      ResponseHelper.void_request!(@adapter, @v1.profiles.by_name(name).to_delete_request_information(nil))
     end
   end
 
   # Manage webhooks — /v1/webhooks.
   class WebhooksClient
-    def initialize(v1)
+    def initialize(v1, adapter = nil)
       @v1 = v1
+      @adapter = adapter
     end
 
     # List all webhooks.
@@ -426,10 +610,13 @@ module Ghostcrawl
     end
 
     # Delete a webhook.
-    # Delegates to DELETE /v1/webhooks/{id} via the generated builder.
+    # Delegates to DELETE /v1/webhooks/{id}. The generated builder passes a +nil+
+    # response factory (204 void), which the Kiota adapter rejects with a bare
+    # StandardError BEFORE sending the request — so we build the request info and
+    # run it through {ResponseHelper.void_request!} instead, which actually fires
+    # the DELETE and returns +{}+ on success.
     def delete(webhook_id)
-      @v1.webhooks.by_webhook_id(webhook_id).delete
-      {}
+      ResponseHelper.void_request!(@adapter, @v1.webhooks.by_webhook_id(webhook_id).to_delete_request_information(nil))
     end
 
     # Rotate the signing secret for a webhook.
@@ -441,8 +628,9 @@ module Ghostcrawl
 
   # Manage schedules — /v1/schedules.
   class SchedulesClient
-    def initialize(v1)
+    def initialize(v1, adapter = nil)
       @v1 = v1
+      @adapter = adapter
     end
 
     # List all schedules.
@@ -459,16 +647,45 @@ module Ghostcrawl
 
     # Create a new schedule.
     # Delegates to POST /v1/schedules via the generated SchedulesRequestBuilder.
-    def create(cron:, task:, **opts)
-      data = { "cron_expr" => cron, "task" => task }.merge(opts.transform_keys(&:to_s))
+    #
+    # The API's ScheduleCreateRequest requires +name+, +job_type+
+    # ("scrape" | "crawl" | "change_monitor"), +cron_expr+, and +job_params+
+    # (the full scrape/crawl request body). It rejects any other top-level field
+    # with a 422 — so a legacy +task+ key must be TRANSLATED, never forwarded.
+    #
+    # @param cron [String] cron expression (sent as +cron_expr+)
+    # @param name [String] schedule name (required by the API)
+    # @param job_type [String, nil] "scrape" | "crawl" | "change_monitor"
+    # @param job_params [Hash, nil] full job request body (e.g. { url: ... })
+    # @param task [Hash, nil] DEPRECATED legacy shape { "action" => ..., ...rest };
+    #   when given (and job_type/job_params are absent) it is split into
+    #   +job_type+ (from +task["action"]+) and +job_params+ (the remaining keys).
+    def create(cron:, name: nil, job_type: nil, job_params: nil, task: nil, **opts)
+      # Back-compat: derive job_type/job_params from a legacy `task` hash.
+      if task.is_a?(Hash)
+        t = task.transform_keys(&:to_s)
+        job_type ||= t["action"] || t["job_type"] || t["type"]
+        job_params ||= t.reject { |k, _| %w[action job_type type].include?(k) }
+      end
+
+      data = { "cron_expr" => cron }
+      data["name"]       = name       unless name.nil?
+      data["job_type"]   = job_type   unless job_type.nil?
+      data["job_params"] = job_params unless job_params.nil?
+      # opts may override/supply name/job_type/job_params/notify_webhook/monitor_mode.
+      # Never forward a raw `task` field — the API 422s on it.
+      data.merge!(opts.transform_keys(&:to_s).reject { |k, _| k == "task" })
+
       ResponseHelper.to_hash(@v1.schedules.post(AdditionalDataBody.new(data)))
     end
 
     # Delete a schedule.
-    # Delegates to DELETE /v1/schedules/{id} via the generated builder.
+    # Delegates to DELETE /v1/schedules/{id}. The generated builder returns a Fiber
+    # that must be resumed to actually send the request (the old +; {}+ discarded it,
+    # so the DELETE never fired). {ResponseHelper.void_request!} runs it and tolerates
+    # the empty 204 body.
     def delete(schedule_id)
-      @v1.schedules.by_schedule_id(schedule_id).delete
-      {}
+      ResponseHelper.void_request!(@adapter, @v1.schedules.by_schedule_id(schedule_id).to_delete_request_information(nil))
     end
   end
 
@@ -519,8 +736,9 @@ module Ghostcrawl
 
   # Manage session recordings — /v1/recordings.
   class RecordingsClient
-    def initialize(v1)
+    def initialize(v1, adapter = nil)
       @v1 = v1
+      @adapter = adapter
     end
 
     # List all recordings.
@@ -531,15 +749,17 @@ module Ghostcrawl
 
     # Get a recording by ID.
     # Delegates to GET /v1/recordings/{id} via the generated builder.
+    # (The generated accessor is +by_recording_id+, single underscore.)
     def get(recording_id)
-      ResponseHelper.to_hash(@v1.recordings.by_recording__id(recording_id).get)
+      ResponseHelper.to_hash(@v1.recordings.by_recording_id(recording_id).get)
     end
 
     # Delete a recording.
-    # Delegates to DELETE /v1/recordings/{id} via the generated builder.
+    # Delegates to DELETE /v1/recordings/{id}. Same void-DELETE defect as webhooks
+    # (generated builder passes a +nil+ factory → adapter raises before sending), so
+    # we route through {ResponseHelper.void_request!}.
     def delete(recording_id)
-      @v1.recordings.by_recording__id(recording_id).delete
-      {}
+      ResponseHelper.void_request!(@adapter, @v1.recordings.by_recording_id(recording_id).to_delete_request_information(nil))
     end
   end
 
@@ -630,11 +850,14 @@ module Ghostcrawl
 
       @crawl_runs  = CrawlRunsClient.new(@v1)
       @sessions    = SessionsClient.new(@v1)
-      @profiles    = ProfilesClient.new(@v1)
-      @webhooks    = WebhooksClient.new(@v1)
-      @schedules   = SchedulesClient.new(@v1)
+      # Sub-clients with void-DELETE endpoints (204 No Content) also need the raw
+      # adapter so ResponseHelper.void_request! can work around the broken generated
+      # delete builders. See ResponseHelper.void_request! for the two Kiota defects.
+      @profiles    = ProfilesClient.new(@v1, @adapter)
+      @webhooks    = WebhooksClient.new(@v1, @adapter)
+      @schedules   = SchedulesClient.new(@v1, @adapter)
       @datasets    = DatasetsClient.new(@v1)
-      @recordings  = RecordingsClient.new(@v1)
+      @recordings  = RecordingsClient.new(@v1, @adapter)
       @kv          = KVClient.new(@v1)
     end
 

data/lib/ghostcrawl/error_codes.rb

@@ -92,7 +92,14 @@ module Ghostcrawl
       when 403 then FORBIDDEN
       when 404 then NOT_FOUND
       when 409 then CONFLICT
-      when 422 then BYO_PROXY_INVALID
+      # 422 is the generic validation status the API returns for ANY malformed
+      # request (its canonical code is "bad_request": missing field, failed
+      # extraction validation, etc.). byo_proxy_invalid is only ONE 422 sub-case
+      # and, when it genuinely occurs, arrives WITH a problem+json body carrying
+      # code:"byo_proxy_invalid" (which raise_translated reads first). Using the
+      # BYO-specific code as the blanket 422 fallback mislabels every unrelated
+      # validation 422. Fall back to the generic bad_request instead.
+      when 422 then BAD_REQUEST
       when 429 then RATE_LIMITED
       when 503 then SERVICE_UNAVAILABLE
       when 504 then FLEET_TIMEOUT

data/lib/ghostcrawl/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ghostcrawl
-  VERSION = "2.2.2"
+  VERSION = "2.2.3"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ghostcrawl
 version: !ruby/object:Gem::Version
-  version: 2.2.2
+  version: 2.2.3
 platform: ruby
 authors:
 - GhostCrawl