ghostcrawl 2.3.5 → 2.3.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +59 -12
  3. data/_generated/models/arrival_body.rb +325 -0
  4. data/_generated/models/arrival_body_profile_id_member1.rb +62 -0
  5. data/_generated/models/arrival_body_referrer_shape.rb +8 -0
  6. data/_generated/models/arrival_body_session_id_member1.rb +62 -0
  7. data/_generated/models/close_popups_body.rb +264 -0
  8. data/_generated/models/close_popups_body_keep_host_member1.rb +62 -0
  9. data/_generated/models/close_popups_body_profile_id_member1.rb +62 -0
  10. data/_generated/models/models.rb +18 -0
  11. data/_generated/models/mouse_body.rb +264 -0
  12. data/_generated/models/mouse_body_ops.rb +62 -0
  13. data/_generated/models/mouse_body_profile_id_member1.rb +62 -0
  14. data/_generated/models/mouse_body_session_id_member1.rb +62 -0
  15. data/_generated/models/scrape_request.rb +6 -6
  16. data/_generated/models/scrape_request_engine.rb +1 -1
  17. data/_generated/models/session_create_request.rb +3 -3
  18. data/_generated/models/tab_visit_body.rb +285 -0
  19. data/_generated/models/tab_visit_body_profile_id_member1.rb +62 -0
  20. data/_generated/models/tab_visit_body_session_id_member1.rb +62 -0
  21. data/_generated/models/touch_body.rb +264 -0
  22. data/_generated/models/touch_body_ops.rb +62 -0
  23. data/_generated/models/touch_body_profile_id_member1.rb +62 -0
  24. data/_generated/models/touch_body_session_id_member1.rb +62 -0
  25. data/_generated/v1/cdp/frame/frame_request_builder.rb +2 -2
  26. data/_generated/v1/crawl_runs/item/with_run_item_request_builder.rb +3 -3
  27. data/_generated/v1/page/arrival/arrival.rb +0 -0
  28. data/_generated/v1/page/arrival/arrival_request_builder.rb +74 -0
  29. data/_generated/v1/page/close_popups/close_popups.rb +0 -0
  30. data/_generated/v1/page/close_popups/close_popups_request_builder.rb +74 -0
  31. data/_generated/v1/page/mouse/mouse.rb +0 -0
  32. data/_generated/v1/page/mouse/mouse_request_builder.rb +74 -0
  33. data/_generated/v1/page/page_request_builder.rb +30 -0
  34. data/_generated/v1/page/tab_visit/tab_visit.rb +0 -0
  35. data/_generated/v1/page/tab_visit/tab_visit_request_builder.rb +74 -0
  36. data/_generated/v1/page/touch/touch.rb +0 -0
  37. data/_generated/v1/page/touch/touch_request_builder.rb +74 -0
  38. data/_generated/v1/profiles/item/with_name_item_request_builder.rb +2 -2
  39. data/_generated/v1/schedules/item/runs/runs_request_builder.rb +2 -2
  40. data/_generated/v1/search/jobs/item/with_job_item_request_builder.rb +2 -2
  41. data/_generated/v1/sessions/item/takeover/takeover_request_builder.rb +2 -2
  42. data/_generated/v1/sessions/sessions_request_builder.rb +3 -3
  43. data/lib/ghostcrawl/client.rb +1001 -46
  44. data/lib/ghostcrawl/error_codes.rb +26 -25
  45. data/lib/ghostcrawl/errors.rb +35 -2
  46. data/lib/ghostcrawl/version.rb +1 -1
  47. metadata +30 -2
@@ -19,6 +19,7 @@ module GhostCrawl
19
19
  CONFLICT = "conflict"
20
20
  BYO_PROXY_INVALID = "byo_proxy_invalid"
21
21
  TIER_UNAVAILABLE = "tier_unavailable"
22
+ ROUTING_MODE_INCOHERENT_CLAIM_OS = "routing_mode_incoherent_claim_os"
22
23
  RATE_LIMITED = "rate_limited"
23
24
  QUOTA_BACKEND_UNAVAILABLE = "quota_backend_unavailable"
24
25
  POOL_EXHAUSTED = "pool_exhausted"
@@ -26,7 +27,7 @@ module GhostCrawl
26
27
  RENDER_HUNG = "render_hung"
27
28
  ENGINE_CRASHED = "engine_crashed"
28
29
  RENDER_TIMEOUT = "render_timeout"
29
- ENGINE_TIMEOUT = "engine_timeout"
30
+ ENGINE_TIMEOUT = "engine_timeout"
30
31
  SERVICE_UNAVAILABLE = "service_unavailable"
31
32
  INTERNAL_ERROR = "internal_error"
32
33
 
@@ -39,29 +40,29 @@ module GhostCrawl
39
40
 
40
41
  # Full catalog: code => {http:, retryable:, channel:}. Mirrors codes.json.
41
42
  CATALOG = {
42
- BAD_REQUEST => { http: 400, retryable: false, channel: "problem" },
43
- UNAUTHORIZED => { http: 401, retryable: false, channel: "problem" },
44
- FORBIDDEN => { http: 403, retryable: false, channel: "problem" },
45
- PAYMENT_REQUIRED => { http: 402, retryable: false, channel: "problem" },
46
- NOT_FOUND => { http: 404, retryable: false, channel: "problem" },
47
- CONFLICT => { http: 409, retryable: false, channel: "problem" },
48
- BYO_PROXY_INVALID => { http: 422, retryable: false, channel: "problem" },
49
- TIER_UNAVAILABLE => { http: 400, retryable: false, channel: "problem" },
50
- RATE_LIMITED => { http: 429, retryable: true, channel: "problem" },
51
- QUOTA_BACKEND_UNAVAILABLE => { http: 503, retryable: true, channel: "problem" },
52
- POOL_EXHAUSTED => { http: 503, retryable: true, channel: "problem" },
53
- EGRESS_INTEGRITY_FAILED => { http: 503, retryable: true, channel: "problem" },
54
- RENDER_HUNG => { http: 503, retryable: true, channel: "problem" },
55
- ENGINE_CRASHED => { http: 503, retryable: true, channel: "problem" },
56
- RENDER_TIMEOUT => { http: 504, retryable: true, channel: "problem" },
57
- ENGINE_TIMEOUT => { http: 504, retryable: true, channel: "problem" },
58
- SERVICE_UNAVAILABLE => { http: 503, retryable: true, channel: "problem" },
59
- INTERNAL_ERROR => { http: 500, retryable: true, channel: "problem" },
60
- TARGET_HTTP_ERROR => { http: 200, retryable: false, channel: "result" },
61
- NAVIGATION_FAILED => { http: 200, retryable: false, channel: "result" },
62
- BLOCKED => { http: 200, retryable: true, channel: "result" },
63
- CAPTCHA_REQUIRED => { http: 200, retryable: true, channel: "result" },
64
- EMPTY_CONTENT => { http: 200, retryable: false, channel: "result" }
43
+ BAD_REQUEST => { http: 400, retryable: false, channel: "problem" },
44
+ UNAUTHORIZED => { http: 401, retryable: false, channel: "problem" },
45
+ FORBIDDEN => { http: 403, retryable: false, channel: "problem" },
46
+ PAYMENT_REQUIRED => { http: 402, retryable: false, channel: "problem" },
47
+ NOT_FOUND => { http: 404, retryable: false, channel: "problem" },
48
+ CONFLICT => { http: 409, retryable: false, channel: "problem" },
49
+ BYO_PROXY_INVALID => { http: 422, retryable: false, channel: "problem" },
50
+ TIER_UNAVAILABLE => { http: 400, retryable: false, channel: "problem" },
51
+ RATE_LIMITED => { http: 429, retryable: true, channel: "problem" },
52
+ QUOTA_BACKEND_UNAVAILABLE => { http: 503, retryable: true, channel: "problem" },
53
+ POOL_EXHAUSTED => { http: 503, retryable: true, channel: "problem" },
54
+ EGRESS_INTEGRITY_FAILED => { http: 503, retryable: true, channel: "problem" },
55
+ RENDER_HUNG => { http: 503, retryable: true, channel: "problem" },
56
+ ENGINE_CRASHED => { http: 503, retryable: true, channel: "problem" },
57
+ RENDER_TIMEOUT => { http: 504, retryable: true, channel: "problem" },
58
+ ENGINE_TIMEOUT => { http: 504, retryable: true, channel: "problem" },
59
+ SERVICE_UNAVAILABLE => { http: 503, retryable: true, channel: "problem" },
60
+ INTERNAL_ERROR => { http: 500, retryable: true, channel: "problem" },
61
+ TARGET_HTTP_ERROR => { http: 200, retryable: false, channel: "result" },
62
+ NAVIGATION_FAILED => { http: 200, retryable: false, channel: "result" },
63
+ BLOCKED => { http: 200, retryable: true, channel: "result" },
64
+ CAPTCHA_REQUIRED => { http: 200, retryable: true, channel: "result" },
65
+ EMPTY_CONTENT => { http: 200, retryable: false, channel: "result" }
65
66
  }.freeze
66
67
 
67
68
  # code => retryable? boolean lookup, derived from {CATALOG}.
@@ -99,7 +100,7 @@ module GhostCrawl
99
100
  # code:"byo_proxy_invalid" (which raise_translated reads first). Using the
100
101
  # BYO-specific code as the blanket 422 fallback mislabels every unrelated
101
102
  # validation 422. Fall back to the generic bad_request instead.
102
- when 422 then BAD_REQUEST
103
+ when 422 then BAD_REQUEST # rubocop:disable Lint/DuplicateBranch -- 400 and 422 intentionally share bad_request
103
104
  when 429 then RATE_LIMITED
104
105
  when 503 then SERVICE_UNAVAILABLE
105
106
  when 504 then ENGINE_TIMEOUT
@@ -75,6 +75,18 @@ module GhostCrawl
75
75
  status = validation ? 422 : extract_status(err)
76
76
  body = err.message
77
77
 
78
+ # A genuine transport failure (DNS, connection refused/reset, timeout, TLS)
79
+ # never reaches an HTTP status line, so +extract_status+ returns 0. Map it to
80
+ # a retryable {APIError} instead of letting the status -> code fallback below
81
+ # mislabel it as a non-retryable 400-class +bad_request+.
82
+ if status.zero? && transport_error?(err)
83
+ raise APIError.new(
84
+ "could not reach GhostCrawl — #{err.message} (transport failure, retryable)",
85
+ status_code: 0, body: body,
86
+ code: ErrorCodes::SERVICE_UNAVAILABLE, retryable: true
87
+ )
88
+ end
89
+
78
90
  # The problem+json body is frequently dropped by the Kiota runtime before it
79
91
  # raises (no spec error-mapping for the status). Try to read a canonical
80
92
  # +code+ and +instance+ (request id) from whatever message/body text exists;
@@ -114,10 +126,31 @@ module GhostCrawl
114
126
  {}
115
127
  end
116
128
 
129
+ # Whether +err+ is a low-level transport failure (no HTTP response was ever
130
+ # received) rather than a server status response. Matched by class name so the
131
+ # SDK does not hard-depend on Faraday / Net exception constants being loaded.
132
+ # @api private
133
+ def self.transport_error?(err)
134
+ name = err.class.name.to_s
135
+ return true if name.start_with?("Errno::")
136
+
137
+ [
138
+ "Faraday::ConnectionFailed", "Faraday::TimeoutError", "Faraday::SSLError",
139
+ "Net::OpenTimeout", "Net::ReadTimeout", "SocketError", "Timeout::Error"
140
+ ].include?(name)
141
+ end
142
+
117
143
  # @api private
118
144
  def self.extract_status(err)
119
- # The Faraday adapter appends ":<code>" to its ApiError message.
120
- m = err.message.to_s.match(/:(\d{3})\b/)
145
+ # The Faraday adapter appends the status to its ApiError message. Two shapes
146
+ # occur: the hand-synthesized message uses "status code:429" (no space) while
147
+ # the modeled Kiota builder path (e.g. scrape) surfaces Kiota's NATIVE message
148
+ # "...unexpected status code: 402" (WITH a space). Anchor on the "status code"
149
+ # phrase first (tolerating optional whitespace), then fall back to a bare
150
+ # colon-number — so both paths resolve the status instead of dropping the
151
+ # modeled path to a bare GhostCrawlError with status 0 (190-08 fix).
152
+ msg = err.message.to_s
153
+ m = msg.match(/status code:?\s*(\d{3})\b/i) || msg.match(/:\s*(\d{3})\b/)
121
154
  m ? m[1].to_i : 0
122
155
  end
123
156
 
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module GhostCrawl
4
- VERSION = "2.3.5"
4
+ VERSION = "2.3.7"
5
5
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: ghostcrawl
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.3.5
4
+ version: 2.3.7
5
5
  platform: ruby
6
6
  authors:
7
7
  - GhostCrawl
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2026-07-27 00:00:00.000000000 Z
11
+ date: 2026-11-16 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: microsoft_kiota_abstractions
@@ -70,6 +70,10 @@ files:
70
70
  - _generated/models/agent_request_task_steps.rb
71
71
  - _generated/models/agent_request_task_steps_schema.rb
72
72
  - _generated/models/agent_request_task_steps_type.rb
73
+ - _generated/models/arrival_body.rb
74
+ - _generated/models/arrival_body_profile_id_member1.rb
75
+ - _generated/models/arrival_body_referrer_shape.rb
76
+ - _generated/models/arrival_body_session_id_member1.rb
73
77
  - _generated/models/auth_context.rb
74
78
  - _generated/models/batch_scrape_request.rb
75
79
  - _generated/models/batch_scrape_request_behavior_actions_member1.rb
@@ -82,6 +86,9 @@ files:
82
86
  - _generated/models/batch_scrape_request_profile_member1.rb
83
87
  - _generated/models/cdp_frame_request.rb
84
88
  - _generated/models/cdp_url_request.rb
89
+ - _generated/models/close_popups_body.rb
90
+ - _generated/models/close_popups_body_keep_host_member1.rb
91
+ - _generated/models/close_popups_body_profile_id_member1.rb
85
92
  - _generated/models/content_request.rb
86
93
  - _generated/models/content_request_engine.rb
87
94
  - _generated/models/content_request_format.rb
@@ -132,6 +139,10 @@ files:
132
139
  - _generated/models/map_response.rb
133
140
  - _generated/models/map_response_truncated_to_server_max_member1.rb
134
141
  - _generated/models/models.rb
142
+ - _generated/models/mouse_body.rb
143
+ - _generated/models/mouse_body_ops.rb
144
+ - _generated/models/mouse_body_profile_id_member1.rb
145
+ - _generated/models/mouse_body_session_id_member1.rb
135
146
  - _generated/models/pdf_request.rb
136
147
  - _generated/models/pdf_request_engine.rb
137
148
  - _generated/models/pdf_request_identity_country_member1.rb
@@ -199,7 +210,14 @@ files:
199
210
  - _generated/models/session_create_request_profile_member1.rb
200
211
  - _generated/models/storage_state_attach_request.rb
201
212
  - _generated/models/storage_state_detach_request.rb
213
+ - _generated/models/tab_visit_body.rb
214
+ - _generated/models/tab_visit_body_profile_id_member1.rb
215
+ - _generated/models/tab_visit_body_session_id_member1.rb
202
216
  - _generated/models/takeover_action_body.rb
217
+ - _generated/models/touch_body.rb
218
+ - _generated/models/touch_body_ops.rb
219
+ - _generated/models/touch_body_profile_id_member1.rb
220
+ - _generated/models/touch_body_session_id_member1.rb
203
221
  - _generated/models/upload_body.rb
204
222
  - _generated/models/wait_body.rb
205
223
  - _generated/models/wait_body_selector_member1.rb
@@ -268,6 +286,10 @@ files:
268
286
  - _generated/v1/kv/kv_request_builder.rb
269
287
  - _generated/v1/map/map.rb
270
288
  - _generated/v1/map/map_request_builder.rb
289
+ - _generated/v1/page/arrival/arrival.rb
290
+ - _generated/v1/page/arrival/arrival_request_builder.rb
291
+ - _generated/v1/page/close_popups/close_popups.rb
292
+ - _generated/v1/page/close_popups/close_popups_request_builder.rb
271
293
  - _generated/v1/page/cookies/cookies.rb
272
294
  - _generated/v1/page/cookies/cookies_request_builder.rb
273
295
  - _generated/v1/page/dom_snapshot/dom_snapshot.rb
@@ -278,10 +300,16 @@ files:
278
300
  - _generated/v1/page/eval/eval_request_builder.rb
279
301
  - _generated/v1/page/har/har.rb
280
302
  - _generated/v1/page/har/har_request_builder.rb
303
+ - _generated/v1/page/mouse/mouse.rb
304
+ - _generated/v1/page/mouse/mouse_request_builder.rb
281
305
  - _generated/v1/page/page.rb
282
306
  - _generated/v1/page/page_request_builder.rb
283
307
  - _generated/v1/page/scroll/scroll.rb
284
308
  - _generated/v1/page/scroll/scroll_request_builder.rb
309
+ - _generated/v1/page/tab_visit/tab_visit.rb
310
+ - _generated/v1/page/tab_visit/tab_visit_request_builder.rb
311
+ - _generated/v1/page/touch/touch.rb
312
+ - _generated/v1/page/touch/touch_request_builder.rb
285
313
  - _generated/v1/page/upload/upload.rb
286
314
  - _generated/v1/page/upload/upload_request_builder.rb
287
315
  - _generated/v1/page/wait/wait.rb