ghostcrawl 2.3.5 → 2.3.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +59 -12
- data/_generated/models/arrival_body.rb +325 -0
- data/_generated/models/arrival_body_profile_id_member1.rb +62 -0
- data/_generated/models/arrival_body_referrer_shape.rb +8 -0
- data/_generated/models/arrival_body_session_id_member1.rb +62 -0
- data/_generated/models/close_popups_body.rb +264 -0
- data/_generated/models/close_popups_body_keep_host_member1.rb +62 -0
- data/_generated/models/close_popups_body_profile_id_member1.rb +62 -0
- data/_generated/models/models.rb +18 -0
- data/_generated/models/mouse_body.rb +264 -0
- data/_generated/models/mouse_body_ops.rb +62 -0
- data/_generated/models/mouse_body_profile_id_member1.rb +62 -0
- data/_generated/models/mouse_body_session_id_member1.rb +62 -0
- data/_generated/models/scrape_request.rb +6 -6
- data/_generated/models/scrape_request_engine.rb +1 -1
- data/_generated/models/session_create_request.rb +3 -3
- data/_generated/models/tab_visit_body.rb +285 -0
- data/_generated/models/tab_visit_body_profile_id_member1.rb +62 -0
- data/_generated/models/tab_visit_body_session_id_member1.rb +62 -0
- data/_generated/models/touch_body.rb +264 -0
- data/_generated/models/touch_body_ops.rb +62 -0
- data/_generated/models/touch_body_profile_id_member1.rb +62 -0
- data/_generated/models/touch_body_session_id_member1.rb +62 -0
- data/_generated/v1/cdp/frame/frame_request_builder.rb +2 -2
- data/_generated/v1/crawl_runs/item/with_run_item_request_builder.rb +3 -3
- data/_generated/v1/page/arrival/arrival.rb +0 -0
- data/_generated/v1/page/arrival/arrival_request_builder.rb +74 -0
- data/_generated/v1/page/close_popups/close_popups.rb +0 -0
- data/_generated/v1/page/close_popups/close_popups_request_builder.rb +74 -0
- data/_generated/v1/page/mouse/mouse.rb +0 -0
- data/_generated/v1/page/mouse/mouse_request_builder.rb +74 -0
- data/_generated/v1/page/page_request_builder.rb +30 -0
- data/_generated/v1/page/tab_visit/tab_visit.rb +0 -0
- data/_generated/v1/page/tab_visit/tab_visit_request_builder.rb +74 -0
- data/_generated/v1/page/touch/touch.rb +0 -0
- data/_generated/v1/page/touch/touch_request_builder.rb +74 -0
- data/_generated/v1/profiles/item/with_name_item_request_builder.rb +2 -2
- data/_generated/v1/schedules/item/runs/runs_request_builder.rb +2 -2
- data/_generated/v1/search/jobs/item/with_job_item_request_builder.rb +2 -2
- data/_generated/v1/sessions/item/takeover/takeover_request_builder.rb +2 -2
- data/_generated/v1/sessions/sessions_request_builder.rb +3 -3
- data/lib/ghostcrawl/client.rb +1001 -46
- data/lib/ghostcrawl/error_codes.rb +26 -25
- data/lib/ghostcrawl/errors.rb +35 -2
- data/lib/ghostcrawl/version.rb +1 -1
- metadata +30 -2
|
@@ -19,6 +19,7 @@ module GhostCrawl
|
|
|
19
19
|
CONFLICT = "conflict"
|
|
20
20
|
BYO_PROXY_INVALID = "byo_proxy_invalid"
|
|
21
21
|
TIER_UNAVAILABLE = "tier_unavailable"
|
|
22
|
+
ROUTING_MODE_INCOHERENT_CLAIM_OS = "routing_mode_incoherent_claim_os"
|
|
22
23
|
RATE_LIMITED = "rate_limited"
|
|
23
24
|
QUOTA_BACKEND_UNAVAILABLE = "quota_backend_unavailable"
|
|
24
25
|
POOL_EXHAUSTED = "pool_exhausted"
|
|
@@ -26,7 +27,7 @@ module GhostCrawl
|
|
|
26
27
|
RENDER_HUNG = "render_hung"
|
|
27
28
|
ENGINE_CRASHED = "engine_crashed"
|
|
28
29
|
RENDER_TIMEOUT = "render_timeout"
|
|
29
|
-
ENGINE_TIMEOUT
|
|
30
|
+
ENGINE_TIMEOUT = "engine_timeout"
|
|
30
31
|
SERVICE_UNAVAILABLE = "service_unavailable"
|
|
31
32
|
INTERNAL_ERROR = "internal_error"
|
|
32
33
|
|
|
@@ -39,29 +40,29 @@ module GhostCrawl
|
|
|
39
40
|
|
|
40
41
|
# Full catalog: code => {http:, retryable:, channel:}. Mirrors codes.json.
|
|
41
42
|
CATALOG = {
|
|
42
|
-
BAD_REQUEST
|
|
43
|
-
UNAUTHORIZED
|
|
44
|
-
FORBIDDEN
|
|
45
|
-
PAYMENT_REQUIRED
|
|
46
|
-
NOT_FOUND
|
|
47
|
-
CONFLICT
|
|
48
|
-
BYO_PROXY_INVALID
|
|
49
|
-
TIER_UNAVAILABLE
|
|
50
|
-
RATE_LIMITED
|
|
51
|
-
QUOTA_BACKEND_UNAVAILABLE => { http: 503, retryable: true,
|
|
52
|
-
POOL_EXHAUSTED
|
|
53
|
-
EGRESS_INTEGRITY_FAILED
|
|
54
|
-
RENDER_HUNG
|
|
55
|
-
ENGINE_CRASHED
|
|
56
|
-
RENDER_TIMEOUT
|
|
57
|
-
ENGINE_TIMEOUT
|
|
58
|
-
SERVICE_UNAVAILABLE
|
|
59
|
-
INTERNAL_ERROR
|
|
60
|
-
TARGET_HTTP_ERROR
|
|
61
|
-
NAVIGATION_FAILED
|
|
62
|
-
BLOCKED
|
|
63
|
-
CAPTCHA_REQUIRED
|
|
64
|
-
EMPTY_CONTENT
|
|
43
|
+
BAD_REQUEST => { http: 400, retryable: false, channel: "problem" },
|
|
44
|
+
UNAUTHORIZED => { http: 401, retryable: false, channel: "problem" },
|
|
45
|
+
FORBIDDEN => { http: 403, retryable: false, channel: "problem" },
|
|
46
|
+
PAYMENT_REQUIRED => { http: 402, retryable: false, channel: "problem" },
|
|
47
|
+
NOT_FOUND => { http: 404, retryable: false, channel: "problem" },
|
|
48
|
+
CONFLICT => { http: 409, retryable: false, channel: "problem" },
|
|
49
|
+
BYO_PROXY_INVALID => { http: 422, retryable: false, channel: "problem" },
|
|
50
|
+
TIER_UNAVAILABLE => { http: 400, retryable: false, channel: "problem" },
|
|
51
|
+
RATE_LIMITED => { http: 429, retryable: true, channel: "problem" },
|
|
52
|
+
QUOTA_BACKEND_UNAVAILABLE => { http: 503, retryable: true, channel: "problem" },
|
|
53
|
+
POOL_EXHAUSTED => { http: 503, retryable: true, channel: "problem" },
|
|
54
|
+
EGRESS_INTEGRITY_FAILED => { http: 503, retryable: true, channel: "problem" },
|
|
55
|
+
RENDER_HUNG => { http: 503, retryable: true, channel: "problem" },
|
|
56
|
+
ENGINE_CRASHED => { http: 503, retryable: true, channel: "problem" },
|
|
57
|
+
RENDER_TIMEOUT => { http: 504, retryable: true, channel: "problem" },
|
|
58
|
+
ENGINE_TIMEOUT => { http: 504, retryable: true, channel: "problem" },
|
|
59
|
+
SERVICE_UNAVAILABLE => { http: 503, retryable: true, channel: "problem" },
|
|
60
|
+
INTERNAL_ERROR => { http: 500, retryable: true, channel: "problem" },
|
|
61
|
+
TARGET_HTTP_ERROR => { http: 200, retryable: false, channel: "result" },
|
|
62
|
+
NAVIGATION_FAILED => { http: 200, retryable: false, channel: "result" },
|
|
63
|
+
BLOCKED => { http: 200, retryable: true, channel: "result" },
|
|
64
|
+
CAPTCHA_REQUIRED => { http: 200, retryable: true, channel: "result" },
|
|
65
|
+
EMPTY_CONTENT => { http: 200, retryable: false, channel: "result" }
|
|
65
66
|
}.freeze
|
|
66
67
|
|
|
67
68
|
# code => retryable? boolean lookup, derived from {CATALOG}.
|
|
@@ -99,7 +100,7 @@ module GhostCrawl
|
|
|
99
100
|
# code:"byo_proxy_invalid" (which raise_translated reads first). Using the
|
|
100
101
|
# BYO-specific code as the blanket 422 fallback mislabels every unrelated
|
|
101
102
|
# validation 422. Fall back to the generic bad_request instead.
|
|
102
|
-
when 422 then BAD_REQUEST
|
|
103
|
+
when 422 then BAD_REQUEST # rubocop:disable Lint/DuplicateBranch -- 400 and 422 intentionally share bad_request
|
|
103
104
|
when 429 then RATE_LIMITED
|
|
104
105
|
when 503 then SERVICE_UNAVAILABLE
|
|
105
106
|
when 504 then ENGINE_TIMEOUT
|
data/lib/ghostcrawl/errors.rb
CHANGED
|
@@ -75,6 +75,18 @@ module GhostCrawl
|
|
|
75
75
|
status = validation ? 422 : extract_status(err)
|
|
76
76
|
body = err.message
|
|
77
77
|
|
|
78
|
+
# A genuine transport failure (DNS, connection refused/reset, timeout, TLS)
|
|
79
|
+
# never reaches an HTTP status line, so +extract_status+ returns 0. Map it to
|
|
80
|
+
# a retryable {APIError} instead of letting the status -> code fallback below
|
|
81
|
+
# mislabel it as a non-retryable 400-class +bad_request+.
|
|
82
|
+
if status.zero? && transport_error?(err)
|
|
83
|
+
raise APIError.new(
|
|
84
|
+
"could not reach GhostCrawl — #{err.message} (transport failure, retryable)",
|
|
85
|
+
status_code: 0, body: body,
|
|
86
|
+
code: ErrorCodes::SERVICE_UNAVAILABLE, retryable: true
|
|
87
|
+
)
|
|
88
|
+
end
|
|
89
|
+
|
|
78
90
|
# The problem+json body is frequently dropped by the Kiota runtime before it
|
|
79
91
|
# raises (no spec error-mapping for the status). Try to read a canonical
|
|
80
92
|
# +code+ and +instance+ (request id) from whatever message/body text exists;
|
|
@@ -114,10 +126,31 @@ module GhostCrawl
|
|
|
114
126
|
{}
|
|
115
127
|
end
|
|
116
128
|
|
|
129
|
+
# Whether +err+ is a low-level transport failure (no HTTP response was ever
|
|
130
|
+
# received) rather than a server status response. Matched by class name so the
|
|
131
|
+
# SDK does not hard-depend on Faraday / Net exception constants being loaded.
|
|
132
|
+
# @api private
|
|
133
|
+
def self.transport_error?(err)
|
|
134
|
+
name = err.class.name.to_s
|
|
135
|
+
return true if name.start_with?("Errno::")
|
|
136
|
+
|
|
137
|
+
[
|
|
138
|
+
"Faraday::ConnectionFailed", "Faraday::TimeoutError", "Faraday::SSLError",
|
|
139
|
+
"Net::OpenTimeout", "Net::ReadTimeout", "SocketError", "Timeout::Error"
|
|
140
|
+
].include?(name)
|
|
141
|
+
end
|
|
142
|
+
|
|
117
143
|
# @api private
|
|
118
144
|
def self.extract_status(err)
|
|
119
|
-
# The Faraday adapter appends
|
|
120
|
-
|
|
145
|
+
# The Faraday adapter appends the status to its ApiError message. Two shapes
|
|
146
|
+
# occur: the hand-synthesized message uses "status code:429" (no space) while
|
|
147
|
+
# the modeled Kiota builder path (e.g. scrape) surfaces Kiota's NATIVE message
|
|
148
|
+
# "...unexpected status code: 402" (WITH a space). Anchor on the "status code"
|
|
149
|
+
# phrase first (tolerating optional whitespace), then fall back to a bare
|
|
150
|
+
# colon-number — so both paths resolve the status instead of dropping the
|
|
151
|
+
# modeled path to a bare GhostCrawlError with status 0 (190-08 fix).
|
|
152
|
+
msg = err.message.to_s
|
|
153
|
+
m = msg.match(/status code:?\s*(\d{3})\b/i) || msg.match(/:\s*(\d{3})\b/)
|
|
121
154
|
m ? m[1].to_i : 0
|
|
122
155
|
end
|
|
123
156
|
|
data/lib/ghostcrawl/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: ghostcrawl
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 2.3.
|
|
4
|
+
version: 2.3.7
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- GhostCrawl
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-
|
|
11
|
+
date: 2026-11-16 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: microsoft_kiota_abstractions
|
|
@@ -70,6 +70,10 @@ files:
|
|
|
70
70
|
- _generated/models/agent_request_task_steps.rb
|
|
71
71
|
- _generated/models/agent_request_task_steps_schema.rb
|
|
72
72
|
- _generated/models/agent_request_task_steps_type.rb
|
|
73
|
+
- _generated/models/arrival_body.rb
|
|
74
|
+
- _generated/models/arrival_body_profile_id_member1.rb
|
|
75
|
+
- _generated/models/arrival_body_referrer_shape.rb
|
|
76
|
+
- _generated/models/arrival_body_session_id_member1.rb
|
|
73
77
|
- _generated/models/auth_context.rb
|
|
74
78
|
- _generated/models/batch_scrape_request.rb
|
|
75
79
|
- _generated/models/batch_scrape_request_behavior_actions_member1.rb
|
|
@@ -82,6 +86,9 @@ files:
|
|
|
82
86
|
- _generated/models/batch_scrape_request_profile_member1.rb
|
|
83
87
|
- _generated/models/cdp_frame_request.rb
|
|
84
88
|
- _generated/models/cdp_url_request.rb
|
|
89
|
+
- _generated/models/close_popups_body.rb
|
|
90
|
+
- _generated/models/close_popups_body_keep_host_member1.rb
|
|
91
|
+
- _generated/models/close_popups_body_profile_id_member1.rb
|
|
85
92
|
- _generated/models/content_request.rb
|
|
86
93
|
- _generated/models/content_request_engine.rb
|
|
87
94
|
- _generated/models/content_request_format.rb
|
|
@@ -132,6 +139,10 @@ files:
|
|
|
132
139
|
- _generated/models/map_response.rb
|
|
133
140
|
- _generated/models/map_response_truncated_to_server_max_member1.rb
|
|
134
141
|
- _generated/models/models.rb
|
|
142
|
+
- _generated/models/mouse_body.rb
|
|
143
|
+
- _generated/models/mouse_body_ops.rb
|
|
144
|
+
- _generated/models/mouse_body_profile_id_member1.rb
|
|
145
|
+
- _generated/models/mouse_body_session_id_member1.rb
|
|
135
146
|
- _generated/models/pdf_request.rb
|
|
136
147
|
- _generated/models/pdf_request_engine.rb
|
|
137
148
|
- _generated/models/pdf_request_identity_country_member1.rb
|
|
@@ -199,7 +210,14 @@ files:
|
|
|
199
210
|
- _generated/models/session_create_request_profile_member1.rb
|
|
200
211
|
- _generated/models/storage_state_attach_request.rb
|
|
201
212
|
- _generated/models/storage_state_detach_request.rb
|
|
213
|
+
- _generated/models/tab_visit_body.rb
|
|
214
|
+
- _generated/models/tab_visit_body_profile_id_member1.rb
|
|
215
|
+
- _generated/models/tab_visit_body_session_id_member1.rb
|
|
202
216
|
- _generated/models/takeover_action_body.rb
|
|
217
|
+
- _generated/models/touch_body.rb
|
|
218
|
+
- _generated/models/touch_body_ops.rb
|
|
219
|
+
- _generated/models/touch_body_profile_id_member1.rb
|
|
220
|
+
- _generated/models/touch_body_session_id_member1.rb
|
|
203
221
|
- _generated/models/upload_body.rb
|
|
204
222
|
- _generated/models/wait_body.rb
|
|
205
223
|
- _generated/models/wait_body_selector_member1.rb
|
|
@@ -268,6 +286,10 @@ files:
|
|
|
268
286
|
- _generated/v1/kv/kv_request_builder.rb
|
|
269
287
|
- _generated/v1/map/map.rb
|
|
270
288
|
- _generated/v1/map/map_request_builder.rb
|
|
289
|
+
- _generated/v1/page/arrival/arrival.rb
|
|
290
|
+
- _generated/v1/page/arrival/arrival_request_builder.rb
|
|
291
|
+
- _generated/v1/page/close_popups/close_popups.rb
|
|
292
|
+
- _generated/v1/page/close_popups/close_popups_request_builder.rb
|
|
271
293
|
- _generated/v1/page/cookies/cookies.rb
|
|
272
294
|
- _generated/v1/page/cookies/cookies_request_builder.rb
|
|
273
295
|
- _generated/v1/page/dom_snapshot/dom_snapshot.rb
|
|
@@ -278,10 +300,16 @@ files:
|
|
|
278
300
|
- _generated/v1/page/eval/eval_request_builder.rb
|
|
279
301
|
- _generated/v1/page/har/har.rb
|
|
280
302
|
- _generated/v1/page/har/har_request_builder.rb
|
|
303
|
+
- _generated/v1/page/mouse/mouse.rb
|
|
304
|
+
- _generated/v1/page/mouse/mouse_request_builder.rb
|
|
281
305
|
- _generated/v1/page/page.rb
|
|
282
306
|
- _generated/v1/page/page_request_builder.rb
|
|
283
307
|
- _generated/v1/page/scroll/scroll.rb
|
|
284
308
|
- _generated/v1/page/scroll/scroll_request_builder.rb
|
|
309
|
+
- _generated/v1/page/tab_visit/tab_visit.rb
|
|
310
|
+
- _generated/v1/page/tab_visit/tab_visit_request_builder.rb
|
|
311
|
+
- _generated/v1/page/touch/touch.rb
|
|
312
|
+
- _generated/v1/page/touch/touch_request_builder.rb
|
|
285
313
|
- _generated/v1/page/upload/upload.rb
|
|
286
314
|
- _generated/v1/page/upload/upload_request_builder.rb
|
|
287
315
|
- _generated/v1/page/wait/wait.rb
|