ghostcrawl 2.3.5 → 2.3.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +59 -12
- data/_generated/models/arrival_body.rb +325 -0
- data/_generated/models/arrival_body_profile_id_member1.rb +62 -0
- data/_generated/models/arrival_body_referrer_shape.rb +8 -0
- data/_generated/models/arrival_body_session_id_member1.rb +62 -0
- data/_generated/models/close_popups_body.rb +264 -0
- data/_generated/models/close_popups_body_keep_host_member1.rb +62 -0
- data/_generated/models/close_popups_body_profile_id_member1.rb +62 -0
- data/_generated/models/models.rb +18 -0
- data/_generated/models/mouse_body.rb +264 -0
- data/_generated/models/mouse_body_ops.rb +62 -0
- data/_generated/models/mouse_body_profile_id_member1.rb +62 -0
- data/_generated/models/mouse_body_session_id_member1.rb +62 -0
- data/_generated/models/scrape_request.rb +6 -6
- data/_generated/models/scrape_request_engine.rb +1 -1
- data/_generated/models/session_create_request.rb +3 -3
- data/_generated/models/tab_visit_body.rb +285 -0
- data/_generated/models/tab_visit_body_profile_id_member1.rb +62 -0
- data/_generated/models/tab_visit_body_session_id_member1.rb +62 -0
- data/_generated/models/touch_body.rb +264 -0
- data/_generated/models/touch_body_ops.rb +62 -0
- data/_generated/models/touch_body_profile_id_member1.rb +62 -0
- data/_generated/models/touch_body_session_id_member1.rb +62 -0
- data/_generated/v1/cdp/frame/frame_request_builder.rb +2 -2
- data/_generated/v1/crawl_runs/item/with_run_item_request_builder.rb +3 -3
- data/_generated/v1/page/arrival/arrival.rb +0 -0
- data/_generated/v1/page/arrival/arrival_request_builder.rb +74 -0
- data/_generated/v1/page/close_popups/close_popups.rb +0 -0
- data/_generated/v1/page/close_popups/close_popups_request_builder.rb +74 -0
- data/_generated/v1/page/mouse/mouse.rb +0 -0
- data/_generated/v1/page/mouse/mouse_request_builder.rb +74 -0
- data/_generated/v1/page/page_request_builder.rb +30 -0
- data/_generated/v1/page/tab_visit/tab_visit.rb +0 -0
- data/_generated/v1/page/tab_visit/tab_visit_request_builder.rb +74 -0
- data/_generated/v1/page/touch/touch.rb +0 -0
- data/_generated/v1/page/touch/touch_request_builder.rb +74 -0
- data/_generated/v1/profiles/item/with_name_item_request_builder.rb +2 -2
- data/_generated/v1/schedules/item/runs/runs_request_builder.rb +2 -2
- data/_generated/v1/search/jobs/item/with_job_item_request_builder.rb +2 -2
- data/_generated/v1/sessions/item/takeover/takeover_request_builder.rb +2 -2
- data/_generated/v1/sessions/sessions_request_builder.rb +3 -3
- data/lib/ghostcrawl/client.rb +1001 -46
- data/lib/ghostcrawl/error_codes.rb +26 -25
- data/lib/ghostcrawl/errors.rb +35 -2
- data/lib/ghostcrawl/version.rb +1 -1
- metadata +30 -2
data/lib/ghostcrawl/client.rb
CHANGED
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
# result = client.scrape(url: "https://example.com")
|
|
18
18
|
|
|
19
19
|
require "json"
|
|
20
|
+
require "uri"
|
|
20
21
|
require "microsoft_kiota_abstractions"
|
|
21
22
|
require "microsoft_kiota_faraday"
|
|
22
23
|
require "microsoft_kiota_serialization_json"
|
|
@@ -51,6 +52,7 @@ module GhostCrawl
|
|
|
51
52
|
module KiotaWriterFix
|
|
52
53
|
def write_object_value(key, value)
|
|
53
54
|
return unless value
|
|
55
|
+
|
|
54
56
|
if key.nil?
|
|
55
57
|
# Fix: serialize into self, not a temp. Merges all fields into @writer.
|
|
56
58
|
value.serialize(self)
|
|
@@ -76,6 +78,7 @@ module GhostCrawl
|
|
|
76
78
|
# broken `.writer` deref.
|
|
77
79
|
def write_collection_of_object_values(key, values)
|
|
78
80
|
return unless values
|
|
81
|
+
|
|
79
82
|
hashes = values.map do |v|
|
|
80
83
|
temp = MicrosoftKiotaSerializationJson::JsonSerializationWriter.new
|
|
81
84
|
v.serialize(temp)
|
|
@@ -108,6 +111,7 @@ module GhostCrawl
|
|
|
108
111
|
def write_any_value(key, value)
|
|
109
112
|
if value.is_a?(Hash) || value.is_a?(Array)
|
|
110
113
|
return super unless key # nil-key arrays keep upstream behavior
|
|
114
|
+
|
|
111
115
|
writer[key] = KiotaWriterFix.jsonable(value)
|
|
112
116
|
return
|
|
113
117
|
end
|
|
@@ -215,6 +219,41 @@ end
|
|
|
215
219
|
module GhostCrawl
|
|
216
220
|
DEFAULT_BASE_URL = "https://api.ghostcrawl.io"
|
|
217
221
|
|
|
222
|
+
# ---------------------------------------------------------------------------
|
|
223
|
+
# Body-capturing Faraday adapter (190-08 fix)
|
|
224
|
+
# ---------------------------------------------------------------------------
|
|
225
|
+
#
|
|
226
|
+
# The pinned +microsoft_kiota_faraday+ runtime's +throw_if_failed_reponse+ has a
|
|
227
|
+
# defect on a non-2xx WITHOUT a matching spec error factory: when a generated
|
|
228
|
+
# builder registers a "4XX" factory (e.g. scrape/crawl), the runtime does
|
|
229
|
+
# +raise error+ where +error+ is the DESERIALIZED body object — NOT an Exception
|
|
230
|
+
# — producing a bare +TypeError: exception class/object expected+ that carries
|
|
231
|
+
# neither the HTTP status nor the problem+json body. A caller then sees a base
|
|
232
|
+
# +GhostCrawlError+ with +status_code == 0+ for a 402 (etc.) instead of the
|
|
233
|
+
# documented typed +PaymentRequiredError+.
|
|
234
|
+
#
|
|
235
|
+
# This subclass replaces that broken path with a single-execution translation:
|
|
236
|
+
# for ANY >= 400 it raises a +ApiError+ whose message carries the status token
|
|
237
|
+
# {GhostCrawl.raise_translated} keys off, plus the response body — so every
|
|
238
|
+
# modeled builder path (scrape/extract/crawl) surfaces the correct typed error
|
|
239
|
+
# with status + body. It NEVER re-issues the request (no double-execute).
|
|
240
|
+
# A < 400 response is left untouched (normal deserialization).
|
|
241
|
+
#
|
|
242
|
+
# @api private
|
|
243
|
+
class BodyCapturingFaradayAdapter < MicrosoftKiotaFaraday::FaradayRequestAdapter
|
|
244
|
+
def throw_if_failed_reponse(response, _errors_mapping)
|
|
245
|
+
raise StandardError, "response cannot be null" unless response
|
|
246
|
+
|
|
247
|
+
status_code = response.status
|
|
248
|
+
return if status_code < 400
|
|
249
|
+
|
|
250
|
+
body = response.respond_to?(:body) ? response.body.to_s : ""
|
|
251
|
+
message = "The server returned an unexpected status code:#{status_code}"
|
|
252
|
+
message += " #{body}" unless body.empty?
|
|
253
|
+
raise MicrosoftKiotaAbstractions::ApiError, message
|
|
254
|
+
end
|
|
255
|
+
end
|
|
256
|
+
|
|
218
257
|
# ---------------------------------------------------------------------------
|
|
219
258
|
# Static bearer token provider — implements AccessTokenProvider
|
|
220
259
|
# ---------------------------------------------------------------------------
|
|
@@ -305,7 +344,11 @@ module GhostCrawl
|
|
|
305
344
|
elsif value.respond_to?(:to_h)
|
|
306
345
|
value.to_h.transform_values { |v| to_hash(v) }
|
|
307
346
|
elsif value.respond_to?(:additional_data) && value.additional_data
|
|
347
|
+
# rubocop:disable Lint/DuplicateBranch -- intentional final fallback: an
|
|
348
|
+
# object exposing (possibly empty) additional_data but no #to_h still
|
|
349
|
+
# normalizes through the same transform as the non-empty branch above.
|
|
308
350
|
value.additional_data.transform_values { |v| to_hash(v) }
|
|
351
|
+
# rubocop:enable Lint/DuplicateBranch
|
|
309
352
|
else
|
|
310
353
|
value
|
|
311
354
|
end
|
|
@@ -330,6 +373,30 @@ module GhostCrawl
|
|
|
330
373
|
nil
|
|
331
374
|
end
|
|
332
375
|
|
|
376
|
+
# Parse a JSON response body, raising a typed {GhostCrawl::APIError} — never a
|
|
377
|
+
# raw +JSON::ParserError+ — when the server returns a body that is not valid
|
|
378
|
+
# JSON. This keeps every facade decode (cdp / page / downloads / storage-states
|
|
379
|
+
# / sessions / webhooks) inside the documented
|
|
380
|
+
# +rescue GhostCrawl::GhostCrawlError+ contract. A blank/empty body returns
|
|
381
|
+
# +nil+ (callers coerce that to +{}+), matching the previous behavior.
|
|
382
|
+
#
|
|
383
|
+
# @param bytes [String, nil] the raw response body
|
|
384
|
+
# @return [Object, nil] the parsed JSON value, or +nil+ for an empty body
|
|
385
|
+
# @raise [GhostCrawl::APIError] on a non-empty, non-JSON body
|
|
386
|
+
# @api private
|
|
387
|
+
def self.parse_json!(bytes)
|
|
388
|
+
text = bytes.to_s
|
|
389
|
+
return nil if text.strip.empty?
|
|
390
|
+
|
|
391
|
+
JSON.parse(text)
|
|
392
|
+
rescue JSON::ParserError => e
|
|
393
|
+
raise GhostCrawl::APIError.new(
|
|
394
|
+
"GhostCrawl returned a malformed (non-JSON) response body: #{e.message}",
|
|
395
|
+
status_code: 0, body: text,
|
|
396
|
+
code: GhostCrawl::ErrorCodes::INTERNAL_ERROR, retryable: false
|
|
397
|
+
)
|
|
398
|
+
end
|
|
399
|
+
|
|
333
400
|
# Executes a request that returns NO response body (an HTTP 204, or any DELETE
|
|
334
401
|
# whose spec maps the success response to +void+) and returns a plain Hash.
|
|
335
402
|
#
|
|
@@ -379,7 +446,7 @@ module GhostCrawl
|
|
|
379
446
|
body = response.body.to_s
|
|
380
447
|
err = MicrosoftKiotaAbstractions::ApiError.new(
|
|
381
448
|
"The server returned an unexpected status code:#{status}" \
|
|
382
|
-
"#{
|
|
449
|
+
"#{" #{body}" unless body.empty?}"
|
|
383
450
|
)
|
|
384
451
|
GhostCrawl.raise_translated(err)
|
|
385
452
|
end
|
|
@@ -387,7 +454,11 @@ module GhostCrawl
|
|
|
387
454
|
body = response.body.to_s
|
|
388
455
|
return {} if body.strip.empty?
|
|
389
456
|
|
|
390
|
-
parsed =
|
|
457
|
+
parsed = begin
|
|
458
|
+
JSON.parse(body)
|
|
459
|
+
rescue StandardError
|
|
460
|
+
nil
|
|
461
|
+
end
|
|
391
462
|
parsed.is_a?(Hash) ? parsed.transform_values { |v| to_hash(v) } : {}
|
|
392
463
|
rescue GhostCrawl::GhostCrawlError
|
|
393
464
|
raise
|
|
@@ -424,7 +495,7 @@ module GhostCrawl
|
|
|
424
495
|
body = response.body.to_s
|
|
425
496
|
err = MicrosoftKiotaAbstractions::ApiError.new(
|
|
426
497
|
"The server returned an unexpected status code:#{status}" \
|
|
427
|
-
"#{
|
|
498
|
+
"#{" #{body}" unless body.empty?}"
|
|
428
499
|
)
|
|
429
500
|
GhostCrawl.raise_translated(err)
|
|
430
501
|
end
|
|
@@ -436,6 +507,87 @@ module GhostCrawl
|
|
|
436
507
|
GhostCrawl.raise_translated(e)
|
|
437
508
|
end
|
|
438
509
|
|
|
510
|
+
# Routes an ad-hoc JSON request through the SAME authenticated adapter pipeline
|
|
511
|
+
# (base URL + bearer auth + branded UA + transport) as the modeled calls, for
|
|
512
|
+
# first-class facade methods whose generated void/empty-body POST/DELETE builders
|
|
513
|
+
# trip the two documented Kiota Ruby runtime defects (see {void_request!}). This
|
|
514
|
+
# is the shared "RequestAdapter raw-JSON path" the FACADE-PATTERN sanctions for
|
|
515
|
+
# routes without a clean generated builder — it NEVER opens a hand-rolled HTTP
|
|
516
|
+
# client and NEVER hand-rolls auth; it reuses the adapter's own transport.
|
|
517
|
+
#
|
|
518
|
+
# A 2xx with a JSON-object body returns the decoded Hash (recursively normalized);
|
|
519
|
+
# an empty 2xx returns +{}+; a >= 400 is translated into the typed {GhostCrawlError}.
|
|
520
|
+
#
|
|
521
|
+
# @param adapter [MicrosoftKiotaFaraday::FaradayRequestAdapter]
|
|
522
|
+
# @param method [Symbol] +:GET+ / +:POST+ / +:DELETE+ / +:PUT+
|
|
523
|
+
# @param path [String] absolute API path (e.g. +/v1/cdp/frame+)
|
|
524
|
+
# @param body [Object, nil] request body (serialized to JSON for non-GET)
|
|
525
|
+
# @return [Hash] the decoded response ({} on an empty body)
|
|
526
|
+
# @api private
|
|
527
|
+
def self.json_request!(adapter, method, path, body = nil)
|
|
528
|
+
request_info = MicrosoftKiotaAbstractions::RequestInformation.new
|
|
529
|
+
request_info.http_method = method
|
|
530
|
+
request_info.url_template = "{+baseurl}#{path}"
|
|
531
|
+
request_info.headers.try_add("Accept", "application/json")
|
|
532
|
+
request_info.set_stream_content(JSON.generate(body), "application/json") if !body.nil? && method != :GET
|
|
533
|
+
bytes = binary_request!(adapter, request_info)
|
|
534
|
+
parsed = parse_json!(bytes)
|
|
535
|
+
case parsed
|
|
536
|
+
when Hash then parsed.transform_values { |v| to_hash(v) }
|
|
537
|
+
when Array then { "items" => parsed.map { |v| to_hash(v) } }
|
|
538
|
+
else {}
|
|
539
|
+
end
|
|
540
|
+
end
|
|
541
|
+
|
|
542
|
+
# Executes a request through the SAME adapter pipeline (base URL + bearer auth
|
|
543
|
+
# + transport) as the modeled calls and returns the RAW Faraday response —
|
|
544
|
+
# WITHOUT raising on a >= 400 status. The caller inspects +response.status+ /
|
|
545
|
+
# +response.headers+ / +response.body+ itself.
|
|
546
|
+
#
|
|
547
|
+
# This is the status-aware counterpart of {binary_request!}: the +search(wait:)+
|
|
548
|
+
# bounded poll must distinguish a warm +200+ (results ready) from a cold
|
|
549
|
+
# +202 running+ (a poll handle) and read the +Retry-After+ header — information
|
|
550
|
+
# {binary_request!} discards. A genuine transport failure (DNS, connection
|
|
551
|
+
# reset, …) is still translated into the typed {GhostCrawlError} hierarchy.
|
|
552
|
+
#
|
|
553
|
+
# @param adapter [MicrosoftKiotaFaraday::FaradayRequestAdapter]
|
|
554
|
+
# @param request_info [MicrosoftKiotaAbstractions::RequestInformation]
|
|
555
|
+
# @return [Faraday::Response] the raw response (status/headers/body intact)
|
|
556
|
+
# @api private
|
|
557
|
+
def self.raw_response!(adapter, request_info)
|
|
558
|
+
request_info.path_parameters["baseurl"] = adapter.get_base_url
|
|
559
|
+
auth_fiber = adapter.authentication_provider.authenticate_request(request_info)
|
|
560
|
+
auth_fiber.resume if auth_fiber.respond_to?(:resume)
|
|
561
|
+
request = adapter.get_request_from_request_info(request_info)
|
|
562
|
+
adapter.client.run_request(
|
|
563
|
+
request.http_method, request.path, request.body, request.headers
|
|
564
|
+
)
|
|
565
|
+
rescue GhostCrawl::GhostCrawlError
|
|
566
|
+
raise
|
|
567
|
+
rescue StandardError => e
|
|
568
|
+
GhostCrawl.raise_translated(e)
|
|
569
|
+
end
|
|
570
|
+
|
|
571
|
+
# Raises the documented typed {GhostCrawlError} when +response+ carries a
|
|
572
|
+
# >= 400 status, keying the translation off a synthesized +ApiError+ whose
|
|
573
|
+
# message carries the +":<status>"+ token {GhostCrawl.raise_translated} matches
|
|
574
|
+
# (plus any problem+json body text). A < 400 response is returned untouched.
|
|
575
|
+
#
|
|
576
|
+
# @param response [Faraday::Response]
|
|
577
|
+
# @return [Faraday::Response] the same response, when it is not a failure
|
|
578
|
+
# @api private
|
|
579
|
+
def self.raise_from_response!(response)
|
|
580
|
+
status = response.status
|
|
581
|
+
return response if status < 400
|
|
582
|
+
|
|
583
|
+
body = response.body.to_s
|
|
584
|
+
err = MicrosoftKiotaAbstractions::ApiError.new(
|
|
585
|
+
"The server returned an unexpected status code:#{status}" \
|
|
586
|
+
"#{" #{body}" unless body.empty?}"
|
|
587
|
+
)
|
|
588
|
+
GhostCrawl.raise_translated(err)
|
|
589
|
+
end
|
|
590
|
+
|
|
439
591
|
# Inspects a decoded HTTP-200 response hash for a RESULT-channel failure (the
|
|
440
592
|
# target page could not be scraped) and raises {GhostCrawl::ScrapeError} when
|
|
441
593
|
# one is present. This is the reliable, highest-value error path: the body is
|
|
@@ -485,7 +637,7 @@ module GhostCrawl
|
|
|
485
637
|
code ||= GhostCrawl::ErrorCodes::EMPTY_CONTENT
|
|
486
638
|
|
|
487
639
|
retryable =
|
|
488
|
-
if result_error
|
|
640
|
+
if result_error&.key?("retryable")
|
|
489
641
|
result_error["retryable"]
|
|
490
642
|
else
|
|
491
643
|
GhostCrawl::ErrorCodes::RETRYABLE.fetch(code, false)
|
|
@@ -493,7 +645,7 @@ module GhostCrawl
|
|
|
493
645
|
|
|
494
646
|
target_status = nil
|
|
495
647
|
target_status = result_error["target_status"] if result_error
|
|
496
|
-
target_status ||= hash["target_status"]
|
|
648
|
+
target_status ||= hash["target_status"] # flat markdown-envelope path
|
|
497
649
|
reason = result_error && result_error["reason"]
|
|
498
650
|
|
|
499
651
|
msg = "scrape failed (#{code})"
|
|
@@ -512,6 +664,248 @@ module GhostCrawl
|
|
|
512
664
|
end
|
|
513
665
|
end
|
|
514
666
|
|
|
667
|
+
# ---------------------------------------------------------------------------
|
|
668
|
+
# Typed-but-hash-accessible result models
|
|
669
|
+
#
|
|
670
|
+
# Every scrape/search/crawl response is wrapped in one of these OWNED types so
|
|
671
|
+
# the raw Kiota-generated builders NEVER reach the user surface. Each is a
|
|
672
|
+
# Hash subclass — so it stays 100% back-compatible (String-key access,
|
|
673
|
+
# +is_a?(Hash)+, +to_json+, splat, Enumerable all keep working) — that ALSO
|
|
674
|
+
# exposes the canonical per-lane field names as reader methods and accepts a
|
|
675
|
+
# Symbol key interchangeably with a String key (+result.markdown+ /
|
|
676
|
+
# +result["markdown"]+ / +result[:markdown]+ all resolve).
|
|
677
|
+
#
|
|
678
|
+
# Field names are shared across the GhostCrawl SDKs so results read uniformly.
|
|
679
|
+
# ---------------------------------------------------------------------------
|
|
680
|
+
|
|
681
|
+
# Base for the typed results — a Hash subclass with attribute readers and
|
|
682
|
+
# Symbol/String-interchangeable key access.
|
|
683
|
+
class Result < Hash
|
|
684
|
+
# Declare the canonical lane field names as reader methods. Each reads the
|
|
685
|
+
# underlying payload by String key (returning +nil+ when absent).
|
|
686
|
+
def self.fields(*names)
|
|
687
|
+
names.each do |name|
|
|
688
|
+
key = name.to_s
|
|
689
|
+
define_method(name) { self[key] }
|
|
690
|
+
end
|
|
691
|
+
end
|
|
692
|
+
|
|
693
|
+
# Wrap a plain Hash into this typed result. Idempotent (an already-wrapped
|
|
694
|
+
# value passes through) and a no-op for non-Hash values, so it is always safe
|
|
695
|
+
# to call on a facade return.
|
|
696
|
+
def self.wrap(data)
|
|
697
|
+
return data if data.is_a?(Result) || !data.is_a?(Hash)
|
|
698
|
+
|
|
699
|
+
obj = new
|
|
700
|
+
obj.replace(data)
|
|
701
|
+
obj
|
|
702
|
+
end
|
|
703
|
+
|
|
704
|
+
# Hash-style access that also accepts a Symbol key against a String-keyed
|
|
705
|
+
# payload (and vice-versa), so +result[:markdown]+ and +result["markdown"]+
|
|
706
|
+
# both resolve. Falls back to the native lookup otherwise.
|
|
707
|
+
def [](key)
|
|
708
|
+
return super if key?(key)
|
|
709
|
+
|
|
710
|
+
alt =
|
|
711
|
+
if key.is_a?(Symbol) then key.to_s
|
|
712
|
+
elsif key.is_a?(String) then key.to_sym
|
|
713
|
+
end
|
|
714
|
+
return super(alt) if !alt.nil? && key?(alt)
|
|
715
|
+
|
|
716
|
+
super
|
|
717
|
+
end
|
|
718
|
+
end
|
|
719
|
+
|
|
720
|
+
# Result of +scrape+ / +content+ — category "scrape". +content+ is the
|
|
721
|
+
# cross-format alias for the rendered output, surfaced by the facade.
|
|
722
|
+
class ScrapeResult < Result
|
|
723
|
+
fields :status, :content, :markdown, :html, :status_code, :bytes, :url, :format, :warnings
|
|
724
|
+
end
|
|
725
|
+
|
|
726
|
+
# Result of +search+ — category "search".
|
|
727
|
+
class SearchResult < Result
|
|
728
|
+
fields :status, :query, :items
|
|
729
|
+
|
|
730
|
+
# Guarantee a STABLE result shape regardless of poll outcome. The cold-rotation
|
|
731
|
+
# search-job contract can hand back a raw running handle
|
|
732
|
+
# (+{status:"running", job_id, poll_url,...}+ — no +items+) when the client-side
|
|
733
|
+
# poll budget elapses before the job reaches terminal. This is the last-line
|
|
734
|
+
# safety for ANY construction path: alias +items+ -> +results+, default a nil
|
|
735
|
+
# +results+ to +[]+, and ensure +query+/+status+ keys exist so BOTH
|
|
736
|
+
# +result["results"]+ and +result.results+ never surprise the caller.
|
|
737
|
+
def self.wrap(data)
|
|
738
|
+
obj = super
|
|
739
|
+
return obj unless obj.is_a?(SearchResult)
|
|
740
|
+
|
|
741
|
+
obj["results"] = obj["items"] if obj["results"].nil? && obj["items"].is_a?(Array)
|
|
742
|
+
obj["results"] = [] if obj["results"].nil?
|
|
743
|
+
obj["query"] = nil unless obj.key?("query")
|
|
744
|
+
obj["status"] = nil unless obj.key?("status")
|
|
745
|
+
obj
|
|
746
|
+
end
|
|
747
|
+
|
|
748
|
+
# The +/v1/search+ payload carries hits under the +items+ key, so the
|
|
749
|
+
# canonical +results+ lane falls back to it (mirrors the Python alias) and is
|
|
750
|
+
# never nil — an empty list when the job produced no hits.
|
|
751
|
+
def results
|
|
752
|
+
self["results"] || self["items"] || []
|
|
753
|
+
end
|
|
754
|
+
end
|
|
755
|
+
|
|
756
|
+
# A crawl run — category "crawl". Returned by +crawl+ and the
|
|
757
|
+
# +crawl_runs+ start/get/wait helpers.
|
|
758
|
+
class CrawlRun < Result
|
|
759
|
+
fields :run_id, :status, :results
|
|
760
|
+
end
|
|
761
|
+
|
|
762
|
+
# A single crawled/searched document — category "document".
|
|
763
|
+
class Document < Result
|
|
764
|
+
fields :title, :url, :content
|
|
765
|
+
end
|
|
766
|
+
|
|
767
|
+
# Binary lane result (+pdf+ / +screenshot+) — category "pdf" / "screenshot".
|
|
768
|
+
# The binary methods return raw bytes (a String) for back-compat; this type
|
|
769
|
+
# names the shared canonical fields (+bytes+, +content_type+) so the binary
|
|
770
|
+
# surface reads uniformly with the other typed results across the GhostCrawl SDKs.
|
|
771
|
+
class BinaryResult < Result
|
|
772
|
+
fields :bytes, :content_type
|
|
773
|
+
end
|
|
774
|
+
|
|
775
|
+
# ---------------------------------------------------------------------------
|
|
776
|
+
# Search(wait:) — SDK-owned bounded client-side poll
|
|
777
|
+
#
|
|
778
|
+
# Unlike crawl — whose +GET /v1/crawl-runs/{id}?wait=true&timeout_s=N+ blocks
|
|
779
|
+
# SERVER-side on a completion event — the search-job contract is a CLIENT poll:
|
|
780
|
+
# +POST /v1/search+ on a cold rotation returns
|
|
781
|
+
#
|
|
782
|
+
# 202 { status: "running", poll_url, deadline, poll_max_seconds, poll_terminal_on_404 }
|
|
783
|
+
#
|
|
784
|
+
# and +GET {poll_url}+ returns 202 running (with a +Retry-After+ header) until a
|
|
785
|
+
# terminal 200. The SDK owns the loop so the caller writes zero poll code — the
|
|
786
|
+
# same +wait: true+ DX crawl has, a different mechanism under the hood. Mirrors
|
|
787
|
+
# the Go (search_wait.go), Node (_searchPoll.ts) and Python (_search_poll.py)
|
|
788
|
+
# references.
|
|
789
|
+
#
|
|
790
|
+
# The engine is a pure function of an injected +get+ getter (and +sleeper+), so
|
|
791
|
+
# the poll contract is unit-testable with no network (see
|
|
792
|
+
# test/test_typed_results_and_search_wait.rb).
|
|
793
|
+
# ---------------------------------------------------------------------------
|
|
794
|
+
|
|
795
|
+
# @api private
|
|
796
|
+
module SearchPoll
|
|
797
|
+
# Terminal search-job states (mirrors the Go/Node/Python terminal set).
|
|
798
|
+
TERMINAL_STATES = %w[completed done failed cancelled error].freeze
|
|
799
|
+
# Overall client-side wait budget (seconds) when the server gives no bound.
|
|
800
|
+
DEFAULT_TIMEOUT = 300
|
|
801
|
+
# Inter-poll fallback (seconds) when the server sends no Retry-After.
|
|
802
|
+
DEFAULT_INTERVAL = 3
|
|
803
|
+
|
|
804
|
+
# True when +job+ is a Hash carrying a terminal +status+.
|
|
805
|
+
def self.terminal?(job)
|
|
806
|
+
job.is_a?(Hash) && TERMINAL_STATES.include?(job["status"].to_s)
|
|
807
|
+
end
|
|
808
|
+
|
|
809
|
+
# Parses a +Retry-After+ delta-seconds value to a non-negative Float, or +nil+
|
|
810
|
+
# when absent/invalid. (The search-job contract advertises delta-seconds only,
|
|
811
|
+
# not the HTTP-date form.)
|
|
812
|
+
def self.parse_retry_after(raw)
|
|
813
|
+
return nil if raw.nil?
|
|
814
|
+
|
|
815
|
+
s = raw.to_s.strip
|
|
816
|
+
return nil if s.empty?
|
|
817
|
+
|
|
818
|
+
f = Float(s, exception: false)
|
|
819
|
+
return nil if f.nil? || f.negative?
|
|
820
|
+
|
|
821
|
+
f
|
|
822
|
+
end
|
|
823
|
+
|
|
824
|
+
# Resolves a +poll_url+ that may be absolute or base-relative.
|
|
825
|
+
def self.resolve_url(base_url, poll_url)
|
|
826
|
+
return poll_url if poll_url.start_with?("http://", "https://")
|
|
827
|
+
|
|
828
|
+
"#{base_url.to_s.gsub(%r{/+$}, "")}/#{poll_url.to_s.gsub(%r{^/+}, "")}"
|
|
829
|
+
end
|
|
830
|
+
|
|
831
|
+
# Bounded-polls a cold search job to a terminal result.
|
|
832
|
+
#
|
|
833
|
+
# Copies the STRUCTURE of {CrawlRunsClient#wait_for_completion} (deadline +
|
|
834
|
+
# re-arm loop) but implements the client-poll contract: follows +poll_url+ from
|
|
835
|
+
# the 202 body, honours +Retry-After+ between polls bounded by
|
|
836
|
+
# +poll_max_seconds+ / an explicit +timeout+ / {DEFAULT_TIMEOUT}, treats a
|
|
837
|
+
# terminal 200 (or a terminal-status body) as done, treats 404 as terminal when
|
|
838
|
+
# +poll_terminal_on_404+ (default true), and returns the latest non-terminal
|
|
839
|
+
# record if the deadline elapses first. When the job carries no +poll_url+ there
|
|
840
|
+
# is nothing to poll and the initial body is returned unchanged (warm path).
|
|
841
|
+
#
|
|
842
|
+
# @param job [Hash] the initial (202) job body
|
|
843
|
+
# @param base_url [String] client base URL (to resolve a relative poll_url)
|
|
844
|
+
# @param get [#call] +->(url) { [status, headers, body_hash] }+ — one poll GET
|
|
845
|
+
# @param timeout [Numeric, nil] overall client-side wait budget (seconds)
|
|
846
|
+
# @param initial_retry [Numeric, nil] first inter-poll delay (from the 202's Retry-After)
|
|
847
|
+
# @param sleeper [#call, nil] +->(seconds) {}+ delay hook (injectable for tests)
|
|
848
|
+
# @return [Hash] the terminal job body (or the latest non-terminal on timeout)
|
|
849
|
+
def self.poll_search_job(job, base_url:, get:, timeout: nil, initial_retry: nil, sleeper: nil)
|
|
850
|
+
poll_url = job["poll_url"]
|
|
851
|
+
return job if poll_url.nil? || poll_url.to_s.empty?
|
|
852
|
+
|
|
853
|
+
url = resolve_url(base_url, poll_url.to_s)
|
|
854
|
+
budget = (timeout || job["poll_max_seconds"] || DEFAULT_TIMEOUT).to_f
|
|
855
|
+
terminal_on_404 = job.key?("poll_terminal_on_404") ? job["poll_terminal_on_404"] : true
|
|
856
|
+
sleeper ||= ->(seconds) { sleep(seconds) if seconds.positive? }
|
|
857
|
+
|
|
858
|
+
deadline = monotonic + budget
|
|
859
|
+
retry_after = (initial_retry || DEFAULT_INTERVAL).to_f
|
|
860
|
+
latest = job
|
|
861
|
+
|
|
862
|
+
loop do
|
|
863
|
+
remaining = deadline - monotonic
|
|
864
|
+
return latest if remaining <= 0
|
|
865
|
+
|
|
866
|
+
delay = [retry_after, remaining].min
|
|
867
|
+
sleeper.call(delay) if delay.positive?
|
|
868
|
+
return latest if monotonic > deadline
|
|
869
|
+
|
|
870
|
+
status, headers, data = get.call(url)
|
|
871
|
+
data = {} unless data.is_a?(Hash)
|
|
872
|
+
|
|
873
|
+
if status == 404
|
|
874
|
+
ResponseHelper.raise_from_response!(FakeStatus.new(404, data)) if terminal_on_404
|
|
875
|
+
return latest
|
|
876
|
+
end
|
|
877
|
+
ResponseHelper.raise_from_response!(FakeStatus.new(status, data)) if status >= 400
|
|
878
|
+
|
|
879
|
+
return data if status == 200 || terminal?(data)
|
|
880
|
+
|
|
881
|
+
latest = data unless data.empty?
|
|
882
|
+
ra = parse_retry_after(headers && (headers["Retry-After"] || headers["retry-after"]))
|
|
883
|
+
retry_after = ra if ra
|
|
884
|
+
end
|
|
885
|
+
end
|
|
886
|
+
|
|
887
|
+
# Monotonic clock (seconds) — immune to wall-clock adjustments during a wait.
|
|
888
|
+
def self.monotonic
|
|
889
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
890
|
+
end
|
|
891
|
+
|
|
892
|
+
# Minimal response-shim so a poll-path error routes through the SAME typed
|
|
893
|
+
# translation ({ResponseHelper.raise_from_response!}) the modeled calls use.
|
|
894
|
+
# @api private
|
|
895
|
+
class FakeStatus
|
|
896
|
+
def initialize(status, body)
|
|
897
|
+
@status = status
|
|
898
|
+
@body = body
|
|
899
|
+
end
|
|
900
|
+
|
|
901
|
+
attr_reader :status
|
|
902
|
+
|
|
903
|
+
def body
|
|
904
|
+
@body.is_a?(String) ? @body : JSON.generate(@body || {})
|
|
905
|
+
end
|
|
906
|
+
end
|
|
907
|
+
end
|
|
908
|
+
|
|
515
909
|
# ---------------------------------------------------------------------------
|
|
516
910
|
# Sub-clients — each delegates to the generated v1 request builders
|
|
517
911
|
# ---------------------------------------------------------------------------
|
|
@@ -565,7 +959,8 @@ module GhostCrawl
|
|
|
565
959
|
|
|
566
960
|
unless wait
|
|
567
961
|
hash = ResponseHelper.to_hash(@v1.crawl_runs.post(AdditionalDataBody.new(data)))
|
|
568
|
-
|
|
962
|
+
result = raise_on_result_error ? ResponseHelper.raise_on_result_error!(hash) : hash
|
|
963
|
+
return CrawlRun.wrap(result)
|
|
569
964
|
end
|
|
570
965
|
|
|
571
966
|
# Start-and-wait: ask the server to block until terminal. Bound the first
|
|
@@ -575,12 +970,13 @@ module GhostCrawl
|
|
|
575
970
|
data["wait_until"] = "completed"
|
|
576
971
|
data["timeout_s"] = window
|
|
577
972
|
hash = ResponseHelper.to_hash(@v1.crawl_runs.post(AdditionalDataBody.new(data)))
|
|
578
|
-
return hash if terminal?(hash)
|
|
973
|
+
return CrawlRun.wrap(hash) if terminal?(hash)
|
|
579
974
|
|
|
580
975
|
run_id = hash["run_id"]
|
|
581
976
|
remaining = timeout - window
|
|
582
|
-
return hash if run_id.nil? || run_id.to_s.empty? || remaining <= 0
|
|
583
|
-
|
|
977
|
+
return CrawlRun.wrap(hash) if run_id.nil? || run_id.to_s.empty? || remaining <= 0
|
|
978
|
+
|
|
979
|
+
CrawlRun.wrap(wait_for_completion(run_id, timeout: remaining))
|
|
584
980
|
end
|
|
585
981
|
|
|
586
982
|
# Block until an existing crawl run reaches a terminal state, or +timeout+
|
|
@@ -597,9 +993,7 @@ module GhostCrawl
|
|
|
597
993
|
# @param timeout [Integer] total seconds to wait (default 300)
|
|
598
994
|
# @return [Hash] the run record (terminal when it finished in time)
|
|
599
995
|
def wait_for_completion(run_id, timeout: 300)
|
|
600
|
-
if run_id.nil? || run_id.to_s.empty?
|
|
601
|
-
raise ArgumentError, "run_id is required"
|
|
602
|
-
end
|
|
996
|
+
raise ArgumentError, "run_id is required" if run_id.nil? || run_id.to_s.empty?
|
|
603
997
|
|
|
604
998
|
deadline = monotonic + timeout
|
|
605
999
|
loop do
|
|
@@ -609,7 +1003,7 @@ module GhostCrawl
|
|
|
609
1003
|
|
|
610
1004
|
window = [remaining, WAIT_WINDOW_S].min
|
|
611
1005
|
run = fetch_waiting(run_id, window)
|
|
612
|
-
return run if terminal?(run)
|
|
1006
|
+
return CrawlRun.wrap(run) if terminal?(run)
|
|
613
1007
|
# The server just blocked for ~window seconds; loop straight into the
|
|
614
1008
|
# next blocking window. No client-side delay.
|
|
615
1009
|
end
|
|
@@ -624,7 +1018,7 @@ module GhostCrawl
|
|
|
624
1018
|
# Get a single crawl run by ID.
|
|
625
1019
|
# Delegates to GET /v1/crawl-runs/{run_id} via the generated builder.
|
|
626
1020
|
def get(run_id)
|
|
627
|
-
ResponseHelper.to_hash(@v1.crawl_runs.by_run_id(run_id).get)
|
|
1021
|
+
CrawlRun.wrap(ResponseHelper.to_hash(@v1.crawl_runs.by_run_id(run_id).get))
|
|
628
1022
|
end
|
|
629
1023
|
|
|
630
1024
|
# Cancel a running crawl run.
|
|
@@ -633,6 +1027,13 @@ module GhostCrawl
|
|
|
633
1027
|
ResponseHelper.to_hash(@v1.crawl_runs.by_run_id(run_id).cancel.post)
|
|
634
1028
|
end
|
|
635
1029
|
|
|
1030
|
+
# Resume a paused/cancelled crawl run.
|
|
1031
|
+
# Delegates to POST /v1/crawl-runs/{run_id}/resume through the shared adapter.
|
|
1032
|
+
def resume(run_id, **opts)
|
|
1033
|
+
ResponseHelper.json_request!(@adapter, :POST, "/v1/crawl-runs/#{run_id}/resume",
|
|
1034
|
+
opts.transform_keys(&:to_s))
|
|
1035
|
+
end
|
|
1036
|
+
|
|
636
1037
|
private
|
|
637
1038
|
|
|
638
1039
|
# True when +run+ is a Hash carrying a terminal +status+.
|
|
@@ -663,8 +1064,9 @@ module GhostCrawl
|
|
|
663
1064
|
|
|
664
1065
|
# Manage browser sessions — /v1/sessions.
|
|
665
1066
|
class SessionsClient
|
|
666
|
-
def initialize(v1)
|
|
1067
|
+
def initialize(v1, adapter = nil)
|
|
667
1068
|
@v1 = v1
|
|
1069
|
+
@adapter = adapter
|
|
668
1070
|
end
|
|
669
1071
|
|
|
670
1072
|
# List all active sessions.
|
|
@@ -675,9 +1077,20 @@ module GhostCrawl
|
|
|
675
1077
|
|
|
676
1078
|
# Create a new browser session.
|
|
677
1079
|
# Delegates to POST /v1/sessions/create via the generated builder.
|
|
678
|
-
#
|
|
679
|
-
|
|
680
|
-
|
|
1080
|
+
#
|
|
1081
|
+
# Two creation modes (pass exactly one — passing both is a server-side
|
|
1082
|
+
# validation error):
|
|
1083
|
+
# - profile-based: pass +profile_name+ to rehydrate a saved identity profile
|
|
1084
|
+
# (the org must already own a profile of that name, else the API 404s).
|
|
1085
|
+
# - engine-based (ad-hoc): omit +profile_name+ and the API opens a fresh
|
|
1086
|
+
# session on +engine+ ("chrome"/"firefox"/"webkit"/"auto"; default "auto")
|
|
1087
|
+
# with no saved profile required — the zero-setup path that works on a
|
|
1088
|
+
# brand-new key with no profiles.
|
|
1089
|
+
# @param profile_name [String, nil] identity profile to rehydrate (optional)
|
|
1090
|
+
# @param engine [String, nil] engine for an ad-hoc session when no profile
|
|
1091
|
+
def create(profile_name: nil, engine: nil, **opts)
|
|
1092
|
+
base = profile_name.nil? ? { "engine" => engine || "auto" } : { "profile" => profile_name }
|
|
1093
|
+
data = base.merge(opts.transform_keys(&:to_s))
|
|
681
1094
|
ResponseHelper.to_hash(@v1.sessions.create.post(AdditionalDataBody.new(data)))
|
|
682
1095
|
end
|
|
683
1096
|
|
|
@@ -685,8 +1098,8 @@ module GhostCrawl
|
|
|
685
1098
|
# Delegates to POST /v1/sessions/{id}/extend via the generated builder.
|
|
686
1099
|
def extend(session_id, duration_seconds: 300)
|
|
687
1100
|
ResponseHelper.to_hash(@v1.sessions.by_profile_id(session_id).extend.post(
|
|
688
|
-
|
|
689
|
-
|
|
1101
|
+
AdditionalDataBody.new({ "ttl_seconds" => duration_seconds })
|
|
1102
|
+
))
|
|
690
1103
|
end
|
|
691
1104
|
|
|
692
1105
|
# Release a session back to the pool.
|
|
@@ -694,6 +1107,262 @@ module GhostCrawl
|
|
|
694
1107
|
def release(session_id)
|
|
695
1108
|
ResponseHelper.to_hash(@v1.sessions.by_profile_id(session_id).release.post)
|
|
696
1109
|
end
|
|
1110
|
+
|
|
1111
|
+
# Pin a session so subsequent calls stick to the same live browser.
|
|
1112
|
+
# Delegates to POST /v1/sessions/{session_id}/pin via the shared adapter.
|
|
1113
|
+
def pin(session_id, **opts)
|
|
1114
|
+
ResponseHelper.json_request!(@adapter, :POST, "/v1/sessions/#{session_id}/pin",
|
|
1115
|
+
opts.transform_keys(&:to_s))
|
|
1116
|
+
end
|
|
1117
|
+
|
|
1118
|
+
# Start or release an interactive takeover of a live browser session.
|
|
1119
|
+
# Delegates to POST /v1/sessions/{session_id}/takeover via the shared adapter.
|
|
1120
|
+
# @param action [String] "mint" (begin) or "release" (hand control back)
|
|
1121
|
+
def takeover(session_id, action: "mint", **opts)
|
|
1122
|
+
data = { "action" => action }.merge(opts.transform_keys(&:to_s))
|
|
1123
|
+
ResponseHelper.json_request!(@adapter, :POST, "/v1/sessions/#{session_id}/takeover", data)
|
|
1124
|
+
end
|
|
1125
|
+
|
|
1126
|
+
# Mint a short-lived interactive-takeover token for a session.
|
|
1127
|
+
# Delegates to POST /v1/sessions/{session_id}/takeover_token via the shared adapter.
|
|
1128
|
+
def takeover_token(session_id, **opts)
|
|
1129
|
+
ResponseHelper.json_request!(@adapter, :POST, "/v1/sessions/#{session_id}/takeover_token",
|
|
1130
|
+
opts.transform_keys(&:to_s))
|
|
1131
|
+
end
|
|
1132
|
+
|
|
1133
|
+
# Release interactive-takeover control of a session (return it to view-only).
|
|
1134
|
+
# Delegates to POST /v1/sessions/{session_id}/takeover_release via the shared adapter.
|
|
1135
|
+
def takeover_release(session_id, **opts)
|
|
1136
|
+
ResponseHelper.json_request!(@adapter, :POST, "/v1/sessions/#{session_id}/takeover_release",
|
|
1137
|
+
opts.transform_keys(&:to_s))
|
|
1138
|
+
end
|
|
1139
|
+
|
|
1140
|
+
# Terminate a session immediately (tear down the browser and free the slot).
|
|
1141
|
+
# Delegates to POST /v1/sessions/{session_id}/terminate via the shared adapter.
|
|
1142
|
+
def terminate(session_id, **opts)
|
|
1143
|
+
ResponseHelper.json_request!(@adapter, :POST, "/v1/sessions/#{session_id}/terminate",
|
|
1144
|
+
opts.transform_keys(&:to_s))
|
|
1145
|
+
end
|
|
1146
|
+
|
|
1147
|
+
# Start a screen recording for a pinned session.
|
|
1148
|
+
# Delegates to POST /v1/sessions/{profile_id}/recording/start via the shared adapter.
|
|
1149
|
+
def recording_start(profile_id, **opts)
|
|
1150
|
+
ResponseHelper.json_request!(@adapter, :POST, "/v1/sessions/#{profile_id}/recording/start",
|
|
1151
|
+
opts.transform_keys(&:to_s))
|
|
1152
|
+
end
|
|
1153
|
+
|
|
1154
|
+
# Stop a session's screen recording.
|
|
1155
|
+
# Delegates to POST /v1/sessions/{profile_id}/recording/stop via the shared adapter.
|
|
1156
|
+
def recording_stop(profile_id, **opts)
|
|
1157
|
+
ResponseHelper.json_request!(@adapter, :POST, "/v1/sessions/#{profile_id}/recording/stop",
|
|
1158
|
+
opts.transform_keys(&:to_s))
|
|
1159
|
+
end
|
|
1160
|
+
end
|
|
1161
|
+
|
|
1162
|
+
# Low-level CDP primitives — /v1/cdp/*. These drive a live session directly.
|
|
1163
|
+
# Each delegates through the shared
|
|
1164
|
+
# authenticated adapter pipeline.
|
|
1165
|
+
class CdpClient
|
|
1166
|
+
def initialize(v1, adapter = nil)
|
|
1167
|
+
@v1 = v1
|
|
1168
|
+
@adapter = adapter
|
|
1169
|
+
end
|
|
1170
|
+
|
|
1171
|
+
# Get the current CDP frame tree for a session.
|
|
1172
|
+
# Delegates to POST /v1/cdp/frame.
|
|
1173
|
+
def frame(session_id: nil, **opts)
|
|
1174
|
+
data = opts.transform_keys(&:to_s)
|
|
1175
|
+
data["session_id"] = session_id unless session_id.nil?
|
|
1176
|
+
ResponseHelper.json_request!(@adapter, :POST, "/v1/cdp/frame", data)
|
|
1177
|
+
end
|
|
1178
|
+
|
|
1179
|
+
# Dispatch a raw CDP input event (mouse/keyboard) to a session.
|
|
1180
|
+
# Delegates to POST /v1/cdp/input.
|
|
1181
|
+
def input(session_id: nil, **opts)
|
|
1182
|
+
data = opts.transform_keys(&:to_s)
|
|
1183
|
+
data["session_id"] = session_id unless session_id.nil?
|
|
1184
|
+
ResponseHelper.json_request!(@adapter, :POST, "/v1/cdp/input", data)
|
|
1185
|
+
end
|
|
1186
|
+
|
|
1187
|
+
# Get the current page URL for a session via CDP.
|
|
1188
|
+
# Delegates to POST /v1/cdp/url.
|
|
1189
|
+
def url(session_id: nil, **opts)
|
|
1190
|
+
data = opts.transform_keys(&:to_s)
|
|
1191
|
+
data["session_id"] = session_id unless session_id.nil?
|
|
1192
|
+
ResponseHelper.json_request!(@adapter, :POST, "/v1/cdp/url", data)
|
|
1193
|
+
end
|
|
1194
|
+
end
|
|
1195
|
+
|
|
1196
|
+
# Page-level primitives against a live session — /v1/page/*. First-class
|
|
1197
|
+
# customer surface; each delegates through the shared adapter.
|
|
1198
|
+
class PageClient
|
|
1199
|
+
def initialize(v1, adapter = nil)
|
|
1200
|
+
@v1 = v1
|
|
1201
|
+
@adapter = adapter
|
|
1202
|
+
end
|
|
1203
|
+
|
|
1204
|
+
# Read the session's cookies.
|
|
1205
|
+
# Delegates to GET /v1/page/cookies.
|
|
1206
|
+
def get_cookies(**opts)
|
|
1207
|
+
qs = opts.empty? ? "" : "?#{URI.encode_www_form(opts)}"
|
|
1208
|
+
ResponseHelper.json_request!(@adapter, :GET, "/v1/page/cookies#{qs}", nil)
|
|
1209
|
+
end
|
|
1210
|
+
|
|
1211
|
+
# Set cookies on the session.
|
|
1212
|
+
# Delegates to POST /v1/page/cookies.
|
|
1213
|
+
def set_cookies(**opts)
|
|
1214
|
+
ResponseHelper.json_request!(@adapter, :POST, "/v1/page/cookies", opts.transform_keys(&:to_s))
|
|
1215
|
+
end
|
|
1216
|
+
|
|
1217
|
+
# Delete cookies from the session.
|
|
1218
|
+
# Delegates to DELETE /v1/page/cookies.
|
|
1219
|
+
def delete_cookies(**opts)
|
|
1220
|
+
ResponseHelper.json_request!(@adapter, :DELETE, "/v1/page/cookies", opts.transform_keys(&:to_s))
|
|
1221
|
+
end
|
|
1222
|
+
|
|
1223
|
+
# Capture a DOM snapshot of the current page.
|
|
1224
|
+
# Delegates to POST /v1/page/dom_snapshot.
|
|
1225
|
+
def dom_snapshot(**opts)
|
|
1226
|
+
ResponseHelper.json_request!(@adapter, :POST, "/v1/page/dom_snapshot", opts.transform_keys(&:to_s))
|
|
1227
|
+
end
|
|
1228
|
+
|
|
1229
|
+
# Trigger a download from the current page.
|
|
1230
|
+
# Delegates to POST /v1/page/download.
|
|
1231
|
+
def download(**opts)
|
|
1232
|
+
ResponseHelper.json_request!(@adapter, :POST, "/v1/page/download", opts.transform_keys(&:to_s))
|
|
1233
|
+
end
|
|
1234
|
+
|
|
1235
|
+
# Evaluate JavaScript in the page context.
|
|
1236
|
+
# Delegates to POST /v1/page/eval.
|
|
1237
|
+
def eval(**opts)
|
|
1238
|
+
ResponseHelper.json_request!(@adapter, :POST, "/v1/page/eval", opts.transform_keys(&:to_s))
|
|
1239
|
+
end
|
|
1240
|
+
|
|
1241
|
+
# Export the page's network activity as a HAR archive.
|
|
1242
|
+
# Delegates to POST /v1/page/har.
|
|
1243
|
+
def har(**opts)
|
|
1244
|
+
ResponseHelper.json_request!(@adapter, :POST, "/v1/page/har", opts.transform_keys(&:to_s))
|
|
1245
|
+
end
|
|
1246
|
+
|
|
1247
|
+
# Scroll the page.
|
|
1248
|
+
# Delegates to POST /v1/page/scroll.
|
|
1249
|
+
def scroll(**opts)
|
|
1250
|
+
ResponseHelper.json_request!(@adapter, :POST, "/v1/page/scroll", opts.transform_keys(&:to_s))
|
|
1251
|
+
end
|
|
1252
|
+
|
|
1253
|
+
# Upload a file into a file input on the page.
|
|
1254
|
+
# Delegates to POST /v1/page/upload.
|
|
1255
|
+
def upload(**opts)
|
|
1256
|
+
ResponseHelper.json_request!(@adapter, :POST, "/v1/page/upload", opts.transform_keys(&:to_s))
|
|
1257
|
+
end
|
|
1258
|
+
|
|
1259
|
+
# Wait for a page condition (selector / navigation / timeout).
|
|
1260
|
+
# Delegates to POST /v1/page/wait.
|
|
1261
|
+
def wait(**opts)
|
|
1262
|
+
ResponseHelper.json_request!(@adapter, :POST, "/v1/page/wait", opts.transform_keys(&:to_s))
|
|
1263
|
+
end
|
|
1264
|
+
|
|
1265
|
+
# Replay a mouse op-stream on the live page.
|
|
1266
|
+
# +ops+ is an ordered list of { "kind" => "move|down|up|wheel|dwell", ... } driven
|
|
1267
|
+
# natively on the live page (engine-agnostic: the same trusted trajectory + settled
|
|
1268
|
+
# click on every engine). Extra keyword options (e.g. +session_id:+) pass through.
|
|
1269
|
+
# Delegates to POST /v1/page/mouse.
|
|
1270
|
+
def mouse(ops, **opts)
|
|
1271
|
+
raise ArgumentError, "ops must be an Array" unless ops.is_a?(Array)
|
|
1272
|
+
|
|
1273
|
+
body = { "ops" => ops }.merge(opts.transform_keys(&:to_s))
|
|
1274
|
+
ResponseHelper.json_request!(@adapter, :POST, "/v1/page/mouse", body)
|
|
1275
|
+
end
|
|
1276
|
+
|
|
1277
|
+
# Arrive at +dest_url+ through a native link click from +source_url+, so the live
|
|
1278
|
+
# page's document.referrer, Referer and Sec-Fetch-* are produced natively by the
|
|
1279
|
+
# click (engine-agnostic). Optional: +referrer_shape:+ ("origin" | "full", default
|
|
1280
|
+
# "origin"), +source_title:+, +session_id:+.
|
|
1281
|
+
# Delegates to POST /v1/page/arrival.
|
|
1282
|
+
def arrival(dest_url, source_url, **opts)
|
|
1283
|
+
body = { "dest_url" => dest_url, "source_url" => source_url }.merge(opts.transform_keys(&:to_s))
|
|
1284
|
+
ResponseHelper.json_request!(@adapter, :POST, "/v1/page/arrival", body)
|
|
1285
|
+
end
|
|
1286
|
+
|
|
1287
|
+
# Visit a URL in a real new tab of the session's context, read it briefly, then
|
|
1288
|
+
# close it (the session continues on its original tab). e.g.
|
|
1289
|
+
# +tab_visit(url: "https://example.com", dwell_s: 3, session_id: sid)+.
|
|
1290
|
+
# Chrome engine only; other engines report +unsupported+.
|
|
1291
|
+
# Delegates to POST /v1/page/tab-visit.
|
|
1292
|
+
def tab_visit(**opts)
|
|
1293
|
+
ResponseHelper.json_request!(@adapter, :POST, "/v1/page/tab-visit", opts.transform_keys(&:to_s))
|
|
1294
|
+
end
|
|
1295
|
+
|
|
1296
|
+
# Dismiss overlay popups on the live page. Delegates to POST /v1/page/close-popups.
|
|
1297
|
+
def close_popups(**opts)
|
|
1298
|
+
ResponseHelper.json_request!(@adapter, :POST, "/v1/page/close-popups", opts.transform_keys(&:to_s))
|
|
1299
|
+
end
|
|
1300
|
+
|
|
1301
|
+
# Replay a touch op-stream on the live page. +ops+ is an ordered list of native
|
|
1302
|
+
# tap/gesture ops (engine-agnostic). Delegates to POST /v1/page/touch.
|
|
1303
|
+
def touch(ops, **opts)
|
|
1304
|
+
raise ArgumentError, "ops must be an Array" unless ops.is_a?(Array)
|
|
1305
|
+
|
|
1306
|
+
body = { "ops" => ops }.merge(opts.transform_keys(&:to_s))
|
|
1307
|
+
ResponseHelper.json_request!(@adapter, :POST, "/v1/page/touch", body)
|
|
1308
|
+
end
|
|
1309
|
+
end
|
|
1310
|
+
|
|
1311
|
+
# Manage completed-crawl download artifacts — /v1/downloads.
|
|
1312
|
+
class DownloadsClient
|
|
1313
|
+
def initialize(v1, adapter = nil)
|
|
1314
|
+
@v1 = v1
|
|
1315
|
+
@adapter = adapter
|
|
1316
|
+
end
|
|
1317
|
+
|
|
1318
|
+
# List available download artifacts.
|
|
1319
|
+
# Delegates to GET /v1/downloads.
|
|
1320
|
+
def list(**opts)
|
|
1321
|
+
qs = opts.empty? ? "" : "?#{URI.encode_www_form(opts)}"
|
|
1322
|
+
ResponseHelper.json_request!(@adapter, :GET, "/v1/downloads#{qs}", nil)
|
|
1323
|
+
end
|
|
1324
|
+
|
|
1325
|
+
# Get a single download artifact's metadata by ID.
|
|
1326
|
+
# Delegates to GET /v1/downloads/{download_id}.
|
|
1327
|
+
def get(download_id)
|
|
1328
|
+
ResponseHelper.json_request!(@adapter, :GET, "/v1/downloads/#{download_id}", nil)
|
|
1329
|
+
end
|
|
1330
|
+
|
|
1331
|
+
# Delete a download artifact by ID.
|
|
1332
|
+
# Delegates to DELETE /v1/downloads/{download_id}.
|
|
1333
|
+
def delete(download_id)
|
|
1334
|
+
ResponseHelper.json_request!(@adapter, :DELETE, "/v1/downloads/#{download_id}", nil)
|
|
1335
|
+
end
|
|
1336
|
+
end
|
|
1337
|
+
|
|
1338
|
+
# Manage persisted session storage-states — /v1/storage-states. Exposed via the
|
|
1339
|
+
# top-level +storage_states+ list method and +storage_states_attach/detach+ on
|
|
1340
|
+
# the client; this sub-client groups the attach/detach lifecycle.
|
|
1341
|
+
class StorageStatesClient
|
|
1342
|
+
def initialize(v1, adapter = nil)
|
|
1343
|
+
@v1 = v1
|
|
1344
|
+
@adapter = adapter
|
|
1345
|
+
end
|
|
1346
|
+
|
|
1347
|
+
# List the account's persisted storage-states.
|
|
1348
|
+
# Delegates to GET /v1/storage-states.
|
|
1349
|
+
def list
|
|
1350
|
+
ResponseHelper.json_request!(@adapter, :GET, "/v1/storage-states", nil)
|
|
1351
|
+
end
|
|
1352
|
+
|
|
1353
|
+
# Attach a storage-state to a session/profile.
|
|
1354
|
+
# Delegates to POST /v1/storage-states/{id_or_name}/attach.
|
|
1355
|
+
def attach(id_or_name, **opts)
|
|
1356
|
+
ResponseHelper.json_request!(@adapter, :POST, "/v1/storage-states/#{id_or_name}/attach",
|
|
1357
|
+
opts.transform_keys(&:to_s))
|
|
1358
|
+
end
|
|
1359
|
+
|
|
1360
|
+
# Detach a storage-state.
|
|
1361
|
+
# Delegates to POST /v1/storage-states/detach.
|
|
1362
|
+
def detach(**opts)
|
|
1363
|
+
ResponseHelper.json_request!(@adapter, :POST, "/v1/storage-states/detach",
|
|
1364
|
+
opts.transform_keys(&:to_s))
|
|
1365
|
+
end
|
|
697
1366
|
end
|
|
698
1367
|
|
|
699
1368
|
# Manage identity profiles — /v1/profiles.
|
|
@@ -781,6 +1450,28 @@ module GhostCrawl
|
|
|
781
1450
|
def rotate_secret(webhook_id)
|
|
782
1451
|
ResponseHelper.to_hash(@v1.webhooks.by_webhook_id(webhook_id).rotate_secret.post)
|
|
783
1452
|
end
|
|
1453
|
+
|
|
1454
|
+
# List the delivery attempts for a webhook.
|
|
1455
|
+
# Delegates to GET /v1/webhooks/{webhook_id}/deliveries via the shared adapter.
|
|
1456
|
+
def list_deliveries(webhook_id, **opts)
|
|
1457
|
+
qs = opts.empty? ? "" : "?#{URI.encode_www_form(opts)}"
|
|
1458
|
+
ResponseHelper.json_request!(@adapter, :GET, "/v1/webhooks/#{webhook_id}/deliveries#{qs}", nil)
|
|
1459
|
+
end
|
|
1460
|
+
|
|
1461
|
+
# Alias for {#list_deliveries} — matches the canonical +webhooks.deliveries+ name.
|
|
1462
|
+
def deliveries(webhook_id, **opts)
|
|
1463
|
+
list_deliveries(webhook_id, **opts)
|
|
1464
|
+
end
|
|
1465
|
+
|
|
1466
|
+
# Retry a single webhook delivery attempt.
|
|
1467
|
+
# Delegates to POST /v1/webhooks/{webhook_id}/deliveries/{delivery_id}/retry.
|
|
1468
|
+
def retry_delivery(webhook_id, delivery_id, **opts)
|
|
1469
|
+
ResponseHelper.json_request!(
|
|
1470
|
+
@adapter, :POST,
|
|
1471
|
+
"/v1/webhooks/#{webhook_id}/deliveries/#{delivery_id}/retry",
|
|
1472
|
+
opts.transform_keys(&:to_s)
|
|
1473
|
+
)
|
|
1474
|
+
end
|
|
784
1475
|
end
|
|
785
1476
|
|
|
786
1477
|
# Manage schedules — /v1/schedules.
|
|
@@ -822,7 +1513,7 @@ module GhostCrawl
|
|
|
822
1513
|
if task.is_a?(Hash)
|
|
823
1514
|
t = task.transform_keys(&:to_s)
|
|
824
1515
|
job_type ||= t["action"] || t["job_type"] || t["type"]
|
|
825
|
-
job_params ||= t.
|
|
1516
|
+
job_params ||= t.except("action", "job_type", "type")
|
|
826
1517
|
end
|
|
827
1518
|
|
|
828
1519
|
data = { "cron_expr" => cron }
|
|
@@ -831,7 +1522,7 @@ module GhostCrawl
|
|
|
831
1522
|
data["job_params"] = job_params unless job_params.nil?
|
|
832
1523
|
# opts may override/supply name/job_type/job_params/notify_webhook/monitor_mode.
|
|
833
1524
|
# Never forward a raw `task` field — the API 422s on it.
|
|
834
|
-
data.merge!(opts.transform_keys(&:to_s).
|
|
1525
|
+
data.merge!(opts.transform_keys(&:to_s).except("task"))
|
|
835
1526
|
|
|
836
1527
|
ResponseHelper.to_hash(@v1.schedules.post(AdditionalDataBody.new(data)))
|
|
837
1528
|
end
|
|
@@ -842,7 +1533,8 @@ module GhostCrawl
|
|
|
842
1533
|
# so the DELETE never fired). {ResponseHelper.void_request!} runs it and tolerates
|
|
843
1534
|
# the empty 204 body.
|
|
844
1535
|
def delete(schedule_id)
|
|
845
|
-
ResponseHelper.void_request!(@adapter,
|
|
1536
|
+
ResponseHelper.void_request!(@adapter,
|
|
1537
|
+
@v1.schedules.by_schedule_id(schedule_id).to_delete_request_information(nil))
|
|
846
1538
|
end
|
|
847
1539
|
end
|
|
848
1540
|
|
|
@@ -916,8 +1608,61 @@ module GhostCrawl
|
|
|
916
1608
|
# (generated builder passes a +nil+ factory → adapter raises before sending), so
|
|
917
1609
|
# we route through {ResponseHelper.void_request!}.
|
|
918
1610
|
def delete(recording_id)
|
|
919
|
-
ResponseHelper.void_request!(@adapter,
|
|
1611
|
+
ResponseHelper.void_request!(@adapter,
|
|
1612
|
+
@v1.recordings.by_recording_id(recording_id).to_delete_request_information(nil))
|
|
1613
|
+
end
|
|
1614
|
+
|
|
1615
|
+
# Visual (live-view) recording sub-client for a session — /v1/recordings/{id}/visual.
|
|
1616
|
+
# @param session_id [String] the session to observe
|
|
1617
|
+
# @return [VisualClient]
|
|
1618
|
+
def visual(session_id)
|
|
1619
|
+
VisualClient.new(session_id, @adapter)
|
|
1620
|
+
end
|
|
1621
|
+
end
|
|
1622
|
+
|
|
1623
|
+
# Visual live-view recording of a session — /v1/recordings/{session_id}/visual/*.
|
|
1624
|
+
# Start/stop a frame stream and pull captured frames. Each delegates through the
|
|
1625
|
+
# shared authenticated adapter.
|
|
1626
|
+
class VisualClient
|
|
1627
|
+
def initialize(session_id, adapter = nil)
|
|
1628
|
+
@session_id = session_id
|
|
1629
|
+
@adapter = adapter
|
|
1630
|
+
end
|
|
1631
|
+
|
|
1632
|
+
# Get the visual-recording status for the session.
|
|
1633
|
+
# Delegates to GET /v1/recordings/{session_id}/visual.
|
|
1634
|
+
def get
|
|
1635
|
+
ResponseHelper.json_request!(@adapter, :GET, "/v1/recordings/#{@session_id}/visual", nil)
|
|
920
1636
|
end
|
|
1637
|
+
|
|
1638
|
+
# Start capturing visual frames for the session.
|
|
1639
|
+
# Delegates to POST /v1/recordings/{session_id}/visual/start.
|
|
1640
|
+
def start(**opts)
|
|
1641
|
+
ResponseHelper.json_request!(@adapter, :POST, "/v1/recordings/#{@session_id}/visual/start",
|
|
1642
|
+
opts.transform_keys(&:to_s))
|
|
1643
|
+
end
|
|
1644
|
+
|
|
1645
|
+
# Stop capturing visual frames for the session.
|
|
1646
|
+
# Delegates to POST /v1/recordings/{session_id}/visual/stop.
|
|
1647
|
+
def stop(**opts)
|
|
1648
|
+
ResponseHelper.json_request!(@adapter, :POST, "/v1/recordings/#{@session_id}/visual/stop",
|
|
1649
|
+
opts.transform_keys(&:to_s))
|
|
1650
|
+
end
|
|
1651
|
+
|
|
1652
|
+
# Fetch captured visual frames for the session.
|
|
1653
|
+
# Delegates to GET /v1/recordings/{session_id}/visual/frames.
|
|
1654
|
+
def frames(**opts)
|
|
1655
|
+
qs = opts.empty? ? "" : "?#{URI.encode_www_form(opts)}"
|
|
1656
|
+
ResponseHelper.json_request!(@adapter, :GET, "/v1/recordings/#{@session_id}/visual/frames#{qs}", nil)
|
|
1657
|
+
end
|
|
1658
|
+
|
|
1659
|
+
# Convenience live-view watcher — starts the stream then returns the current
|
|
1660
|
+
# frames handle. Mirrors the +watch()+/+live_view()+ helper in the other SDKs.
|
|
1661
|
+
def watch(**opts)
|
|
1662
|
+
start(**opts)
|
|
1663
|
+
frames
|
|
1664
|
+
end
|
|
1665
|
+
alias live_view watch
|
|
921
1666
|
end
|
|
922
1667
|
|
|
923
1668
|
# Key-value store — /v1/kv.
|
|
@@ -989,7 +1734,24 @@ module GhostCrawl
|
|
|
989
1734
|
auth_provider = MicrosoftKiotaAbstractions::BaseBearerTokenAuthenticationProvider.new(
|
|
990
1735
|
StaticTokenProvider.new(resolved_token)
|
|
991
1736
|
)
|
|
992
|
-
|
|
1737
|
+
|
|
1738
|
+
# Branded User-Agent — set via the Kiota Faraday UserAgentHandler
|
|
1739
|
+
# middleware, NOT a hand-rolled per-request header. A UserAgentOption carrying
|
|
1740
|
+
# productName "ghostcrawl-ruby" + the package VERSION is registered on the
|
|
1741
|
+
# default middleware chain (which includes UserAgentHandler), so every request
|
|
1742
|
+
# this adapter sends carries `User-Agent: ghostcrawl-ruby/<version>`.
|
|
1743
|
+
ua_option = MicrosoftKiotaFaraday::Middleware::UserAgentOption.new
|
|
1744
|
+
ua_option.product_name = "ghostcrawl-ruby"
|
|
1745
|
+
ua_option.product_version = GhostCrawl::VERSION
|
|
1746
|
+
http_client = MicrosoftKiotaFaraday::KiotaClientFactory.get_default_http_client(nil, [ua_option])
|
|
1747
|
+
|
|
1748
|
+
# Use the body-capturing subclass so a non-2xx on a modeled builder path
|
|
1749
|
+
# surfaces the correct typed error with status + body (190-08 fix), instead
|
|
1750
|
+
# of the runtime's status-less TypeError on an unmapped 4XX.
|
|
1751
|
+
adapter = BodyCapturingFaradayAdapter.new(auth_provider)
|
|
1752
|
+
# Swap in the UA-branded Faraday client (default middleware chain + UA option)
|
|
1753
|
+
# so the UserAgentHandler stamps ghostcrawl-ruby/<version> on every request.
|
|
1754
|
+
adapter.client = http_client
|
|
993
1755
|
adapter.set_base_url(resolved_base)
|
|
994
1756
|
|
|
995
1757
|
# The default Faraday connection sets no timeout, so it inherits net/http's
|
|
@@ -1001,6 +1763,10 @@ module GhostCrawl
|
|
|
1001
1763
|
|
|
1002
1764
|
@core = GhostCrawl::GhostCrawlClient.new(adapter)
|
|
1003
1765
|
@v1 = @core.v1
|
|
1766
|
+
# Kept for the raw-dispatch paths that need the base URL / bearer token
|
|
1767
|
+
# directly (the search(wait:) client-side poll).
|
|
1768
|
+
@base_url = resolved_base
|
|
1769
|
+
@token = resolved_token
|
|
1004
1770
|
# Kept for the raw-adapter dispatch paths (void DELETEs, binary PDF/
|
|
1005
1771
|
# screenshot, the long-poll crawl-run wait, and the ad-hoc JSON routes
|
|
1006
1772
|
# the generated core has no builder for).
|
|
@@ -1009,7 +1775,7 @@ module GhostCrawl
|
|
|
1009
1775
|
# CrawlRunsClient needs the raw adapter for the event-driven long-poll wait
|
|
1010
1776
|
# (the generated item builder can't attach ?wait/&timeout_s query params).
|
|
1011
1777
|
@crawl_runs = CrawlRunsClient.new(@v1, @adapter)
|
|
1012
|
-
@sessions = SessionsClient.new(@v1)
|
|
1778
|
+
@sessions = SessionsClient.new(@v1, @adapter)
|
|
1013
1779
|
# Sub-clients with void-DELETE endpoints (204 No Content) also need the raw
|
|
1014
1780
|
# adapter so ResponseHelper.void_request! can work around the broken generated
|
|
1015
1781
|
# delete builders. See ResponseHelper.void_request! for the two Kiota defects.
|
|
@@ -1019,6 +1785,12 @@ module GhostCrawl
|
|
|
1019
1785
|
@datasets = DatasetsClient.new(@v1)
|
|
1020
1786
|
@recordings = RecordingsClient.new(@v1, @adapter)
|
|
1021
1787
|
@kv = KVClient.new(@v1)
|
|
1788
|
+
# Full customer-surface sub-clients — cdp/page/downloads/storage-states
|
|
1789
|
+
# primitives, each delegating through the shared authenticated adapter.
|
|
1790
|
+
@cdp = CdpClient.new(@v1, @adapter)
|
|
1791
|
+
@page = PageClient.new(@v1, @adapter)
|
|
1792
|
+
@downloads = DownloadsClient.new(@v1, @adapter)
|
|
1793
|
+
@storage_states_client = StorageStatesClient.new(@v1, @adapter)
|
|
1022
1794
|
end
|
|
1023
1795
|
|
|
1024
1796
|
# @return [CrawlRunsClient]
|
|
@@ -1037,6 +1809,12 @@ module GhostCrawl
|
|
|
1037
1809
|
attr_reader :recordings
|
|
1038
1810
|
# @return [KVClient]
|
|
1039
1811
|
attr_reader :kv
|
|
1812
|
+
# @return [CdpClient] low-level CDP primitives (/v1/cdp/*)
|
|
1813
|
+
attr_reader :cdp
|
|
1814
|
+
# @return [PageClient] page-level primitives against a live session (/v1/page/*)
|
|
1815
|
+
attr_reader :page
|
|
1816
|
+
# @return [DownloadsClient] crawl download artifacts (/v1/downloads)
|
|
1817
|
+
attr_reader :downloads
|
|
1040
1818
|
|
|
1041
1819
|
# ---------------------------------------------------------------------------
|
|
1042
1820
|
# Top-level facade methods — delegate to generated builders
|
|
@@ -1062,7 +1840,10 @@ module GhostCrawl
|
|
|
1062
1840
|
data["extract_schema"] = extract_schema unless extract_schema.nil?
|
|
1063
1841
|
hash = ResponseHelper.to_hash(@v1.scrape.post(AdditionalDataBody.new(data)))
|
|
1064
1842
|
normalize_scrape_content(hash)
|
|
1065
|
-
raise_on_result_error ? ResponseHelper.raise_on_result_error!(hash) : hash
|
|
1843
|
+
result = raise_on_result_error ? ResponseHelper.raise_on_result_error!(hash) : hash
|
|
1844
|
+
# Wrap in the typed-but-hash-accessible ScrapeResult — the raw
|
|
1845
|
+
# generated builder never reaches the user surface.
|
|
1846
|
+
ScrapeResult.wrap(result)
|
|
1066
1847
|
end
|
|
1067
1848
|
|
|
1068
1849
|
# Search the web and return results.
|
|
@@ -1072,22 +1853,74 @@ module GhostCrawl
|
|
|
1072
1853
|
# charges no markup). Pass it as +provider_key+ — it is sent as the
|
|
1073
1854
|
# +X-Provider-Authorization: Bearer <provider_key>+ header the backend
|
|
1074
1855
|
# requires. Without it the API replies 401 search_backend_key_missing.
|
|
1856
|
+
# Returns a typed {SearchResult} — read +result.results+ /
|
|
1857
|
+
# +result["results"]+ / +result[:results]+ interchangeably.
|
|
1858
|
+
#
|
|
1859
|
+
# +search+ runs on two paths. On a WARM rotation the backend returns the
|
|
1860
|
+
# results synchronously (HTTP 200) and this returns immediately. On a COLD
|
|
1861
|
+
# rotation it returns +202 running+ with a +poll_url+; with +wait: true+ (the
|
|
1862
|
+
# default, matching +crawl+'s DX) the SDK owns a BOUNDED client-side poll of
|
|
1863
|
+
# +/v1/search/jobs/{id}+ that honours +Retry-After+ and stops at the server's
|
|
1864
|
+
# +poll_max_seconds+ / an explicit +timeout+ / a 404 — so you write
|
|
1865
|
+
# ZERO poll code. Unlike crawl's SERVER-side long-poll, search wait is a
|
|
1866
|
+
# CLIENT-side poll (the search-job contract has no +?wait=true+).
|
|
1867
|
+
#
|
|
1075
1868
|
# @param query [String] search query
|
|
1076
1869
|
# @param engine [String] search engine: "google" (default), "bing", "duckduckgo"
|
|
1077
1870
|
# @param limit [Integer] maximum results (default 10)
|
|
1078
1871
|
# @param provider_key [String, nil] BYO search-backend key (sent as X-Provider-Authorization)
|
|
1079
|
-
# @
|
|
1080
|
-
|
|
1872
|
+
# @param wait [Boolean] block until the search job is terminal via the SDK-owned
|
|
1873
|
+
# bounded poll (default true). When false, a warm 200 returns the results and a
|
|
1874
|
+
# cold 202 returns the raw job/poll handle for advanced callers.
|
|
1875
|
+
# @param timeout [Numeric, nil] overall client-side wait budget in seconds
|
|
1876
|
+
# (+wait: true+ only). Defaults to the server's +poll_max_seconds+, else 300.
|
|
1877
|
+
# @return [SearchResult] typed, hash-accessible result with +results+ / +query+ / +status+
|
|
1878
|
+
def search(query:, engine: "google", limit: 10, provider_key: nil, wait: true, timeout: nil, **opts)
|
|
1081
1879
|
data = { "query" => query, "engine" => engine,
|
|
1082
1880
|
"limit" => limit }.merge(opts.transform_keys(&:to_s))
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1881
|
+
|
|
1882
|
+
resp = ResponseHelper.raw_response!(@adapter, search_request_info(data, provider_key))
|
|
1883
|
+
status = resp.status
|
|
1884
|
+
job = parse_json_body(resp.body)
|
|
1885
|
+
|
|
1886
|
+
# Warm path: results returned synchronously.
|
|
1887
|
+
return SearchResult.wrap(finalize_search(job, query, engine)) if status == 200
|
|
1888
|
+
|
|
1889
|
+
# Anything other than a cold 202 that is < 400 falls through as-is; a
|
|
1890
|
+
# >= 400 status raises the documented typed error.
|
|
1891
|
+
unless status == 202
|
|
1892
|
+
ResponseHelper.raise_from_response!(resp)
|
|
1893
|
+
return SearchResult.wrap(finalize_search(job, query, engine))
|
|
1089
1894
|
end
|
|
1090
|
-
|
|
1895
|
+
|
|
1896
|
+
# Cold path: 202 running with a poll handle.
|
|
1897
|
+
return SearchResult.wrap(finalize_search(job, query, engine)) unless wait
|
|
1898
|
+
|
|
1899
|
+
initial_retry = SearchPoll.parse_retry_after(resp.headers["Retry-After"] || resp.headers["retry-after"])
|
|
1900
|
+
terminal = SearchPoll.poll_search_job(
|
|
1901
|
+
job,
|
|
1902
|
+
base_url: @base_url,
|
|
1903
|
+
get: search_poll_getter(provider_key),
|
|
1904
|
+
timeout: timeout,
|
|
1905
|
+
initial_retry: initial_retry
|
|
1906
|
+
)
|
|
1907
|
+
# A timed-out poll returns the raw running handle (no +query+/+items+); the
|
|
1908
|
+
# finalize injects the caller's +query+/+engine+ so the search result carries
|
|
1909
|
+
# a STABLE shape on EVERY path (warm 200, wait:false handle, timed-out poll).
|
|
1910
|
+
SearchResult.wrap(finalize_search(terminal, query, engine))
|
|
1911
|
+
end
|
|
1912
|
+
|
|
1913
|
+
# Guarantee a stable +search+ result shape: alias +items+ -> +results+ (default
|
|
1914
|
+
# +[]+), inject the caller's +query+ when the payload omits it (only the SDK
|
|
1915
|
+
# knows it — a bare running handle has none), and default +engine+. Mirrors the
|
|
1916
|
+
# Python +_finalize_search+ facade helper. The model's +SearchResult.wrap+ adds
|
|
1917
|
+
# the final key guarantees.
|
|
1918
|
+
def finalize_search(data, query, engine)
|
|
1919
|
+
d = data.is_a?(Hash) ? data.dup : {}
|
|
1920
|
+
d["results"] = (d["items"].is_a?(Array) ? d["items"] : []) if d["results"].nil?
|
|
1921
|
+
d["query"] = query if d["query"].nil? || d["query"].to_s.empty?
|
|
1922
|
+
d["engine"] = engine unless d.key?("engine")
|
|
1923
|
+
d
|
|
1091
1924
|
end
|
|
1092
1925
|
|
|
1093
1926
|
# Extract structured data from a URL using a JSON Schema.
|
|
@@ -1127,12 +1960,14 @@ module GhostCrawl
|
|
|
1127
1960
|
hash = ResponseHelper.to_hash(@v1.crawl.deep.post(AdditionalDataBody.new(data)))
|
|
1128
1961
|
|
|
1129
1962
|
unless wait
|
|
1130
|
-
|
|
1963
|
+
result = raise_on_result_error ? ResponseHelper.raise_on_result_error!(hash) : hash
|
|
1964
|
+
return CrawlRun.wrap(result)
|
|
1131
1965
|
end
|
|
1132
1966
|
|
|
1133
1967
|
run_id = hash["run_id"]
|
|
1134
|
-
return hash if run_id.nil? || run_id.to_s.empty?
|
|
1135
|
-
|
|
1968
|
+
return CrawlRun.wrap(hash) if run_id.nil? || run_id.to_s.empty?
|
|
1969
|
+
|
|
1970
|
+
CrawlRun.wrap(@crawl_runs.wait_for_completion(run_id, timeout: timeout))
|
|
1136
1971
|
end
|
|
1137
1972
|
|
|
1138
1973
|
# Map all URLs reachable from a seed URL.
|
|
@@ -1230,7 +2065,7 @@ module GhostCrawl
|
|
|
1230
2065
|
request_info.headers.try_add("Accept", "application/json")
|
|
1231
2066
|
request_info.set_stream_content(JSON.generate(data), "application/json")
|
|
1232
2067
|
bytes = ResponseHelper.binary_request!(@adapter, request_info)
|
|
1233
|
-
parsed =
|
|
2068
|
+
parsed = ResponseHelper.parse_json!(bytes)
|
|
1234
2069
|
parsed.is_a?(Hash) ? parsed : { "content" => parsed }
|
|
1235
2070
|
end
|
|
1236
2071
|
|
|
@@ -1259,7 +2094,7 @@ module GhostCrawl
|
|
|
1259
2094
|
request_info.headers.try_add("Accept", "application/json")
|
|
1260
2095
|
request_info.set_stream_content(JSON.generate(data), "application/json")
|
|
1261
2096
|
bytes = ResponseHelper.binary_request!(@adapter, request_info)
|
|
1262
|
-
parsed =
|
|
2097
|
+
parsed = ResponseHelper.parse_json!(bytes)
|
|
1263
2098
|
parsed.is_a?(Hash) ? parsed : { "detail" => parsed }
|
|
1264
2099
|
rescue GhostCrawl::GhostCrawlError => e
|
|
1265
2100
|
# Agent is account-gated: a 404 not_found is the documented "capability
|
|
@@ -1267,6 +2102,7 @@ module GhostCrawl
|
|
|
1267
2102
|
# as a Hash (carrying "detail") rather than raising. Any other status is a
|
|
1268
2103
|
# real error and is re-raised.
|
|
1269
2104
|
raise unless e.status_code == 404
|
|
2105
|
+
|
|
1270
2106
|
body = e.body.to_s
|
|
1271
2107
|
brace = body.index("{")
|
|
1272
2108
|
decoded =
|
|
@@ -1277,7 +2113,7 @@ module GhostCrawl
|
|
|
1277
2113
|
nil
|
|
1278
2114
|
end
|
|
1279
2115
|
end
|
|
1280
|
-
decoded.is_a?(Hash) ? decoded : { "detail" => e.message, "code" =>
|
|
2116
|
+
decoded.is_a?(Hash) ? decoded : { "detail" => e.message, "code" => e.code || "not_found" }
|
|
1281
2117
|
end
|
|
1282
2118
|
|
|
1283
2119
|
# List the account's persisted session storage-states.
|
|
@@ -1287,6 +2123,75 @@ module GhostCrawl
|
|
|
1287
2123
|
json_request(:GET, "/v1/storage-states", nil)
|
|
1288
2124
|
end
|
|
1289
2125
|
|
|
2126
|
+
# Attach a persisted storage-state to a session/profile.
|
|
2127
|
+
# Delegates to POST /v1/storage-states/{id_or_name}/attach.
|
|
2128
|
+
# @param id_or_name [String] the storage-state id or name
|
|
2129
|
+
def storage_states_attach(id_or_name, **opts)
|
|
2130
|
+
@storage_states_client.attach(id_or_name, **opts)
|
|
2131
|
+
end
|
|
2132
|
+
|
|
2133
|
+
# Detach a persisted storage-state.
|
|
2134
|
+
# Delegates to POST /v1/storage-states/detach.
|
|
2135
|
+
def storage_states_detach(**opts)
|
|
2136
|
+
@storage_states_client.detach(**opts)
|
|
2137
|
+
end
|
|
2138
|
+
|
|
2139
|
+
# Return the caller's account profile (user, primary team, tier).
|
|
2140
|
+
# Delegates to GET /v1/me through the shared authenticated adapter (promoted
|
|
2141
|
+
# customer surface — no generated builder, same Bearer + base URL + branded UA).
|
|
2142
|
+
# @return [Hash] the account profile envelope
|
|
2143
|
+
def me
|
|
2144
|
+
json_request(:GET, "/v1/me", nil)
|
|
2145
|
+
end
|
|
2146
|
+
|
|
2147
|
+
# Return the caller's current usage / quota (the quota-visibility surface).
|
|
2148
|
+
# Delegates to GET /v1/me/usage through the shared authenticated adapter.
|
|
2149
|
+
# @param period [String, nil] optional billing period selector
|
|
2150
|
+
# @return [Hash] the usage envelope
|
|
2151
|
+
def usage(period: nil, **opts)
|
|
2152
|
+
params = opts.transform_keys(&:to_s)
|
|
2153
|
+
params["period"] = period unless period.nil?
|
|
2154
|
+
qs = params.empty? ? "" : "?#{URI.encode_www_form(params)}"
|
|
2155
|
+
json_request(:GET, "/v1/me/usage#{qs}", nil)
|
|
2156
|
+
end
|
|
2157
|
+
|
|
2158
|
+
# Return the customer-safe persona for a running session.
|
|
2159
|
+
# Delegates to GET /v1/me/identities/{session_id}/persona through the shared
|
|
2160
|
+
# authenticated adapter. Whitelist display shape only: +device_class+,
|
|
2161
|
+
# +os_class+, +browser_class+, +locale+ — never a raw user-agent or any
|
|
2162
|
+
# internal detail. Session-scoped, distinct from the account-level +me+.
|
|
2163
|
+
# An unknown session raises a not-found error.
|
|
2164
|
+
# @param session_id [String] the running session's id
|
|
2165
|
+
# @return [Hash] the persona envelope
|
|
2166
|
+
def session_persona(session_id)
|
|
2167
|
+
json_request(:GET, "/v1/me/identities/#{URI.encode_www_form_component(session_id)}/persona", nil)
|
|
2168
|
+
end
|
|
2169
|
+
|
|
2170
|
+
# Scrape many URLs in a single batch request.
|
|
2171
|
+
# Delegates to POST /v1/scrape/batch through the shared authenticated adapter.
|
|
2172
|
+
# @param urls [Array<String>] the URLs to scrape
|
|
2173
|
+
# @return [Hash] the batch envelope
|
|
2174
|
+
def scrape_batch(urls:, **opts)
|
|
2175
|
+
data = { "urls" => urls }.merge(opts.transform_keys(&:to_s))
|
|
2176
|
+
json_request(:POST, "/v1/scrape/batch", data)
|
|
2177
|
+
end
|
|
2178
|
+
|
|
2179
|
+
# Poll a cold search job by ID (advanced; +search(wait: true)+ owns this loop
|
|
2180
|
+
# for you). Delegates to GET /v1/search/jobs/{job_id}.
|
|
2181
|
+
# @param job_id [String] the search-job id
|
|
2182
|
+
# @return [Hash] the job envelope
|
|
2183
|
+
def search_job(job_id)
|
|
2184
|
+
json_request(:GET, "/v1/search/jobs/#{job_id}", nil)
|
|
2185
|
+
end
|
|
2186
|
+
|
|
2187
|
+
# Get the status of a deep-crawl run by run_id.
|
|
2188
|
+
# Delegates to GET /v1/crawl/deep/{run_id}.
|
|
2189
|
+
# @param run_id [String] the crawl run id
|
|
2190
|
+
# @return [CrawlRun] the run status record
|
|
2191
|
+
def crawl_status(run_id)
|
|
2192
|
+
CrawlRun.wrap(json_request(:GET, "/v1/crawl/deep/#{run_id}", nil))
|
|
2193
|
+
end
|
|
2194
|
+
|
|
1290
2195
|
private
|
|
1291
2196
|
|
|
1292
2197
|
# Route an ad-hoc JSON request through the SAME adapter (base URL + bearer
|
|
@@ -1298,11 +2203,61 @@ module GhostCrawl
|
|
|
1298
2203
|
request_info.http_method = method
|
|
1299
2204
|
request_info.url_template = "{+baseurl}#{path}"
|
|
1300
2205
|
request_info.headers.try_add("Accept", "application/json")
|
|
1301
|
-
if !body.nil? && method != :GET
|
|
1302
|
-
request_info.set_stream_content(JSON.generate(body), "application/json")
|
|
1303
|
-
end
|
|
2206
|
+
request_info.set_stream_content(JSON.generate(body), "application/json") if !body.nil? && method != :GET
|
|
1304
2207
|
bytes = ResponseHelper.binary_request!(@adapter, request_info)
|
|
1305
|
-
parsed =
|
|
2208
|
+
parsed = ResponseHelper.parse_json!(bytes)
|
|
2209
|
+
parsed.is_a?(Hash) ? parsed : {}
|
|
2210
|
+
end
|
|
2211
|
+
|
|
2212
|
+
# Builds the +POST /v1/search+ request info. Routed through the raw adapter
|
|
2213
|
+
# dispatch (not the generated builder) so +search+ can read the response
|
|
2214
|
+
# STATUS (warm 200 vs cold 202) and the +Retry-After+ header the poll
|
|
2215
|
+
# needs. The optional BYO search-backend key rides as +X-Provider-Authorization+.
|
|
2216
|
+
# @api private
|
|
2217
|
+
def search_request_info(data, provider_key)
|
|
2218
|
+
request_info = MicrosoftKiotaAbstractions::RequestInformation.new
|
|
2219
|
+
request_info.http_method = :POST
|
|
2220
|
+
request_info.url_template = "{+baseurl}/v1/search"
|
|
2221
|
+
request_info.headers.try_add("Accept", "application/json")
|
|
2222
|
+
request_info.headers.try_add("X-Provider-Authorization", "Bearer #{provider_key}") unless provider_key.nil?
|
|
2223
|
+
request_info.set_stream_content(JSON.generate(data), "application/json")
|
|
2224
|
+
request_info
|
|
2225
|
+
end
|
|
2226
|
+
|
|
2227
|
+
# Returns a +->(url) { [status, headers, body_hash] }+ getter that issues one
|
|
2228
|
+
# search-job poll GET through the SAME adapter pipeline (base URL + bearer auth)
|
|
2229
|
+
# as the modeled calls, carrying the optional BYO provider key. Handed to
|
|
2230
|
+
# {SearchPoll.poll_search_job}.
|
|
2231
|
+
# @api private
|
|
2232
|
+
def search_poll_getter(provider_key)
|
|
2233
|
+
lambda do |url|
|
|
2234
|
+
request_info = MicrosoftKiotaAbstractions::RequestInformation.new
|
|
2235
|
+
request_info.http_method = :GET
|
|
2236
|
+
request_info.url_template =
|
|
2237
|
+
if url.start_with?("http://", "https://")
|
|
2238
|
+
url
|
|
2239
|
+
else
|
|
2240
|
+
"{+baseurl}#{url.start_with?("/") ? url : "/#{url}"}"
|
|
2241
|
+
end
|
|
2242
|
+
request_info.headers.try_add("Accept", "application/json")
|
|
2243
|
+
request_info.headers.try_add("X-Provider-Authorization", "Bearer #{provider_key}") unless provider_key.nil?
|
|
2244
|
+
resp = ResponseHelper.raw_response!(@adapter, request_info)
|
|
2245
|
+
[resp.status, resp.headers, parse_json_body(resp.body)]
|
|
2246
|
+
end
|
|
2247
|
+
end
|
|
2248
|
+
|
|
2249
|
+
# Decodes a response body to a Hash, or +{}+ when it is empty / not a JSON
|
|
2250
|
+
# object. Never raises on a malformed body.
|
|
2251
|
+
# @api private
|
|
2252
|
+
def parse_json_body(body)
|
|
2253
|
+
text = body.to_s
|
|
2254
|
+
return {} if text.strip.empty?
|
|
2255
|
+
|
|
2256
|
+
parsed = begin
|
|
2257
|
+
JSON.parse(text)
|
|
2258
|
+
rescue JSON::ParserError
|
|
2259
|
+
nil
|
|
2260
|
+
end
|
|
1306
2261
|
parsed.is_a?(Hash) ? parsed : {}
|
|
1307
2262
|
end
|
|
1308
2263
|
|