ghostcrawl 2.2.3 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/LICENSE +21 -46
- data/README.md +40 -4
- data/_generated/v1/binary/binary.rb +9 -0
- data/_generated/v1/v1_request_builder.rb +1 -6
- data/lib/ghostcrawl/client.rb +170 -28
- data/lib/ghostcrawl/version.rb +1 -1
- metadata +3 -6
- data/_generated/v1/binary/binary_request_builder.rb +0 -31
- data/_generated/v1/binary/download/download.rb +0 -0
- data/_generated/v1/binary/download/download_request_builder.rb +0 -77
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 482db5d9b31ab1b7293fc19b6834be23062a37d6ef8632fae462e369e5dfc244
|
|
4
|
+
data.tar.gz: 01eababfa0e5cd0ecc5ca39386c4b7a8cdbe2d8f0ff7f079382c89fed5859c9c
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 866336d0c4c835f6ccaafd9a444c03eb4931b97cbf4b65ca003b01ff5d374a68b06500542805951ee25c3276adf64b0d16c433a956f0dbbe8bebd91ef10e0211
|
|
7
|
+
data.tar.gz: 250b128f21670a470961468a6da62c1647495d7b4065a6d1fbe4f13328ac680d7b40ebc6acfd68338e88fa9f8bbfb99ec88ea0d5f6f6ebb56cff56261800ea41
|
data/LICENSE
CHANGED
|
@@ -1,46 +1,21 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
Copyright (c) 2026 GhostCrawl, LLC
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
party, including as a hosted or managed service; (d) reverse engineer,
|
|
23
|
-
decompile, disassemble, or otherwise attempt to derive the source code,
|
|
24
|
-
structure, or internal design of any compiled component of the Software;
|
|
25
|
-
(e) circumvent, disable, or tamper with any license, authentication,
|
|
26
|
-
usage-metering, watermarking, or security mechanism; or (f) remove, alter, or
|
|
27
|
-
obscure any proprietary, copyright, or attribution notice.
|
|
28
|
-
|
|
29
|
-
3. OWNERSHIP. GhostCrawl and its licensors retain all right, title, and interest
|
|
30
|
-
in and to the Software and all intellectual property rights therein. No rights
|
|
31
|
-
are granted to you except as expressly set out in this License. All rights
|
|
32
|
-
reserved.
|
|
33
|
-
|
|
34
|
-
4. TERMINATION. This License terminates automatically, without notice, if you
|
|
35
|
-
breach any of its terms or cease to maintain a valid GhostCrawl account where
|
|
36
|
-
one is required. Upon termination you must cease all use of the Software and
|
|
37
|
-
destroy all copies in your possession or control.
|
|
38
|
-
|
|
39
|
-
5. NO WARRANTY; LIMITATION OF LIABILITY. THE SOFTWARE IS PROVIDED "AS IS" WITHOUT
|
|
40
|
-
WARRANTY OF ANY KIND, EXPRESS OR IMPLIED. TO THE MAXIMUM EXTENT PERMITTED BY
|
|
41
|
-
LAW, GHOSTCRAWL AND ITS LICENSORS WILL NOT BE LIABLE FOR ANY CLAIM, DAMAGES, OR
|
|
42
|
-
OTHER LIABILITY ARISING FROM OR IN CONNECTION WITH THE SOFTWARE OR ITS USE. The
|
|
43
|
-
GhostCrawl Terms of Service and Acceptable Use Policy
|
|
44
|
-
(https://docs.ghostcrawl.io/legal/terms) are incorporated by reference.
|
|
45
|
-
|
|
46
|
-
For licensing inquiries, contact legal@ghostcrawl.io.
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 GhostCrawl, LLC
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
data/README.md
CHANGED
|
@@ -29,9 +29,11 @@ client = Ghostcrawl::Client.new(token: "gck_live_YOUR_KEY")
|
|
|
29
29
|
result = client.scrape(url: "https://example.com", format: "markdown")
|
|
30
30
|
puts result["content"]
|
|
31
31
|
|
|
32
|
-
# Start a crawl run
|
|
33
|
-
|
|
34
|
-
|
|
32
|
+
# Start a crawl run and wait for it to finish — the server blocks until the run
|
|
33
|
+
# is terminal (no client-side poll-with-sleep loop).
|
|
34
|
+
run = client.crawl_runs.start(url: "https://example.com", max_depth: 2, max_pages: 50,
|
|
35
|
+
wait: true, timeout: 300)
|
|
36
|
+
puts "#{run["run_id"]} → #{run["status"]}"
|
|
35
37
|
|
|
36
38
|
# Web search
|
|
37
39
|
results = client.search(query: "latest AI research", engine: "google", limit: 10)
|
|
@@ -72,6 +74,40 @@ data = client.extract(
|
|
|
72
74
|
puts "#{data["name"]} — $#{data["price"]}"
|
|
73
75
|
```
|
|
74
76
|
|
|
77
|
+
## Crawl runs — wait for completion
|
|
78
|
+
|
|
79
|
+
Crawl runs are asynchronous. Instead of hand-writing a poll-with-sleep loop, use
|
|
80
|
+
the event-driven wait API: the **server** blocks until the run reaches a terminal
|
|
81
|
+
state (`completed` / `failed` / `cancelled`) and returns the finished run.
|
|
82
|
+
|
|
83
|
+
```ruby
|
|
84
|
+
# Start and wait in one call — sends wait_until: "completed"
|
|
85
|
+
run = client.crawl_runs.start(
|
|
86
|
+
url: "https://example.com",
|
|
87
|
+
max_depth: 2,
|
|
88
|
+
max_pages: 50,
|
|
89
|
+
wait: true, # block until terminal (no client sleep loop)
|
|
90
|
+
timeout: 300 # give up after 300s, returning the still-running run
|
|
91
|
+
)
|
|
92
|
+
puts run["status"] # "completed" | "failed" | "cancelled"
|
|
93
|
+
|
|
94
|
+
# Or wait on a run you already started (e.g. from a webhook or another process):
|
|
95
|
+
run = client.crawl_runs.wait_for_completion(run_id, timeout: 300)
|
|
96
|
+
|
|
97
|
+
# Deep crawl supports the same wait: keyword:
|
|
98
|
+
run = client.crawl(url: "https://example.com", wait: true, timeout: 300)
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
Under the hood `wait_for_completion` chains server-blocking
|
|
102
|
+
`GET /v1/crawl-runs/{run_id}?wait=true&timeout_s=N` requests until the run is
|
|
103
|
+
terminal or your `timeout` deadline — there is no `sleep` between checks. On
|
|
104
|
+
timeout the current non-terminal run is returned, so you can simply call
|
|
105
|
+
`wait_for_completion` again to keep waiting.
|
|
106
|
+
|
|
107
|
+
**Prefer webhooks for fire-and-forget crawls.** Register an endpoint with
|
|
108
|
+
`client.webhooks.create(url:, event_types: ["crawl.completed"])` and let GhostCrawl
|
|
109
|
+
POST the result to you instead of holding a connection open.
|
|
110
|
+
|
|
75
111
|
## Error handling
|
|
76
112
|
|
|
77
113
|
```ruby
|
|
@@ -113,7 +149,7 @@ client = Ghostcrawl::Client.new(
|
|
|
113
149
|
| Deep crawl | `client.crawl(url:)` | Crawl a site depth-first |
|
|
114
150
|
| URL map | `client.map(url:)` | Discover all reachable URLs |
|
|
115
151
|
| Account | `client.me` | Get account info and usage |
|
|
116
|
-
| Crawl runs | `client.crawl_runs` | start, list, get
|
|
152
|
+
| Crawl runs | `client.crawl_runs` | start (with `wait:`), wait\_for\_completion, list, get, cancel |
|
|
117
153
|
| Sessions | `client.sessions` | create, list, extend, release |
|
|
118
154
|
| Profiles | `client.profiles` | list, get\_profile, create, update, delete |
|
|
119
155
|
| Webhooks | `client.webhooks` | list, get\_webhook, create, delete, rotate\_secret |
|
|
@@ -2,7 +2,7 @@ require 'microsoft_kiota_abstractions'
|
|
|
2
2
|
require_relative '../ghostcrawl'
|
|
3
3
|
require_relative './audit/audit_request_builder'
|
|
4
4
|
require_relative './billing/billing_request_builder'
|
|
5
|
-
require_relative './binary/
|
|
5
|
+
require_relative './binary/binary'
|
|
6
6
|
require_relative './budgets/budgets_request_builder'
|
|
7
7
|
require_relative './cdp/cdp_request_builder'
|
|
8
8
|
require_relative './contact/contact_request_builder'
|
|
@@ -50,11 +50,6 @@ module Ghostcrawl
|
|
|
50
50
|
return Ghostcrawl::V1::Billing::BillingRequestBuilder.new(@path_parameters, @request_adapter)
|
|
51
51
|
end
|
|
52
52
|
##
|
|
53
|
-
# The binary property
|
|
54
|
-
def binary()
|
|
55
|
-
return Ghostcrawl::V1::Binary::BinaryRequestBuilder.new(@path_parameters, @request_adapter)
|
|
56
|
-
end
|
|
57
|
-
##
|
|
58
53
|
# The budgets property
|
|
59
54
|
def budgets()
|
|
60
55
|
return Ghostcrawl::V1::Budgets::BudgetsRequestBuilder.new(@path_parameters, @request_adapter)
|
data/lib/ghostcrawl/client.rb
CHANGED
|
@@ -463,26 +463,101 @@ module Ghostcrawl
|
|
|
463
463
|
|
|
464
464
|
# Manage crawl runs — /v1/crawl-runs.
|
|
465
465
|
class CrawlRunsClient
|
|
466
|
-
|
|
466
|
+
# Terminal run states — a crawl run in any of these will never change again,
|
|
467
|
+
# so a wait can stop. Both spellings of "cancelled" are accepted.
|
|
468
|
+
TERMINAL_STATUSES = %w[completed failed cancelled canceled].freeze
|
|
469
|
+
|
|
470
|
+
# Per-request server-block window (seconds) for the long-poll wait. Each
|
|
471
|
+
# +GET ...?wait=true&timeout_s=WAIT_WINDOW_S+ makes the SERVER block for up
|
|
472
|
+
# to this long, so the client never sleeps between checks. Kept well under
|
|
473
|
+
# the client read timeout ({Client::DEFAULT_TIMEOUT}) so a single blocking
|
|
474
|
+
# request can never trip the transport timeout.
|
|
475
|
+
WAIT_WINDOW_S = 30
|
|
476
|
+
|
|
477
|
+
def initialize(v1, adapter = nil)
|
|
467
478
|
@v1 = v1
|
|
479
|
+
@adapter = adapter
|
|
468
480
|
end
|
|
469
481
|
|
|
470
482
|
# Start a new crawl run from a seed URL.
|
|
471
483
|
# Delegates to POST /v1/crawl-runs via the generated CrawlRunsRequestBuilder.
|
|
472
484
|
# The endpoint is a tagged union: a start request requires +action: "start"+
|
|
473
485
|
# and a +seed_urls+ array (not a bare +url+).
|
|
486
|
+
#
|
|
487
|
+
# Pass +wait: true+ to block until the run reaches a terminal state
|
|
488
|
+
# (completed | failed | cancelled) or +timeout+ seconds elapse. This sends
|
|
489
|
+
# +wait_until: "completed"+ so the SERVER blocks — no client-side poll loop,
|
|
490
|
+
# no +sleep+. When the run is still running at +timeout+ the current
|
|
491
|
+
# (non-terminal) record is returned; call {#wait_for_completion} again to
|
|
492
|
+
# keep waiting.
|
|
493
|
+
#
|
|
474
494
|
# @param url [String] seed URL
|
|
475
495
|
# @param max_depth [Integer] maximum crawl depth (default 2)
|
|
476
496
|
# @param max_pages [Integer] maximum pages (default 100)
|
|
497
|
+
# @param wait [Boolean] block until the run is terminal (default false)
|
|
498
|
+
# @param timeout [Integer] total seconds to wait when +wait: true+ (default 300)
|
|
477
499
|
# @param raise_on_result_error [Boolean] raise {Ghostcrawl::ScrapeError} on a
|
|
478
|
-
# target-side (HTTP-200) failure instead of returning the raw hash (default true)
|
|
479
|
-
#
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
500
|
+
# target-side (HTTP-200) failure instead of returning the raw hash (default true).
|
|
501
|
+
# Ignored on the +wait: true+ path, which always returns the terminal run so
|
|
502
|
+
# the caller can inspect a +failed+ status.
|
|
503
|
+
# @return [Hash] crawl run record with +run_id+ and +status+ (results present
|
|
504
|
+
# when it completed while waiting)
|
|
505
|
+
def start(url:, max_depth: 2, max_pages: 100, wait: false, timeout: 300,
|
|
506
|
+
raise_on_result_error: true, **opts)
|
|
507
|
+
data = { "action" => "start", "seed_urls" => [url],
|
|
508
|
+
"max_depth" => max_depth, "max_pages" => max_pages }
|
|
509
|
+
.merge(opts.transform_keys(&:to_s))
|
|
510
|
+
|
|
511
|
+
unless wait
|
|
512
|
+
hash = ResponseHelper.to_hash(@v1.crawl_runs.post(AdditionalDataBody.new(data)))
|
|
513
|
+
return raise_on_result_error ? ResponseHelper.raise_on_result_error!(hash) : hash
|
|
514
|
+
end
|
|
515
|
+
|
|
516
|
+
# Start-and-wait: ask the server to block until terminal. Bound the first
|
|
517
|
+
# server block to a safe window, then long-poll the rest via GET so a low
|
|
518
|
+
# per-request read timeout can never abort the whole wait.
|
|
519
|
+
window = [timeout, WAIT_WINDOW_S].min
|
|
520
|
+
data["wait_until"] = "completed"
|
|
521
|
+
data["timeout_s"] = window
|
|
522
|
+
hash = ResponseHelper.to_hash(@v1.crawl_runs.post(AdditionalDataBody.new(data)))
|
|
523
|
+
return hash if terminal?(hash)
|
|
524
|
+
|
|
525
|
+
run_id = hash["run_id"]
|
|
526
|
+
remaining = timeout - window
|
|
527
|
+
return hash if run_id.nil? || run_id.to_s.empty? || remaining <= 0
|
|
528
|
+
wait_for_completion(run_id, timeout: remaining)
|
|
529
|
+
end
|
|
530
|
+
|
|
531
|
+
# Block until an existing crawl run reaches a terminal state, or +timeout+
|
|
532
|
+
# seconds elapse. Event-driven: each iteration issues a SERVER-blocking
|
|
533
|
+
# +GET /v1/crawl-runs/{run_id}?wait=true&timeout_s=N+ that returns the moment
|
|
534
|
+
# the run goes terminal (or after its own window). There is no client +sleep+
|
|
535
|
+
# — the wait cost lives on the server, and successive blocking windows are
|
|
536
|
+
# chained until the caller's +timeout+ deadline.
|
|
537
|
+
#
|
|
538
|
+
# On timeout the current non-terminal run is returned (never raises for a
|
|
539
|
+
# slow run); a terminal run is returned as soon as it is observed.
|
|
540
|
+
#
|
|
541
|
+
# @param run_id [String] the run to wait on
|
|
542
|
+
# @param timeout [Integer] total seconds to wait (default 300)
|
|
543
|
+
# @return [Hash] the run record (terminal when it finished in time)
|
|
544
|
+
def wait_for_completion(run_id, timeout: 300)
|
|
545
|
+
if run_id.nil? || run_id.to_s.empty?
|
|
546
|
+
raise ArgumentError, "run_id is required"
|
|
547
|
+
end
|
|
548
|
+
|
|
549
|
+
deadline = monotonic + timeout
|
|
550
|
+
loop do
|
|
551
|
+
remaining = deadline - monotonic
|
|
552
|
+
# Deadline reached — one final non-blocking read of the current state.
|
|
553
|
+
return get(run_id) if remaining <= 0
|
|
554
|
+
|
|
555
|
+
window = [remaining, WAIT_WINDOW_S].min
|
|
556
|
+
run = fetch_waiting(run_id, window)
|
|
557
|
+
return run if terminal?(run)
|
|
558
|
+
# The server just blocked for ~window seconds; loop straight into the
|
|
559
|
+
# next blocking window. No client-side delay.
|
|
560
|
+
end
|
|
486
561
|
end
|
|
487
562
|
|
|
488
563
|
# List crawl runs.
|
|
@@ -502,6 +577,33 @@ module Ghostcrawl
|
|
|
502
577
|
def cancel(run_id)
|
|
503
578
|
ResponseHelper.to_hash(@v1.crawl_runs.by_run_id(run_id).cancel.post)
|
|
504
579
|
end
|
|
580
|
+
|
|
581
|
+
private
|
|
582
|
+
|
|
583
|
+
# True when +run+ is a Hash carrying a terminal +status+.
|
|
584
|
+
def terminal?(run)
|
|
585
|
+
run.is_a?(Hash) && TERMINAL_STATUSES.include?(run["status"].to_s)
|
|
586
|
+
end
|
|
587
|
+
|
|
588
|
+
# Issues one SERVER-blocking GET on the run with +?wait=true&timeout_s=N+.
|
|
589
|
+
#
|
|
590
|
+
# The generated item builder's +to_get_request_information+ does not wire
|
|
591
|
+
# query parameters (its URL template has no query placeholder), so we take
|
|
592
|
+
# the request it builds, extend the template with +{?wait,timeout_s}+, and
|
|
593
|
+
# set the params before sending through the raw adapter — mirroring how
|
|
594
|
+
# {Client#me} re-issues a request with the Binary (raw-JSON) response factory.
|
|
595
|
+
def fetch_waiting(run_id, timeout_s)
|
|
596
|
+
request_info = @v1.crawl_runs.by_run_id(run_id).to_get_request_information(nil)
|
|
597
|
+
request_info.url_template = "{+baseurl}/v1/crawl-runs/{run_id}{?wait,timeout_s}"
|
|
598
|
+
request_info.query_parameters["wait"] = "true"
|
|
599
|
+
request_info.query_parameters["timeout_s"] = timeout_s.to_i.to_s
|
|
600
|
+
ResponseHelper.to_hash(@adapter.send_async(request_info, Ghostcrawl::V1::Binary, {}))
|
|
601
|
+
end
|
|
602
|
+
|
|
603
|
+
# Monotonic clock (seconds) — immune to wall-clock adjustments during a wait.
|
|
604
|
+
def monotonic
|
|
605
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
606
|
+
end
|
|
505
607
|
end
|
|
506
608
|
|
|
507
609
|
# Manage browser sessions — /v1/sessions.
|
|
@@ -848,7 +950,9 @@ module Ghostcrawl
|
|
|
848
950
|
# (Binary) response factory instead of the typed MeResponse model.
|
|
849
951
|
@adapter = adapter
|
|
850
952
|
|
|
851
|
-
|
|
953
|
+
# CrawlRunsClient needs the raw adapter for the event-driven long-poll wait
|
|
954
|
+
# (the generated item builder can't attach ?wait/&timeout_s query params).
|
|
955
|
+
@crawl_runs = CrawlRunsClient.new(@v1, @adapter)
|
|
852
956
|
@sessions = SessionsClient.new(@v1)
|
|
853
957
|
# Sub-clients with void-DELETE endpoints (204 No Content) also need the raw
|
|
854
958
|
# adapter so ResponseHelper.void_request! can work around the broken generated
|
|
@@ -945,17 +1049,34 @@ module Ghostcrawl
|
|
|
945
1049
|
|
|
946
1050
|
# Start a deep crawl from a seed URL.
|
|
947
1051
|
# Delegates to POST /v1/crawl/deep via the generated CrawlDeepRequestBuilder.
|
|
1052
|
+
#
|
|
1053
|
+
# Pass +wait: true+ to block until the run is terminal. The deep-crawl start
|
|
1054
|
+
# returns a +run_id+, then the wait is handled by the same event-driven
|
|
1055
|
+
# server-blocking long-poll as {CrawlRunsClient#wait_for_completion} — no
|
|
1056
|
+
# client-side +sleep+ loop.
|
|
1057
|
+
#
|
|
948
1058
|
# @param url [String] seed URL
|
|
949
1059
|
# @param max_depth [Integer] maximum crawl depth (default 2)
|
|
950
1060
|
# @param max_pages [Integer] maximum pages (default 100)
|
|
1061
|
+
# @param wait [Boolean] block until the run is terminal (default false)
|
|
1062
|
+
# @param timeout [Integer] total seconds to wait when +wait: true+ (default 300)
|
|
951
1063
|
# @param raise_on_result_error [Boolean] raise {Ghostcrawl::ScrapeError} on a
|
|
952
|
-
# target-side (HTTP-200) failure instead of returning the raw hash (default true)
|
|
953
|
-
#
|
|
954
|
-
|
|
1064
|
+
# target-side (HTTP-200) failure instead of returning the raw hash (default true).
|
|
1065
|
+
# Ignored on the +wait: true+ path, which returns the terminal run.
|
|
1066
|
+
# @return [Hash] crawl run record (terminal when +wait: true+ and it finished in time)
|
|
1067
|
+
def crawl(url:, max_depth: 2, max_pages: 100, wait: false, timeout: 300,
|
|
1068
|
+
raise_on_result_error: true, **opts)
|
|
955
1069
|
data = { "seed_urls" => [url], "max_depth" => max_depth,
|
|
956
1070
|
"max_urls" => max_pages }.merge(opts.transform_keys(&:to_s))
|
|
957
1071
|
hash = ResponseHelper.to_hash(@v1.crawl.deep.post(AdditionalDataBody.new(data)))
|
|
958
|
-
|
|
1072
|
+
|
|
1073
|
+
unless wait
|
|
1074
|
+
return raise_on_result_error ? ResponseHelper.raise_on_result_error!(hash) : hash
|
|
1075
|
+
end
|
|
1076
|
+
|
|
1077
|
+
run_id = hash["run_id"]
|
|
1078
|
+
return hash if run_id.nil? || run_id.to_s.empty?
|
|
1079
|
+
@crawl_runs.wait_for_completion(run_id, timeout: timeout)
|
|
959
1080
|
end
|
|
960
1081
|
|
|
961
1082
|
# Map all URLs reachable from a seed URL.
|
|
@@ -986,14 +1107,22 @@ module Ghostcrawl
|
|
|
986
1107
|
|
|
987
1108
|
# Normalizes a +"content"+ key onto a decoded scrape response.
|
|
988
1109
|
#
|
|
989
|
-
# The API
|
|
990
|
-
#
|
|
991
|
-
#
|
|
992
|
-
#
|
|
1110
|
+
# The API has two scrape response shapes:
|
|
1111
|
+
# * the FLAT +format="markdown"+ shape, which carries the rendered page at
|
|
1112
|
+
# the top level under the format-specific key (+"markdown"+/+"html"+/
|
|
1113
|
+
# +"text"+) but omits +identity_id+; and
|
|
1114
|
+
# * the standard ENVELOPE ({ +status+, +results+, +routing_mode+,
|
|
1115
|
+
# +request_class+, +identity_id+ }), where the rendered page lives under
|
|
1116
|
+
# +results[0]+ and the top-level +identity_id+ is present.
|
|
1117
|
+
# The documented quickstart reads +result["content"]+, so this mirrors the
|
|
1118
|
+
# rendered page onto +"content"+ in place from WHICHEVER shape is present,
|
|
1119
|
+
# KEEPING the original keys intact (backward compatible).
|
|
993
1120
|
#
|
|
994
1121
|
# No-op unless +result+ is a Hash that does not already carry +"content"+.
|
|
995
1122
|
# The value chosen is: the field named by +result["format"]+ when that field
|
|
996
|
-
# is a String, else the first String among +"markdown"+, +"html"+, +"text"
|
|
1123
|
+
# is a String, else the first String among +"markdown"+, +"html"+, +"text"+ —
|
|
1124
|
+
# looked up first at the top level (flat shape), then on +results[0]+
|
|
1125
|
+
# (envelope shape).
|
|
997
1126
|
#
|
|
998
1127
|
# @param result [Object] the decoded response (only mutated when a Hash)
|
|
999
1128
|
# @return [Object] the same +result+, unchanged reference
|
|
@@ -1002,21 +1131,34 @@ module Ghostcrawl
|
|
|
1002
1131
|
return result unless result.is_a?(Hash) && !result.key?("content")
|
|
1003
1132
|
|
|
1004
1133
|
fmt = result["format"]
|
|
1005
|
-
value = result
|
|
1006
|
-
value = nil unless value.is_a?(String)
|
|
1134
|
+
value = scrape_content_from(result, fmt)
|
|
1007
1135
|
|
|
1136
|
+
# Envelope shape: the page is under results[0], not at the top level.
|
|
1008
1137
|
unless value.is_a?(String)
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
value = candidate
|
|
1013
|
-
break
|
|
1014
|
-
end
|
|
1015
|
-
end
|
|
1138
|
+
first = result["results"]
|
|
1139
|
+
first = first.first if first.is_a?(Array)
|
|
1140
|
+
value = scrape_content_from(first, fmt) if first.is_a?(Hash)
|
|
1016
1141
|
end
|
|
1017
1142
|
|
|
1018
1143
|
result["content"] = value if value.is_a?(String)
|
|
1019
1144
|
result
|
|
1020
1145
|
end
|
|
1146
|
+
|
|
1147
|
+
# Picks the rendered-page String out of a scrape result Hash: the field named
|
|
1148
|
+
# by +fmt+ when present, else the first String among +"markdown"+, +"html"+,
|
|
1149
|
+
# +"text"+. Returns +nil+ when +src+ carries no such String.
|
|
1150
|
+
# @api private
|
|
1151
|
+
def scrape_content_from(src, fmt)
|
|
1152
|
+
return nil unless src.is_a?(Hash)
|
|
1153
|
+
|
|
1154
|
+
value = src[fmt] if fmt.is_a?(String)
|
|
1155
|
+
return value if value.is_a?(String)
|
|
1156
|
+
|
|
1157
|
+
%w[markdown html text].each do |key|
|
|
1158
|
+
candidate = src[key]
|
|
1159
|
+
return candidate if candidate.is_a?(String)
|
|
1160
|
+
end
|
|
1161
|
+
nil
|
|
1162
|
+
end
|
|
1021
1163
|
end
|
|
1022
1164
|
end
|
data/lib/ghostcrawl/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: ghostcrawl
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 2.
|
|
4
|
+
version: 2.3.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- GhostCrawl
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-07-
|
|
11
|
+
date: 2026-07-19 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: microsoft_kiota_abstractions
|
|
@@ -225,9 +225,6 @@ files:
|
|
|
225
225
|
- _generated/v1/billing/usage/usage.rb
|
|
226
226
|
- _generated/v1/billing/usage/usage_request_builder.rb
|
|
227
227
|
- _generated/v1/binary/binary.rb
|
|
228
|
-
- _generated/v1/binary/binary_request_builder.rb
|
|
229
|
-
- _generated/v1/binary/download/download.rb
|
|
230
|
-
- _generated/v1/binary/download/download_request_builder.rb
|
|
231
228
|
- _generated/v1/budgets/budgets.rb
|
|
232
229
|
- _generated/v1/budgets/budgets_request_builder.rb
|
|
233
230
|
- _generated/v1/budgets/policy/item/item.rb
|
|
@@ -431,7 +428,7 @@ files:
|
|
|
431
428
|
- lib/ghostcrawl/version.rb
|
|
432
429
|
homepage: https://ghostcrawl.io
|
|
433
430
|
licenses:
|
|
434
|
-
-
|
|
431
|
+
- MIT
|
|
435
432
|
metadata:
|
|
436
433
|
homepage_uri: https://ghostcrawl.io
|
|
437
434
|
documentation_uri: https://docs.ghostcrawl.io
|
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
require 'microsoft_kiota_abstractions'
|
|
2
|
-
require_relative '../../ghostcrawl'
|
|
3
|
-
require_relative '../v1'
|
|
4
|
-
require_relative './binary'
|
|
5
|
-
require_relative './download/download_request_builder'
|
|
6
|
-
|
|
7
|
-
module Ghostcrawl
|
|
8
|
-
module V1
|
|
9
|
-
module Binary
|
|
10
|
-
##
|
|
11
|
-
# Builds and executes requests for operations under #v1#binary
|
|
12
|
-
class BinaryRequestBuilder < MicrosoftKiotaAbstractions::BaseRequestBuilder
|
|
13
|
-
|
|
14
|
-
##
|
|
15
|
-
# The download property
|
|
16
|
-
def download()
|
|
17
|
-
return Ghostcrawl::V1::Binary::Download::DownloadRequestBuilder.new(@path_parameters, @request_adapter)
|
|
18
|
-
end
|
|
19
|
-
##
|
|
20
|
-
## Instantiates a new BinaryRequestBuilder and sets the default values.
|
|
21
|
-
## @param path_parameters Path parameters for the request
|
|
22
|
-
## @param request_adapter The request adapter to use to execute the requests.
|
|
23
|
-
## @return a void
|
|
24
|
-
##
|
|
25
|
-
def initialize(path_parameters, request_adapter)
|
|
26
|
-
super(path_parameters, request_adapter, "{+baseurl}/v1/binary")
|
|
27
|
-
end
|
|
28
|
-
end
|
|
29
|
-
end
|
|
30
|
-
end
|
|
31
|
-
end
|
|
File without changes
|
|
@@ -1,77 +0,0 @@
|
|
|
1
|
-
require 'microsoft_kiota_abstractions'
|
|
2
|
-
require_relative '../../../ghostcrawl'
|
|
3
|
-
require_relative '../../../models/h_t_t_p_validation_error'
|
|
4
|
-
require_relative '../../v1'
|
|
5
|
-
require_relative '../binary'
|
|
6
|
-
require_relative './download'
|
|
7
|
-
|
|
8
|
-
module Ghostcrawl
|
|
9
|
-
module V1
|
|
10
|
-
module Binary
|
|
11
|
-
module Download
|
|
12
|
-
##
|
|
13
|
-
# Builds and executes requests for operations under #v1#binary#download
|
|
14
|
-
class DownloadRequestBuilder < MicrosoftKiotaAbstractions::BaseRequestBuilder
|
|
15
|
-
|
|
16
|
-
##
|
|
17
|
-
## Instantiates a new DownloadRequestBuilder and sets the default values.
|
|
18
|
-
## @param path_parameters Path parameters for the request
|
|
19
|
-
## @param request_adapter The request adapter to use to execute the requests.
|
|
20
|
-
## @return a void
|
|
21
|
-
##
|
|
22
|
-
def initialize(path_parameters, request_adapter)
|
|
23
|
-
super(path_parameters, request_adapter, "{+baseurl}/v1/binary/download?engine={engine}{&release_id*}")
|
|
24
|
-
end
|
|
25
|
-
##
|
|
26
|
-
## Download your licensed engine binary (Chrome, Firefox, or WebKit) for self-hosting.Select the engine via the ``engine`` query parameter. The binary is streamedto the client and bound to your account.
|
|
27
|
-
## @param request_configuration Configuration for the request such as headers, query parameters, and middleware options.
|
|
28
|
-
## @return a Fiber of binary
|
|
29
|
-
##
|
|
30
|
-
def get(request_configuration=nil)
|
|
31
|
-
request_info = self.to_get_request_information(
|
|
32
|
-
request_configuration
|
|
33
|
-
)
|
|
34
|
-
error_mapping = Hash.new
|
|
35
|
-
error_mapping["422"] = lambda {|pn| Ghostcrawl::Models::HTTPValidationError.create_from_discriminator_value(pn) }
|
|
36
|
-
return @request_adapter.send_async(request_info, Binary, error_mapping)
|
|
37
|
-
end
|
|
38
|
-
##
|
|
39
|
-
## Download your licensed engine binary (Chrome, Firefox, or WebKit) for self-hosting.Select the engine via the ``engine`` query parameter. The binary is streamedto the client and bound to your account.
|
|
40
|
-
## @param request_configuration Configuration for the request such as headers, query parameters, and middleware options.
|
|
41
|
-
## @return a request_information
|
|
42
|
-
##
|
|
43
|
-
def to_get_request_information(request_configuration=nil)
|
|
44
|
-
request_info = MicrosoftKiotaAbstractions::RequestInformation.new()
|
|
45
|
-
unless request_configuration.nil?
|
|
46
|
-
request_info.add_headers_from_raw_object(request_configuration.headers)
|
|
47
|
-
request_info.set_query_string_parameters_from_raw_object(request_configuration.query_parameters)
|
|
48
|
-
request_info.add_request_options(request_configuration.options)
|
|
49
|
-
end
|
|
50
|
-
request_info.url_template = @url_template
|
|
51
|
-
request_info.path_parameters = @path_parameters
|
|
52
|
-
request_info.http_method = :GET
|
|
53
|
-
request_info.headers.try_add('Accept', 'application/json')
|
|
54
|
-
return request_info
|
|
55
|
-
end
|
|
56
|
-
##
|
|
57
|
-
## Returns a request builder with the provided arbitrary URL. Using this method means any other path or query parameters are ignored.
|
|
58
|
-
## @param raw_url The raw URL to use for the request builder.
|
|
59
|
-
## @return a download_request_builder
|
|
60
|
-
##
|
|
61
|
-
def with_url(raw_url)
|
|
62
|
-
raise StandardError, 'raw_url cannot be null' if raw_url.nil?
|
|
63
|
-
return DownloadRequestBuilder.new(raw_url, @request_adapter)
|
|
64
|
-
end
|
|
65
|
-
|
|
66
|
-
##
|
|
67
|
-
# Download your licensed engine binary (Chrome, Firefox, or WebKit) for self-hosting.Select the engine via the ``engine`` query parameter. The binary is streamedto the client and bound to your account.
|
|
68
|
-
class DownloadRequestBuilderGetQueryParameters
|
|
69
|
-
|
|
70
|
-
attr_accessor :engine
|
|
71
|
-
attr_accessor :release_id
|
|
72
|
-
end
|
|
73
|
-
end
|
|
74
|
-
end
|
|
75
|
-
end
|
|
76
|
-
end
|
|
77
|
-
end
|