crawlberg 1.11.0-x86_64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/README.md +168 -0
- data/Steepfile +14 -0
- data/ext/crawlberg_rb/native/Cargo.lock +3706 -0
- data/ext/crawlberg_rb/native/Cargo.toml +42 -0
- data/ext/crawlberg_rb/native/extconf.rb +19 -0
- data/ext/crawlberg_rb/src/lib.rs +10607 -0
- data/lib/crawlberg/native.rb +649 -0
- data/lib/crawlberg/version.rb +10 -0
- data/lib/crawlberg.rb +21 -0
- data/lib/crawlberg_rb/3.3.0/crawlberg_rb.so +4 -0
- data/sig/types.rbs +563 -0
- metadata +96 -0
|
@@ -0,0 +1,649 @@
|
|
|
1
|
+
# This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
+
# alef:hash:d323ed0aab9764791b48678b68c8fedcf8e7b3e1dbe50ec190a9023e68acc3bb
|
|
3
|
+
# To regenerate: alef generate
|
|
4
|
+
# To verify freshness: alef verify
|
|
5
|
+
# frozen_string_literal: true
|
|
6
|
+
|
|
7
|
+
require "json"
|
|
8
|
+
require "sorbet-runtime"
|
|
9
|
+
require "rbconfig"
|
|
10
|
+
|
|
11
|
+
extension_name = "crawlberg_rb"
|
|
12
|
+
# Ruby may expose more than one shared-library extension via DLEXT/DLEXT2.
|
|
13
|
+
dlexts = [RbConfig::CONFIG["DLEXT"], RbConfig::CONFIG["DLEXT2"]].compact.uniq
|
|
14
|
+
ruby_abi = RbConfig::CONFIG.fetch("ruby_version")
|
|
15
|
+
|
|
16
|
+
library_candidates = dlexts.flat_map do |dlext|
|
|
17
|
+
[
|
|
18
|
+
File.join(__dir__, "..", extension_name, ruby_abi, "#{extension_name}.#{dlext}"),
|
|
19
|
+
File.join(__dir__, "..", extension_name, ruby_abi, "lib#{extension_name}.#{dlext}"),
|
|
20
|
+
File.join(__dir__, "..", "#{extension_name}.#{dlext}"),
|
|
21
|
+
File.join(__dir__, "..", "lib#{extension_name}.#{dlext}")
|
|
22
|
+
]
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
expanded_candidates = library_candidates.map { |path| File.expand_path(path) }
|
|
26
|
+
native_extension = expanded_candidates.find { |path| File.file?(path) }
|
|
27
|
+
|
|
28
|
+
unless native_extension
|
|
29
|
+
raise(
|
|
30
|
+
LoadError,
|
|
31
|
+
"cannot find #{extension_name} native extension for Ruby ABI #{ruby_abi}; " \
|
|
32
|
+
"searched: #{expanded_candidates.join(", ")}"
|
|
33
|
+
)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
require native_extension
|
|
37
|
+
|
|
38
|
+
module Crawlberg
|
|
39
|
+
# Authentication configuration.
|
|
40
|
+
module AuthConfig
|
|
41
|
+
extend T::Helpers
|
|
42
|
+
extend T::Sig
|
|
43
|
+
|
|
44
|
+
interface!
|
|
45
|
+
|
|
46
|
+
# Dispatch from a Hash to the appropriate variant constructor.
|
|
47
|
+
# @param hash [Hash] with discriminator field and variant-specific fields
|
|
48
|
+
# @return [variant_class] an instance of the appropriate variant
|
|
49
|
+
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.untyped) }
|
|
50
|
+
def self.from_hash(hash)
|
|
51
|
+
discriminator = hash[:type] || hash["type"]
|
|
52
|
+
case discriminator
|
|
53
|
+
when "basic"
|
|
54
|
+
AuthConfigBasic.from_hash(hash)
|
|
55
|
+
when "bearer"
|
|
56
|
+
AuthConfigBearer.from_hash(hash)
|
|
57
|
+
when "header"
|
|
58
|
+
AuthConfigHeader.from_hash(hash)
|
|
59
|
+
else
|
|
60
|
+
raise "Unknown discriminator: #{discriminator}"
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
## HTTP Basic authentication.
|
|
65
|
+
AuthConfigBasic = Data.define(:username, :password) do
|
|
66
|
+
include AuthConfig
|
|
67
|
+
extend T::Sig
|
|
68
|
+
|
|
69
|
+
# Username sent in the `Authorization: Basic` header.
|
|
70
|
+
sig { returns(String) }
|
|
71
|
+
def username = Data.instance_method(:to_h).bind_call(self).fetch(:username)
|
|
72
|
+
# Password sent in the `Authorization: Basic` header.
|
|
73
|
+
sig { returns(String) }
|
|
74
|
+
def password = Data.instance_method(:to_h).bind_call(self).fetch(:password)
|
|
75
|
+
sig { returns(T::Boolean) }
|
|
76
|
+
def basic? = true
|
|
77
|
+
sig { returns(T::Boolean) }
|
|
78
|
+
def bearer? = false
|
|
79
|
+
sig { returns(T::Boolean) }
|
|
80
|
+
def header? = false
|
|
81
|
+
# @param hash [Hash] deserialized from the native extension
|
|
82
|
+
# @return [self]
|
|
83
|
+
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
84
|
+
def self.from_hash(hash)
|
|
85
|
+
new(username: hash[:username] || hash["username"], password: hash[:password] || hash["password"])
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
## Bearer token authentication.
|
|
89
|
+
AuthConfigBearer = Data.define(:token) do
|
|
90
|
+
include AuthConfig
|
|
91
|
+
extend T::Sig
|
|
92
|
+
|
|
93
|
+
# Token sent in the `Authorization: Bearer` header.
|
|
94
|
+
sig { returns(String) }
|
|
95
|
+
def token = Data.instance_method(:to_h).bind_call(self).fetch(:token)
|
|
96
|
+
sig { returns(T::Boolean) }
|
|
97
|
+
def basic? = false
|
|
98
|
+
sig { returns(T::Boolean) }
|
|
99
|
+
def bearer? = true
|
|
100
|
+
sig { returns(T::Boolean) }
|
|
101
|
+
def header? = false
|
|
102
|
+
# @param hash [Hash] deserialized from the native extension
|
|
103
|
+
# @return [self]
|
|
104
|
+
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
105
|
+
def self.from_hash(hash)
|
|
106
|
+
new(token: hash[:token] || hash["token"])
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
## Custom authentication header.
|
|
110
|
+
AuthConfigHeader = Data.define(:name, :value) do
|
|
111
|
+
include AuthConfig
|
|
112
|
+
extend T::Sig
|
|
113
|
+
|
|
114
|
+
# HTTP header name to set on each request.
|
|
115
|
+
sig { returns(String) }
|
|
116
|
+
def name = Data.instance_method(:to_h).bind_call(self).fetch(:name)
|
|
117
|
+
# HTTP header value to send.
|
|
118
|
+
sig { returns(String) }
|
|
119
|
+
def value = Data.instance_method(:to_h).bind_call(self).fetch(:value)
|
|
120
|
+
sig { returns(T::Boolean) }
|
|
121
|
+
def basic? = false
|
|
122
|
+
sig { returns(T::Boolean) }
|
|
123
|
+
def bearer? = false
|
|
124
|
+
sig { returns(T::Boolean) }
|
|
125
|
+
def header? = true
|
|
126
|
+
# @param hash [Hash] deserialized from the native extension
|
|
127
|
+
# @return [self]
|
|
128
|
+
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
129
|
+
def self.from_hash(hash)
|
|
130
|
+
new(name: hash[:name] || hash["name"], value: hash[:value] || hash["value"])
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
module Crawlberg
|
|
136
|
+
# An event emitted during a streaming crawl operation.
|
|
137
|
+
#
|
|
138
|
+
# Not available on `wasm32` targets — streaming requires native concurrency
|
|
139
|
+
# primitives (tokio channels, `JoinSet`) that are not supported on wasm32.
|
|
140
|
+
#
|
|
141
|
+
# Delivered to bindings through each target's native streaming idiom.
|
|
142
|
+
module CrawlEvent
|
|
143
|
+
extend T::Helpers
|
|
144
|
+
extend T::Sig
|
|
145
|
+
|
|
146
|
+
interface!
|
|
147
|
+
|
|
148
|
+
# Dispatch from a Hash to the appropriate variant constructor.
|
|
149
|
+
# @param hash [Hash] with discriminator field and variant-specific fields
|
|
150
|
+
# @return [variant_class] an instance of the appropriate variant
|
|
151
|
+
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.untyped) }
|
|
152
|
+
def self.from_hash(hash)
|
|
153
|
+
discriminator = hash[:type] || hash["type"]
|
|
154
|
+
case discriminator
|
|
155
|
+
when "page"
|
|
156
|
+
CrawlEventPage.from_hash(hash)
|
|
157
|
+
when "error"
|
|
158
|
+
CrawlEventError.from_hash(hash)
|
|
159
|
+
when "complete"
|
|
160
|
+
CrawlEventComplete.from_hash(hash)
|
|
161
|
+
else
|
|
162
|
+
raise "Unknown discriminator: #{discriminator}"
|
|
163
|
+
end
|
|
164
|
+
end
|
|
165
|
+
end
|
|
166
|
+
## A single page has been crawled.
|
|
167
|
+
CrawlEventPage = Data.define(:result) do
|
|
168
|
+
include CrawlEvent
|
|
169
|
+
extend T::Sig
|
|
170
|
+
|
|
171
|
+
# The crawled page result.
|
|
172
|
+
sig { returns(CrawlPageResult) }
|
|
173
|
+
def result = Data.instance_method(:to_h).bind_call(self).fetch(:result)
|
|
174
|
+
sig { returns(T::Boolean) }
|
|
175
|
+
def page? = true
|
|
176
|
+
sig { returns(T::Boolean) }
|
|
177
|
+
def error? = false
|
|
178
|
+
sig { returns(T::Boolean) }
|
|
179
|
+
def complete? = false
|
|
180
|
+
# @param hash [Hash] deserialized from the native extension
|
|
181
|
+
# @return [self]
|
|
182
|
+
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
183
|
+
def self.from_hash(hash)
|
|
184
|
+
new(result: hash[:result] || hash["result"])
|
|
185
|
+
end
|
|
186
|
+
end
|
|
187
|
+
## An error occurred while crawling a URL.
|
|
188
|
+
CrawlEventError = Data.define(:url, :error) do
|
|
189
|
+
include CrawlEvent
|
|
190
|
+
extend T::Sig
|
|
191
|
+
|
|
192
|
+
# The URL that failed.
|
|
193
|
+
sig { returns(String) }
|
|
194
|
+
def url = Data.instance_method(:to_h).bind_call(self).fetch(:url)
|
|
195
|
+
# The error message.
|
|
196
|
+
sig { returns(String) }
|
|
197
|
+
def error = Data.instance_method(:to_h).bind_call(self).fetch(:error)
|
|
198
|
+
sig { returns(T::Boolean) }
|
|
199
|
+
def page? = false
|
|
200
|
+
sig { returns(T::Boolean) }
|
|
201
|
+
def error? = true
|
|
202
|
+
sig { returns(T::Boolean) }
|
|
203
|
+
def complete? = false
|
|
204
|
+
# @param hash [Hash] deserialized from the native extension
|
|
205
|
+
# @return [self]
|
|
206
|
+
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
207
|
+
def self.from_hash(hash)
|
|
208
|
+
new(url: hash[:url] || hash["url"], error: hash[:error] || hash["error"])
|
|
209
|
+
end
|
|
210
|
+
end
|
|
211
|
+
## The crawl has completed.
|
|
212
|
+
CrawlEventComplete = Data.define(:pages_crawled) do
|
|
213
|
+
include CrawlEvent
|
|
214
|
+
extend T::Sig
|
|
215
|
+
|
|
216
|
+
# Total number of pages crawled.
|
|
217
|
+
sig { returns(Integer) }
|
|
218
|
+
def pages_crawled = Data.instance_method(:to_h).bind_call(self).fetch(:pages_crawled)
|
|
219
|
+
sig { returns(T::Boolean) }
|
|
220
|
+
def page? = false
|
|
221
|
+
sig { returns(T::Boolean) }
|
|
222
|
+
def error? = false
|
|
223
|
+
sig { returns(T::Boolean) }
|
|
224
|
+
def complete? = true
|
|
225
|
+
# @param hash [Hash] deserialized from the native extension
|
|
226
|
+
# @return [self]
|
|
227
|
+
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
228
|
+
def self.from_hash(hash)
|
|
229
|
+
new(pages_crawled: hash[:pages_crawled] || hash["pages_crawled"])
|
|
230
|
+
end
|
|
231
|
+
end
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
module Crawlberg
|
|
235
|
+
# Hostname/IP allowlist matcher for SSRF policy.
|
|
236
|
+
#
|
|
237
|
+
# Serializes as an internally-tagged object so each variant is distinguishable on the
|
|
238
|
+
# wire and round-trips losslessly:
|
|
239
|
+
#
|
|
240
|
+
# ```json
|
|
241
|
+
# {"type": "exact", "value": "api.example.com"}
|
|
242
|
+
# {"type": "suffix", "value": ".example.com"}
|
|
243
|
+
# {"type": "cidr", "value": "10.0.0.0/8"}
|
|
244
|
+
# ```
|
|
245
|
+
#
|
|
246
|
+
# A bare JSON string is still accepted on deserialization and resolves to [`Exact`],
|
|
247
|
+
# preserving configs written against the previous untagged representation.
|
|
248
|
+
#
|
|
249
|
+
# [`Exact`]: HostMatcher::Exact
|
|
250
|
+
module HostMatcher
|
|
251
|
+
extend T::Helpers
|
|
252
|
+
extend T::Sig
|
|
253
|
+
|
|
254
|
+
interface!
|
|
255
|
+
|
|
256
|
+
# Dispatch from a Hash to the appropriate variant constructor.
|
|
257
|
+
# @param hash [Hash] with discriminator field and variant-specific fields
|
|
258
|
+
# @return [variant_class] an instance of the appropriate variant
|
|
259
|
+
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.untyped) }
|
|
260
|
+
def self.from_hash(hash)
|
|
261
|
+
discriminator = hash[:type] || hash["type"]
|
|
262
|
+
case discriminator
|
|
263
|
+
when "exact"
|
|
264
|
+
HostMatcherExact.from_hash(hash)
|
|
265
|
+
when "suffix"
|
|
266
|
+
HostMatcherSuffix.from_hash(hash)
|
|
267
|
+
when "cidr"
|
|
268
|
+
HostMatcherCidr.from_hash(hash)
|
|
269
|
+
else
|
|
270
|
+
raise "Unknown discriminator: #{discriminator}"
|
|
271
|
+
end
|
|
272
|
+
end
|
|
273
|
+
end
|
|
274
|
+
## Exact hostname match (case-insensitive).
|
|
275
|
+
HostMatcherExact = Data.define(:value) do
|
|
276
|
+
include HostMatcher
|
|
277
|
+
extend T::Sig
|
|
278
|
+
|
|
279
|
+
# The hostname to match.
|
|
280
|
+
sig { returns(String) }
|
|
281
|
+
def value = Data.instance_method(:to_h).bind_call(self).fetch(:value)
|
|
282
|
+
sig { returns(T::Boolean) }
|
|
283
|
+
def exact? = true
|
|
284
|
+
sig { returns(T::Boolean) }
|
|
285
|
+
def suffix? = false
|
|
286
|
+
sig { returns(T::Boolean) }
|
|
287
|
+
def cidr? = false
|
|
288
|
+
# @param hash [Hash] deserialized from the native extension
|
|
289
|
+
# @return [self]
|
|
290
|
+
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
291
|
+
def self.from_hash(hash)
|
|
292
|
+
new(value: hash[:value] || hash["value"])
|
|
293
|
+
end
|
|
294
|
+
end
|
|
295
|
+
## Suffix match: ".xberg.io" matches "api.xberg.io" and "xberg.io".
|
|
296
|
+
HostMatcherSuffix = Data.define(:value) do
|
|
297
|
+
include HostMatcher
|
|
298
|
+
extend T::Sig
|
|
299
|
+
|
|
300
|
+
# The dot-prefixed suffix to match. A leading dot is optional.
|
|
301
|
+
sig { returns(String) }
|
|
302
|
+
def value = Data.instance_method(:to_h).bind_call(self).fetch(:value)
|
|
303
|
+
sig { returns(T::Boolean) }
|
|
304
|
+
def exact? = false
|
|
305
|
+
sig { returns(T::Boolean) }
|
|
306
|
+
def suffix? = true
|
|
307
|
+
sig { returns(T::Boolean) }
|
|
308
|
+
def cidr? = false
|
|
309
|
+
# @param hash [Hash] deserialized from the native extension
|
|
310
|
+
# @return [self]
|
|
311
|
+
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
312
|
+
def self.from_hash(hash)
|
|
313
|
+
new(value: hash[:value] || hash["value"])
|
|
314
|
+
end
|
|
315
|
+
end
|
|
316
|
+
## CIDR match: "10.0.0.0/8" matches IP addresses in that range.
|
|
317
|
+
HostMatcherCidr = Data.define(:value) do
|
|
318
|
+
include HostMatcher
|
|
319
|
+
extend T::Sig
|
|
320
|
+
|
|
321
|
+
# The CIDR block. Validated when built through [`HostMatcher::cidr`] or
|
|
322
|
+
# deserialization.
|
|
323
|
+
sig { returns(String) }
|
|
324
|
+
def value = Data.instance_method(:to_h).bind_call(self).fetch(:value)
|
|
325
|
+
sig { returns(T::Boolean) }
|
|
326
|
+
def exact? = false
|
|
327
|
+
sig { returns(T::Boolean) }
|
|
328
|
+
def suffix? = false
|
|
329
|
+
sig { returns(T::Boolean) }
|
|
330
|
+
def cidr? = true
|
|
331
|
+
# @param hash [Hash] deserialized from the native extension
|
|
332
|
+
# @return [self]
|
|
333
|
+
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
334
|
+
def self.from_hash(hash)
|
|
335
|
+
new(value: hash[:value] || hash["value"])
|
|
336
|
+
end
|
|
337
|
+
end
|
|
338
|
+
end
|
|
339
|
+
|
|
340
|
+
module Crawlberg
|
|
341
|
+
# A single page interaction action.
|
|
342
|
+
#
|
|
343
|
+
# Actions are serialized with a `type` tag using camelCase naming,
|
|
344
|
+
# except `ExecuteJs` which is explicitly renamed to `"executeJs"`.
|
|
345
|
+
module PageAction
|
|
346
|
+
extend T::Helpers
|
|
347
|
+
extend T::Sig
|
|
348
|
+
|
|
349
|
+
interface!
|
|
350
|
+
|
|
351
|
+
# Dispatch from a Hash to the appropriate variant constructor.
|
|
352
|
+
# @param hash [Hash] with discriminator field and variant-specific fields
|
|
353
|
+
# @return [variant_class] an instance of the appropriate variant
|
|
354
|
+
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.untyped) }
|
|
355
|
+
def self.from_hash(hash)
|
|
356
|
+
discriminator = hash[:type] || hash["type"]
|
|
357
|
+
case discriminator
|
|
358
|
+
when "click"
|
|
359
|
+
PageActionClick.from_hash(hash)
|
|
360
|
+
when "type"
|
|
361
|
+
PageActionTypeText.from_hash(hash)
|
|
362
|
+
when "press"
|
|
363
|
+
PageActionPress.from_hash(hash)
|
|
364
|
+
when "scroll"
|
|
365
|
+
PageActionScroll.from_hash(hash)
|
|
366
|
+
when "wait"
|
|
367
|
+
PageActionWait.from_hash(hash)
|
|
368
|
+
when "screenshot"
|
|
369
|
+
PageActionScreenshot.from_hash(hash)
|
|
370
|
+
when "executeJs"
|
|
371
|
+
PageActionExecuteJs.from_hash(hash)
|
|
372
|
+
when "scrape"
|
|
373
|
+
PageActionScrape.from_hash(hash)
|
|
374
|
+
else
|
|
375
|
+
raise "Unknown discriminator: #{discriminator}"
|
|
376
|
+
end
|
|
377
|
+
end
|
|
378
|
+
end
|
|
379
|
+
## Click on an element matching the given CSS selector.
|
|
380
|
+
PageActionClick = Data.define(:selector) do
|
|
381
|
+
include PageAction
|
|
382
|
+
extend T::Sig
|
|
383
|
+
|
|
384
|
+
# CSS selector for the element to click.
|
|
385
|
+
sig { returns(String) }
|
|
386
|
+
def selector = Data.instance_method(:to_h).bind_call(self).fetch(:selector)
|
|
387
|
+
sig { returns(T::Boolean) }
|
|
388
|
+
def click? = true
|
|
389
|
+
sig { returns(T::Boolean) }
|
|
390
|
+
def type_text? = false
|
|
391
|
+
sig { returns(T::Boolean) }
|
|
392
|
+
def press? = false
|
|
393
|
+
sig { returns(T::Boolean) }
|
|
394
|
+
def scroll? = false
|
|
395
|
+
sig { returns(T::Boolean) }
|
|
396
|
+
def wait? = false
|
|
397
|
+
sig { returns(T::Boolean) }
|
|
398
|
+
def screenshot? = false
|
|
399
|
+
sig { returns(T::Boolean) }
|
|
400
|
+
def execute_js? = false
|
|
401
|
+
sig { returns(T::Boolean) }
|
|
402
|
+
def scrape? = false
|
|
403
|
+
# @param hash [Hash] deserialized from the native extension
|
|
404
|
+
# @return [self]
|
|
405
|
+
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
406
|
+
def self.from_hash(hash)
|
|
407
|
+
new(selector: hash[:selector] || hash["selector"])
|
|
408
|
+
end
|
|
409
|
+
end
|
|
410
|
+
## Type text into an element matching the given CSS selector.
|
|
411
|
+
PageActionTypeText = Data.define(:selector, :text) do
|
|
412
|
+
include PageAction
|
|
413
|
+
extend T::Sig
|
|
414
|
+
|
|
415
|
+
# CSS selector for the input element.
|
|
416
|
+
sig { returns(String) }
|
|
417
|
+
def selector = Data.instance_method(:to_h).bind_call(self).fetch(:selector)
|
|
418
|
+
# Text to type into the element.
|
|
419
|
+
sig { returns(String) }
|
|
420
|
+
def text = Data.instance_method(:to_h).bind_call(self).fetch(:text)
|
|
421
|
+
sig { returns(T::Boolean) }
|
|
422
|
+
def click? = false
|
|
423
|
+
sig { returns(T::Boolean) }
|
|
424
|
+
def type_text? = true
|
|
425
|
+
sig { returns(T::Boolean) }
|
|
426
|
+
def press? = false
|
|
427
|
+
sig { returns(T::Boolean) }
|
|
428
|
+
def scroll? = false
|
|
429
|
+
sig { returns(T::Boolean) }
|
|
430
|
+
def wait? = false
|
|
431
|
+
sig { returns(T::Boolean) }
|
|
432
|
+
def screenshot? = false
|
|
433
|
+
sig { returns(T::Boolean) }
|
|
434
|
+
def execute_js? = false
|
|
435
|
+
sig { returns(T::Boolean) }
|
|
436
|
+
def scrape? = false
|
|
437
|
+
# @param hash [Hash] deserialized from the native extension
|
|
438
|
+
# @return [self]
|
|
439
|
+
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
440
|
+
def self.from_hash(hash)
|
|
441
|
+
new(selector: hash[:selector] || hash["selector"], text: hash[:text] || hash["text"])
|
|
442
|
+
end
|
|
443
|
+
end
|
|
444
|
+
## Press a keyboard key (e.g. "Enter", "Tab", "Escape").
|
|
445
|
+
PageActionPress = Data.define(:key) do
|
|
446
|
+
include PageAction
|
|
447
|
+
extend T::Sig
|
|
448
|
+
|
|
449
|
+
# Key name to press.
|
|
450
|
+
sig { returns(String) }
|
|
451
|
+
def key = Data.instance_method(:to_h).bind_call(self).fetch(:key)
|
|
452
|
+
sig { returns(T::Boolean) }
|
|
453
|
+
def click? = false
|
|
454
|
+
sig { returns(T::Boolean) }
|
|
455
|
+
def type_text? = false
|
|
456
|
+
sig { returns(T::Boolean) }
|
|
457
|
+
def press? = true
|
|
458
|
+
sig { returns(T::Boolean) }
|
|
459
|
+
def scroll? = false
|
|
460
|
+
sig { returns(T::Boolean) }
|
|
461
|
+
def wait? = false
|
|
462
|
+
sig { returns(T::Boolean) }
|
|
463
|
+
def screenshot? = false
|
|
464
|
+
sig { returns(T::Boolean) }
|
|
465
|
+
def execute_js? = false
|
|
466
|
+
sig { returns(T::Boolean) }
|
|
467
|
+
def scrape? = false
|
|
468
|
+
# @param hash [Hash] deserialized from the native extension
|
|
469
|
+
# @return [self]
|
|
470
|
+
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
471
|
+
def self.from_hash(hash)
|
|
472
|
+
new(key: hash[:key] || hash["key"])
|
|
473
|
+
end
|
|
474
|
+
end
|
|
475
|
+
## Scroll the page or a specific element.
|
|
476
|
+
PageActionScroll = Data.define(:direction, :selector, :amount) do
|
|
477
|
+
include PageAction
|
|
478
|
+
extend T::Sig
|
|
479
|
+
|
|
480
|
+
# Direction to scroll.
|
|
481
|
+
sig { returns(ScrollDirection) }
|
|
482
|
+
def direction = Data.instance_method(:to_h).bind_call(self).fetch(:direction)
|
|
483
|
+
# Optional CSS selector for a scrollable element. Scrolls the page if absent.
|
|
484
|
+
sig { returns(T.nilable(String)) }
|
|
485
|
+
def selector = Data.instance_method(:to_h).bind_call(self).fetch(:selector)
|
|
486
|
+
# Optional pixel amount to scroll. Uses a default if absent.
|
|
487
|
+
sig { returns(T.nilable(Integer)) }
|
|
488
|
+
def amount = Data.instance_method(:to_h).bind_call(self).fetch(:amount)
|
|
489
|
+
sig { returns(T::Boolean) }
|
|
490
|
+
def click? = false
|
|
491
|
+
sig { returns(T::Boolean) }
|
|
492
|
+
def type_text? = false
|
|
493
|
+
sig { returns(T::Boolean) }
|
|
494
|
+
def press? = false
|
|
495
|
+
sig { returns(T::Boolean) }
|
|
496
|
+
def scroll? = true
|
|
497
|
+
sig { returns(T::Boolean) }
|
|
498
|
+
def wait? = false
|
|
499
|
+
sig { returns(T::Boolean) }
|
|
500
|
+
def screenshot? = false
|
|
501
|
+
sig { returns(T::Boolean) }
|
|
502
|
+
def execute_js? = false
|
|
503
|
+
sig { returns(T::Boolean) }
|
|
504
|
+
def scrape? = false
|
|
505
|
+
# @param hash [Hash] deserialized from the native extension
|
|
506
|
+
# @return [self]
|
|
507
|
+
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
508
|
+
def self.from_hash(hash)
|
|
509
|
+
new(
|
|
510
|
+
direction: hash[:direction] || hash["direction"],
|
|
511
|
+
selector: hash[:selector] || hash["selector"],
|
|
512
|
+
amount: hash[:amount] || hash["amount"]
|
|
513
|
+
)
|
|
514
|
+
end
|
|
515
|
+
end
|
|
516
|
+
## Wait for a duration or for an element to appear.
|
|
517
|
+
PageActionWait = Data.define(:milliseconds, :selector) do
|
|
518
|
+
include PageAction
|
|
519
|
+
extend T::Sig
|
|
520
|
+
|
|
521
|
+
# Milliseconds to wait. Ignored if `selector` is provided.
|
|
522
|
+
sig { returns(T.nilable(Integer)) }
|
|
523
|
+
def milliseconds = Data.instance_method(:to_h).bind_call(self).fetch(:milliseconds)
|
|
524
|
+
# CSS selector to wait for.
|
|
525
|
+
sig { returns(T.nilable(String)) }
|
|
526
|
+
def selector = Data.instance_method(:to_h).bind_call(self).fetch(:selector)
|
|
527
|
+
sig { returns(T::Boolean) }
|
|
528
|
+
def click? = false
|
|
529
|
+
sig { returns(T::Boolean) }
|
|
530
|
+
def type_text? = false
|
|
531
|
+
sig { returns(T::Boolean) }
|
|
532
|
+
def press? = false
|
|
533
|
+
sig { returns(T::Boolean) }
|
|
534
|
+
def scroll? = false
|
|
535
|
+
sig { returns(T::Boolean) }
|
|
536
|
+
def wait? = true
|
|
537
|
+
sig { returns(T::Boolean) }
|
|
538
|
+
def screenshot? = false
|
|
539
|
+
sig { returns(T::Boolean) }
|
|
540
|
+
def execute_js? = false
|
|
541
|
+
sig { returns(T::Boolean) }
|
|
542
|
+
def scrape? = false
|
|
543
|
+
# @param hash [Hash] deserialized from the native extension
|
|
544
|
+
# @return [self]
|
|
545
|
+
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
546
|
+
def self.from_hash(hash)
|
|
547
|
+
new(milliseconds: hash[:milliseconds] || hash["milliseconds"], selector: hash[:selector] || hash["selector"])
|
|
548
|
+
end
|
|
549
|
+
end
|
|
550
|
+
## Take a screenshot of the current page.
|
|
551
|
+
PageActionScreenshot = Data.define(:full_page) do
|
|
552
|
+
include PageAction
|
|
553
|
+
extend T::Sig
|
|
554
|
+
|
|
555
|
+
# Whether to capture the full scrollable page. Defaults to viewport only.
|
|
556
|
+
#
|
|
557
|
+
# Accepts both the canonical `fullPage` (camelCase) form and the
|
|
558
|
+
# `full_page` (snake_case) alias so language bindings and fixtures can
|
|
559
|
+
# use either convention without error.
|
|
560
|
+
sig { returns(T.nilable(T::Boolean)) }
|
|
561
|
+
def full_page = Data.instance_method(:to_h).bind_call(self).fetch(:full_page)
|
|
562
|
+
sig { returns(T::Boolean) }
|
|
563
|
+
def click? = false
|
|
564
|
+
sig { returns(T::Boolean) }
|
|
565
|
+
def type_text? = false
|
|
566
|
+
sig { returns(T::Boolean) }
|
|
567
|
+
def press? = false
|
|
568
|
+
sig { returns(T::Boolean) }
|
|
569
|
+
def scroll? = false
|
|
570
|
+
sig { returns(T::Boolean) }
|
|
571
|
+
def wait? = false
|
|
572
|
+
sig { returns(T::Boolean) }
|
|
573
|
+
def screenshot? = true
|
|
574
|
+
sig { returns(T::Boolean) }
|
|
575
|
+
def execute_js? = false
|
|
576
|
+
sig { returns(T::Boolean) }
|
|
577
|
+
def scrape? = false
|
|
578
|
+
# @param hash [Hash] deserialized from the native extension
|
|
579
|
+
# @return [self]
|
|
580
|
+
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
581
|
+
def self.from_hash(hash)
|
|
582
|
+
new(full_page: hash[:full_page] || hash["full_page"])
|
|
583
|
+
end
|
|
584
|
+
end
|
|
585
|
+
## Execute arbitrary JavaScript in the page context.
|
|
586
|
+
##
|
|
587
|
+
## # Safety
|
|
588
|
+
##
|
|
589
|
+
## The script runs with full page privileges in the browser context.
|
|
590
|
+
## Only execute scripts from trusted sources.
|
|
591
|
+
PageActionExecuteJs = Data.define(:script) do
|
|
592
|
+
include PageAction
|
|
593
|
+
extend T::Sig
|
|
594
|
+
|
|
595
|
+
# JavaScript source code to execute. Max 1 MB.
|
|
596
|
+
sig { returns(String) }
|
|
597
|
+
def script = Data.instance_method(:to_h).bind_call(self).fetch(:script)
|
|
598
|
+
sig { returns(T::Boolean) }
|
|
599
|
+
def click? = false
|
|
600
|
+
sig { returns(T::Boolean) }
|
|
601
|
+
def type_text? = false
|
|
602
|
+
sig { returns(T::Boolean) }
|
|
603
|
+
def press? = false
|
|
604
|
+
sig { returns(T::Boolean) }
|
|
605
|
+
def scroll? = false
|
|
606
|
+
sig { returns(T::Boolean) }
|
|
607
|
+
def wait? = false
|
|
608
|
+
sig { returns(T::Boolean) }
|
|
609
|
+
def screenshot? = false
|
|
610
|
+
sig { returns(T::Boolean) }
|
|
611
|
+
def execute_js? = true
|
|
612
|
+
sig { returns(T::Boolean) }
|
|
613
|
+
def scrape? = false
|
|
614
|
+
# @param hash [Hash] deserialized from the native extension
|
|
615
|
+
# @return [self]
|
|
616
|
+
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
617
|
+
def self.from_hash(hash)
|
|
618
|
+
new(script: hash[:script] || hash["script"])
|
|
619
|
+
end
|
|
620
|
+
end
|
|
621
|
+
## Scrape the current page HTML.
|
|
622
|
+
PageActionScrape = Data.define do
|
|
623
|
+
include PageAction
|
|
624
|
+
extend T::Sig
|
|
625
|
+
|
|
626
|
+
sig { returns(T::Boolean) }
|
|
627
|
+
def click? = false
|
|
628
|
+
sig { returns(T::Boolean) }
|
|
629
|
+
def type_text? = false
|
|
630
|
+
sig { returns(T::Boolean) }
|
|
631
|
+
def press? = false
|
|
632
|
+
sig { returns(T::Boolean) }
|
|
633
|
+
def scroll? = false
|
|
634
|
+
sig { returns(T::Boolean) }
|
|
635
|
+
def wait? = false
|
|
636
|
+
sig { returns(T::Boolean) }
|
|
637
|
+
def screenshot? = false
|
|
638
|
+
sig { returns(T::Boolean) }
|
|
639
|
+
def execute_js? = false
|
|
640
|
+
sig { returns(T::Boolean) }
|
|
641
|
+
def scrape? = true
|
|
642
|
+
# @param hash [Hash] deserialized from the native extension
|
|
643
|
+
# @return [self]
|
|
644
|
+
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
645
|
+
def self.from_hash(hash)
|
|
646
|
+
new
|
|
647
|
+
end
|
|
648
|
+
end
|
|
649
|
+
end
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
+
# alef:hash:93297a4dfefbe27a3aa25aae988b939f292098b0ce5ca75dcbafac1118e3cdd6
|
|
3
|
+
# To regenerate: alef generate
|
|
4
|
+
# To verify freshness: alef verify
|
|
5
|
+
# frozen_string_literal: true
|
|
6
|
+
|
|
7
|
+
module Crawlberg
|
|
8
|
+
## The version string for this package.
|
|
9
|
+
VERSION = "1.11.0"
|
|
10
|
+
end
|
data/lib/crawlberg.rb
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
+
# alef:hash:ed4c2274eb4834de21c62fc789eebe8acea49536e10970e2634ab6608966ec3d
|
|
3
|
+
# To regenerate: alef generate
|
|
4
|
+
# To verify freshness: alef verify
|
|
5
|
+
# frozen_string_literal: true
|
|
6
|
+
|
|
7
|
+
require_relative "crawlberg/version"
|
|
8
|
+
require_relative "crawlberg/native"
|
|
9
|
+
|
|
10
|
+
# Top-level namespace for the Crawlberg Ruby binding.
|
|
11
|
+
#
|
|
12
|
+
# All type and function symbols are re-exported from the native extension
|
|
13
|
+
# loaded via `require_relative "crawlberg/native"`. See
|
|
14
|
+
# {file:README.md} for usage examples and the upstream documentation
|
|
15
|
+
# for the full API reference.
|
|
16
|
+
module Crawlberg
|
|
17
|
+
# Re-export all types and functions from native extension
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# Keep generated types under Crawlberg. Publishing generated constants
|
|
21
|
+
# into Object makes generic names collide with unrelated Ruby libraries.
|