crawlberg 1.0.1 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ext/crawlberg_rb/native/Cargo.lock +68 -32
- data/ext/crawlberg_rb/native/Cargo.toml +2 -2
- data/ext/crawlberg_rb/src/lib.rs +36 -99
- data/lib/crawlberg/native.rb +81 -39
- data/lib/crawlberg/version.rb +2 -2
- data/lib/crawlberg.rb +10 -1
- data/lib/crawlberg_rb.so +0 -0
- data/sig/types.rbs +372 -372
- metadata +2 -2
data/lib/crawlberg/native.rb
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
-
# alef:hash:
|
|
2
|
+
# alef:hash:e0600612767fdd3d51e9128afdb0ffcd1fb490e9942f3839a5120ab7a4e3b658
|
|
3
3
|
# To regenerate: alef generate
|
|
4
4
|
# To verify freshness: alef verify --exit-code
|
|
5
5
|
# frozen_string_literal: true
|
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
require "json"
|
|
8
8
|
require "sorbet-runtime"
|
|
9
9
|
require "crawlberg_rb"
|
|
10
|
+
|
|
10
11
|
module Crawlberg
|
|
11
12
|
# Authentication configuration.
|
|
12
13
|
module AuthConfig
|
|
@@ -22,10 +23,14 @@ module Crawlberg
|
|
|
22
23
|
def self.from_hash(hash)
|
|
23
24
|
discriminator = hash[:type] || hash["type"]
|
|
24
25
|
case discriminator
|
|
25
|
-
when "basic"
|
|
26
|
-
|
|
27
|
-
when "
|
|
28
|
-
|
|
26
|
+
when "basic"
|
|
27
|
+
AuthConfigBasic.from_hash(hash)
|
|
28
|
+
when "bearer"
|
|
29
|
+
AuthConfigBearer.from_hash(hash)
|
|
30
|
+
when "header"
|
|
31
|
+
AuthConfigHeader.from_hash(hash)
|
|
32
|
+
else
|
|
33
|
+
raise "Unknown discriminator: #{discriminator}"
|
|
29
34
|
end
|
|
30
35
|
end
|
|
31
36
|
end
|
|
@@ -36,10 +41,12 @@ module Crawlberg
|
|
|
36
41
|
|
|
37
42
|
# Username sent in the `Authorization: Basic` header.
|
|
38
43
|
sig { returns(String) }
|
|
39
|
-
|
|
44
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
45
|
+
def username = super
|
|
40
46
|
# Password sent in the `Authorization: Basic` header.
|
|
41
47
|
sig { returns(String) }
|
|
42
|
-
|
|
48
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
49
|
+
def password = super
|
|
43
50
|
sig { returns(T::Boolean) }
|
|
44
51
|
def basic? = true
|
|
45
52
|
sig { returns(T::Boolean) }
|
|
@@ -60,7 +67,8 @@ module Crawlberg
|
|
|
60
67
|
|
|
61
68
|
# Token sent in the `Authorization: Bearer` header.
|
|
62
69
|
sig { returns(String) }
|
|
63
|
-
|
|
70
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
71
|
+
def token = super
|
|
64
72
|
sig { returns(T::Boolean) }
|
|
65
73
|
def basic? = false
|
|
66
74
|
sig { returns(T::Boolean) }
|
|
@@ -81,10 +89,12 @@ module Crawlberg
|
|
|
81
89
|
|
|
82
90
|
# HTTP header name to set on each request.
|
|
83
91
|
sig { returns(String) }
|
|
84
|
-
|
|
92
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
93
|
+
def name = super
|
|
85
94
|
# HTTP header value to send.
|
|
86
95
|
sig { returns(String) }
|
|
87
|
-
|
|
96
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
97
|
+
def value = super
|
|
88
98
|
sig { returns(T::Boolean) }
|
|
89
99
|
def basic? = false
|
|
90
100
|
sig { returns(T::Boolean) }
|
|
@@ -120,10 +130,14 @@ module Crawlberg
|
|
|
120
130
|
def self.from_hash(hash)
|
|
121
131
|
discriminator = hash[:type] || hash["type"]
|
|
122
132
|
case discriminator
|
|
123
|
-
when "page"
|
|
124
|
-
|
|
125
|
-
when "
|
|
126
|
-
|
|
133
|
+
when "page"
|
|
134
|
+
CrawlEventPage.from_hash(hash)
|
|
135
|
+
when "error"
|
|
136
|
+
CrawlEventError.from_hash(hash)
|
|
137
|
+
when "complete"
|
|
138
|
+
CrawlEventComplete.from_hash(hash)
|
|
139
|
+
else
|
|
140
|
+
raise "Unknown discriminator: #{discriminator}"
|
|
127
141
|
end
|
|
128
142
|
end
|
|
129
143
|
end
|
|
@@ -134,7 +148,8 @@ module Crawlberg
|
|
|
134
148
|
|
|
135
149
|
# The crawled page result.
|
|
136
150
|
sig { returns(CrawlPageResult) }
|
|
137
|
-
|
|
151
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
152
|
+
def result = super
|
|
138
153
|
sig { returns(T::Boolean) }
|
|
139
154
|
def page? = true
|
|
140
155
|
sig { returns(T::Boolean) }
|
|
@@ -155,10 +170,12 @@ module Crawlberg
|
|
|
155
170
|
|
|
156
171
|
# The URL that failed.
|
|
157
172
|
sig { returns(String) }
|
|
158
|
-
|
|
173
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
174
|
+
def url = super
|
|
159
175
|
# The error message.
|
|
160
176
|
sig { returns(String) }
|
|
161
|
-
|
|
177
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
178
|
+
def error = super
|
|
162
179
|
sig { returns(T::Boolean) }
|
|
163
180
|
def page? = false
|
|
164
181
|
sig { returns(T::Boolean) }
|
|
@@ -179,7 +196,8 @@ module Crawlberg
|
|
|
179
196
|
|
|
180
197
|
# Total number of pages crawled.
|
|
181
198
|
sig { returns(Integer) }
|
|
182
|
-
|
|
199
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
200
|
+
def pages_crawled = super
|
|
183
201
|
sig { returns(T::Boolean) }
|
|
184
202
|
def page? = false
|
|
185
203
|
sig { returns(T::Boolean) }
|
|
@@ -213,15 +231,24 @@ module Crawlberg
|
|
|
213
231
|
def self.from_hash(hash)
|
|
214
232
|
discriminator = hash[:type] || hash["type"]
|
|
215
233
|
case discriminator
|
|
216
|
-
when "click"
|
|
217
|
-
|
|
218
|
-
when "
|
|
219
|
-
|
|
220
|
-
when "
|
|
221
|
-
|
|
222
|
-
when "
|
|
223
|
-
|
|
224
|
-
|
|
234
|
+
when "click"
|
|
235
|
+
PageActionClick.from_hash(hash)
|
|
236
|
+
when "type"
|
|
237
|
+
PageActionTypeText.from_hash(hash)
|
|
238
|
+
when "press"
|
|
239
|
+
PageActionPress.from_hash(hash)
|
|
240
|
+
when "scroll"
|
|
241
|
+
PageActionScroll.from_hash(hash)
|
|
242
|
+
when "wait"
|
|
243
|
+
PageActionWait.from_hash(hash)
|
|
244
|
+
when "screenshot"
|
|
245
|
+
PageActionScreenshot.from_hash(hash)
|
|
246
|
+
when "executeJs"
|
|
247
|
+
PageActionExecuteJs.from_hash(hash)
|
|
248
|
+
when "scrape"
|
|
249
|
+
PageActionScrape.from_hash(hash)
|
|
250
|
+
else
|
|
251
|
+
raise "Unknown discriminator: #{discriminator}"
|
|
225
252
|
end
|
|
226
253
|
end
|
|
227
254
|
end
|
|
@@ -232,7 +259,8 @@ module Crawlberg
|
|
|
232
259
|
|
|
233
260
|
# CSS selector for the element to click.
|
|
234
261
|
sig { returns(String) }
|
|
235
|
-
|
|
262
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
263
|
+
def selector = super
|
|
236
264
|
sig { returns(T::Boolean) }
|
|
237
265
|
def click? = true
|
|
238
266
|
sig { returns(T::Boolean) }
|
|
@@ -263,10 +291,12 @@ module Crawlberg
|
|
|
263
291
|
|
|
264
292
|
# CSS selector for the input element.
|
|
265
293
|
sig { returns(String) }
|
|
266
|
-
|
|
294
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
295
|
+
def selector = super
|
|
267
296
|
# Text to type into the element.
|
|
268
297
|
sig { returns(String) }
|
|
269
|
-
|
|
298
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
299
|
+
def text = super
|
|
270
300
|
sig { returns(T::Boolean) }
|
|
271
301
|
def click? = false
|
|
272
302
|
sig { returns(T::Boolean) }
|
|
@@ -297,7 +327,8 @@ module Crawlberg
|
|
|
297
327
|
|
|
298
328
|
# Key name to press.
|
|
299
329
|
sig { returns(String) }
|
|
300
|
-
|
|
330
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
331
|
+
def key = super
|
|
301
332
|
sig { returns(T::Boolean) }
|
|
302
333
|
def click? = false
|
|
303
334
|
sig { returns(T::Boolean) }
|
|
@@ -328,13 +359,16 @@ module Crawlberg
|
|
|
328
359
|
|
|
329
360
|
# Direction to scroll.
|
|
330
361
|
sig { returns(ScrollDirection) }
|
|
331
|
-
|
|
362
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
363
|
+
def direction = super
|
|
332
364
|
# Optional CSS selector for a scrollable element. Scrolls the page if absent.
|
|
333
365
|
sig { returns(T.nilable(String)) }
|
|
334
|
-
|
|
366
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
367
|
+
def selector = super
|
|
335
368
|
# Optional pixel amount to scroll. Uses a default if absent.
|
|
336
369
|
sig { returns(T.nilable(Integer)) }
|
|
337
|
-
|
|
370
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
371
|
+
def amount = super
|
|
338
372
|
sig { returns(T::Boolean) }
|
|
339
373
|
def click? = false
|
|
340
374
|
sig { returns(T::Boolean) }
|
|
@@ -355,7 +389,11 @@ module Crawlberg
|
|
|
355
389
|
# @return [self]
|
|
356
390
|
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
357
391
|
def self.from_hash(hash)
|
|
358
|
-
new(
|
|
392
|
+
new(
|
|
393
|
+
direction: hash[:direction] || hash["direction"],
|
|
394
|
+
selector: hash[:selector] || hash["selector"],
|
|
395
|
+
amount: hash[:amount] || hash["amount"]
|
|
396
|
+
)
|
|
359
397
|
end
|
|
360
398
|
end
|
|
361
399
|
## Wait for a duration or for an element to appear.
|
|
@@ -365,10 +403,12 @@ module Crawlberg
|
|
|
365
403
|
|
|
366
404
|
# Milliseconds to wait. Ignored if `selector` is provided.
|
|
367
405
|
sig { returns(T.nilable(Integer)) }
|
|
368
|
-
|
|
406
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
407
|
+
def milliseconds = super
|
|
369
408
|
# CSS selector to wait for.
|
|
370
409
|
sig { returns(T.nilable(String)) }
|
|
371
|
-
|
|
410
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
411
|
+
def selector = super
|
|
372
412
|
sig { returns(T::Boolean) }
|
|
373
413
|
def click? = false
|
|
374
414
|
sig { returns(T::Boolean) }
|
|
@@ -403,7 +443,8 @@ module Crawlberg
|
|
|
403
443
|
# `full_page` (snake_case) alias so language bindings and fixtures can
|
|
404
444
|
# use either convention without error.
|
|
405
445
|
sig { returns(T.nilable(T::Boolean)) }
|
|
406
|
-
|
|
446
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
447
|
+
def full_page = super
|
|
407
448
|
sig { returns(T::Boolean) }
|
|
408
449
|
def click? = false
|
|
409
450
|
sig { returns(T::Boolean) }
|
|
@@ -439,7 +480,8 @@ module Crawlberg
|
|
|
439
480
|
|
|
440
481
|
# JavaScript source code to execute. Max 1 MB.
|
|
441
482
|
sig { returns(String) }
|
|
442
|
-
|
|
483
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
484
|
+
def script = super
|
|
443
485
|
sig { returns(T::Boolean) }
|
|
444
486
|
def click? = false
|
|
445
487
|
sig { returns(T::Boolean) }
|
data/lib/crawlberg/version.rb
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
# This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
-
# alef:hash:
|
|
2
|
+
# alef:hash:e0600612767fdd3d51e9128afdb0ffcd1fb490e9942f3839a5120ab7a4e3b658
|
|
3
3
|
# To regenerate: alef generate
|
|
4
4
|
# To verify freshness: alef verify --exit-code
|
|
5
5
|
# frozen_string_literal: true
|
|
6
6
|
|
|
7
7
|
module Crawlberg
|
|
8
8
|
## The version string for this package.
|
|
9
|
-
VERSION = "1.0.
|
|
9
|
+
VERSION = "1.0.3"
|
|
10
10
|
end
|
data/lib/crawlberg.rb
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
-
# alef:hash:
|
|
2
|
+
# alef:hash:e0600612767fdd3d51e9128afdb0ffcd1fb490e9942f3839a5120ab7a4e3b658
|
|
3
3
|
# To regenerate: alef generate
|
|
4
4
|
# To verify freshness: alef verify --exit-code
|
|
5
5
|
# frozen_string_literal: true
|
|
@@ -16,3 +16,12 @@ require_relative "crawlberg/native"
|
|
|
16
16
|
module Crawlberg
|
|
17
17
|
# Re-export all types and functions from native extension
|
|
18
18
|
end
|
|
19
|
+
|
|
20
|
+
# Bring top-level Crawlberg classes into the global namespace so callers
|
|
21
|
+
# (and the generated e2e suite) can reference them unqualified. The native
|
|
22
|
+
# extension has already been required above, so every type constant is defined
|
|
23
|
+
# under Crawlberg by this point.
|
|
24
|
+
Crawlberg.constants.each do |const_name|
|
|
25
|
+
value = Crawlberg.const_get(const_name)
|
|
26
|
+
::Object.const_set(const_name, value) if value.is_a?(Module) && !::Object.const_defined?(const_name)
|
|
27
|
+
end
|
data/lib/crawlberg_rb.so
CHANGED
|
Binary file
|