crawlberg 1.2.1 → 1.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +5 -0
- data/ext/crawlberg_rb/native/Cargo.lock +111 -142
- data/ext/crawlberg_rb/native/Cargo.toml +19 -2
- data/ext/crawlberg_rb/native/extconf.rb +5 -0
- data/ext/crawlberg_rb/src/lib.rs +223 -30
- data/lib/crawlberg/native.rb +48 -97
- data/lib/crawlberg/version.rb +3 -3
- data/lib/crawlberg.rb +2 -2
- data/sig/types.rbs +456 -449
- metadata +2 -3
- data/lib/crawlberg_rb.so +0 -0
data/lib/crawlberg/native.rb
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
-
# alef:hash:
|
|
2
|
+
# alef:hash:468e05d35782efa941278f63a0e1f24595cf4a534fa469e7747a18fe91938bbf
|
|
3
3
|
# To regenerate: alef generate
|
|
4
|
-
# To verify freshness: alef verify
|
|
4
|
+
# To verify freshness: alef verify
|
|
5
5
|
# frozen_string_literal: true
|
|
6
6
|
|
|
7
7
|
require "json"
|
|
@@ -18,7 +18,7 @@ library_candidates = dlexts.flat_map do |dlext|
|
|
|
18
18
|
File.join(__dir__, "..", extension_name, ruby_abi, "#{extension_name}.#{dlext}"),
|
|
19
19
|
File.join(__dir__, "..", extension_name, ruby_abi, "lib#{extension_name}.#{dlext}"),
|
|
20
20
|
File.join(__dir__, "..", "#{extension_name}.#{dlext}"),
|
|
21
|
-
File.join(__dir__, "..", "lib#{extension_name}.#{dlext}")
|
|
21
|
+
File.join(__dir__, "..", "lib#{extension_name}.#{dlext}"),
|
|
22
22
|
]
|
|
23
23
|
end
|
|
24
24
|
|
|
@@ -34,7 +34,6 @@ unless native_extension
|
|
|
34
34
|
end
|
|
35
35
|
|
|
36
36
|
require native_extension
|
|
37
|
-
|
|
38
37
|
module Crawlberg
|
|
39
38
|
# Authentication configuration.
|
|
40
39
|
module AuthConfig
|
|
@@ -50,14 +49,10 @@ module Crawlberg
|
|
|
50
49
|
def self.from_hash(hash)
|
|
51
50
|
discriminator = hash[:type] || hash["type"]
|
|
52
51
|
case discriminator
|
|
53
|
-
when "basic"
|
|
54
|
-
|
|
55
|
-
when "
|
|
56
|
-
|
|
57
|
-
when "header"
|
|
58
|
-
AuthConfigHeader.from_hash(hash)
|
|
59
|
-
else
|
|
60
|
-
raise "Unknown discriminator: #{discriminator}"
|
|
52
|
+
when "basic" then AuthConfigBasic.from_hash(hash)
|
|
53
|
+
when "bearer" then AuthConfigBearer.from_hash(hash)
|
|
54
|
+
when "header" then AuthConfigHeader.from_hash(hash)
|
|
55
|
+
else raise "Unknown discriminator: #{discriminator}"
|
|
61
56
|
end
|
|
62
57
|
end
|
|
63
58
|
end
|
|
@@ -68,12 +63,10 @@ module Crawlberg
|
|
|
68
63
|
|
|
69
64
|
# Username sent in the `Authorization: Basic` header.
|
|
70
65
|
sig { returns(String) }
|
|
71
|
-
# rubocop:disable Lint/UselessMethodDefinition
|
|
72
|
-
def username = super
|
|
66
|
+
def username = super # rubocop:disable Lint/UselessMethodDefinition
|
|
73
67
|
# Password sent in the `Authorization: Basic` header.
|
|
74
68
|
sig { returns(String) }
|
|
75
|
-
# rubocop:disable Lint/UselessMethodDefinition
|
|
76
|
-
def password = super
|
|
69
|
+
def password = super # rubocop:disable Lint/UselessMethodDefinition
|
|
77
70
|
sig { returns(T::Boolean) }
|
|
78
71
|
def basic? = true
|
|
79
72
|
sig { returns(T::Boolean) }
|
|
@@ -94,8 +87,7 @@ module Crawlberg
|
|
|
94
87
|
|
|
95
88
|
# Token sent in the `Authorization: Bearer` header.
|
|
96
89
|
sig { returns(String) }
|
|
97
|
-
# rubocop:disable Lint/UselessMethodDefinition
|
|
98
|
-
def token = super
|
|
90
|
+
def token = super # rubocop:disable Lint/UselessMethodDefinition
|
|
99
91
|
sig { returns(T::Boolean) }
|
|
100
92
|
def basic? = false
|
|
101
93
|
sig { returns(T::Boolean) }
|
|
@@ -116,12 +108,10 @@ module Crawlberg
|
|
|
116
108
|
|
|
117
109
|
# HTTP header name to set on each request.
|
|
118
110
|
sig { returns(String) }
|
|
119
|
-
# rubocop:disable Lint/UselessMethodDefinition
|
|
120
|
-
def name = super
|
|
111
|
+
def name = super # rubocop:disable Lint/UselessMethodDefinition
|
|
121
112
|
# HTTP header value to send.
|
|
122
113
|
sig { returns(String) }
|
|
123
|
-
# rubocop:disable Lint/UselessMethodDefinition
|
|
124
|
-
def value = super
|
|
114
|
+
def value = super # rubocop:disable Lint/UselessMethodDefinition
|
|
125
115
|
sig { returns(T::Boolean) }
|
|
126
116
|
def basic? = false
|
|
127
117
|
sig { returns(T::Boolean) }
|
|
@@ -157,14 +147,10 @@ module Crawlberg
|
|
|
157
147
|
def self.from_hash(hash)
|
|
158
148
|
discriminator = hash[:type] || hash["type"]
|
|
159
149
|
case discriminator
|
|
160
|
-
when "page"
|
|
161
|
-
|
|
162
|
-
when "
|
|
163
|
-
|
|
164
|
-
when "complete"
|
|
165
|
-
CrawlEventComplete.from_hash(hash)
|
|
166
|
-
else
|
|
167
|
-
raise "Unknown discriminator: #{discriminator}"
|
|
150
|
+
when "page" then CrawlEventPage.from_hash(hash)
|
|
151
|
+
when "error" then CrawlEventError.from_hash(hash)
|
|
152
|
+
when "complete" then CrawlEventComplete.from_hash(hash)
|
|
153
|
+
else raise "Unknown discriminator: #{discriminator}"
|
|
168
154
|
end
|
|
169
155
|
end
|
|
170
156
|
end
|
|
@@ -175,8 +161,7 @@ module Crawlberg
|
|
|
175
161
|
|
|
176
162
|
# The crawled page result.
|
|
177
163
|
sig { returns(CrawlPageResult) }
|
|
178
|
-
# rubocop:disable Lint/UselessMethodDefinition
|
|
179
|
-
def result = super
|
|
164
|
+
def result = super # rubocop:disable Lint/UselessMethodDefinition
|
|
180
165
|
sig { returns(T::Boolean) }
|
|
181
166
|
def page? = true
|
|
182
167
|
sig { returns(T::Boolean) }
|
|
@@ -197,12 +182,10 @@ module Crawlberg
|
|
|
197
182
|
|
|
198
183
|
# The URL that failed.
|
|
199
184
|
sig { returns(String) }
|
|
200
|
-
# rubocop:disable Lint/UselessMethodDefinition
|
|
201
|
-
def url = super
|
|
185
|
+
def url = super # rubocop:disable Lint/UselessMethodDefinition
|
|
202
186
|
# The error message.
|
|
203
187
|
sig { returns(String) }
|
|
204
|
-
# rubocop:disable Lint/UselessMethodDefinition
|
|
205
|
-
def error = super
|
|
188
|
+
def error = super # rubocop:disable Lint/UselessMethodDefinition
|
|
206
189
|
sig { returns(T::Boolean) }
|
|
207
190
|
def page? = false
|
|
208
191
|
sig { returns(T::Boolean) }
|
|
@@ -223,8 +206,7 @@ module Crawlberg
|
|
|
223
206
|
|
|
224
207
|
# Total number of pages crawled.
|
|
225
208
|
sig { returns(Integer) }
|
|
226
|
-
# rubocop:disable Lint/UselessMethodDefinition
|
|
227
|
-
def pages_crawled = super
|
|
209
|
+
def pages_crawled = super # rubocop:disable Lint/UselessMethodDefinition
|
|
228
210
|
sig { returns(T::Boolean) }
|
|
229
211
|
def page? = false
|
|
230
212
|
sig { returns(T::Boolean) }
|
|
@@ -258,24 +240,15 @@ module Crawlberg
|
|
|
258
240
|
def self.from_hash(hash)
|
|
259
241
|
discriminator = hash[:type] || hash["type"]
|
|
260
242
|
case discriminator
|
|
261
|
-
when "click"
|
|
262
|
-
|
|
263
|
-
when "
|
|
264
|
-
|
|
265
|
-
when "
|
|
266
|
-
|
|
267
|
-
when "
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
PageActionWait.from_hash(hash)
|
|
271
|
-
when "screenshot"
|
|
272
|
-
PageActionScreenshot.from_hash(hash)
|
|
273
|
-
when "executeJs"
|
|
274
|
-
PageActionExecuteJs.from_hash(hash)
|
|
275
|
-
when "scrape"
|
|
276
|
-
PageActionScrape.from_hash(hash)
|
|
277
|
-
else
|
|
278
|
-
raise "Unknown discriminator: #{discriminator}"
|
|
243
|
+
when "click" then PageActionClick.from_hash(hash)
|
|
244
|
+
when "type" then PageActionTypeText.from_hash(hash)
|
|
245
|
+
when "press" then PageActionPress.from_hash(hash)
|
|
246
|
+
when "scroll" then PageActionScroll.from_hash(hash)
|
|
247
|
+
when "wait" then PageActionWait.from_hash(hash)
|
|
248
|
+
when "screenshot" then PageActionScreenshot.from_hash(hash)
|
|
249
|
+
when "executeJs" then PageActionExecuteJs.from_hash(hash)
|
|
250
|
+
when "scrape" then PageActionScrape.from_hash(hash)
|
|
251
|
+
else raise "Unknown discriminator: #{discriminator}"
|
|
279
252
|
end
|
|
280
253
|
end
|
|
281
254
|
end
|
|
@@ -286,8 +259,7 @@ module Crawlberg
|
|
|
286
259
|
|
|
287
260
|
# CSS selector for the element to click.
|
|
288
261
|
sig { returns(String) }
|
|
289
|
-
# rubocop:disable Lint/UselessMethodDefinition
|
|
290
|
-
def selector = super
|
|
262
|
+
def selector = super # rubocop:disable Lint/UselessMethodDefinition
|
|
291
263
|
sig { returns(T::Boolean) }
|
|
292
264
|
def click? = true
|
|
293
265
|
sig { returns(T::Boolean) }
|
|
@@ -318,12 +290,10 @@ module Crawlberg
|
|
|
318
290
|
|
|
319
291
|
# CSS selector for the input element.
|
|
320
292
|
sig { returns(String) }
|
|
321
|
-
# rubocop:disable Lint/UselessMethodDefinition
|
|
322
|
-
def selector = super
|
|
293
|
+
def selector = super # rubocop:disable Lint/UselessMethodDefinition
|
|
323
294
|
# Text to type into the element.
|
|
324
295
|
sig { returns(String) }
|
|
325
|
-
# rubocop:disable Lint/UselessMethodDefinition
|
|
326
|
-
def text = super
|
|
296
|
+
def text = super # rubocop:disable Lint/UselessMethodDefinition
|
|
327
297
|
sig { returns(T::Boolean) }
|
|
328
298
|
def click? = false
|
|
329
299
|
sig { returns(T::Boolean) }
|
|
@@ -354,8 +324,7 @@ module Crawlberg
|
|
|
354
324
|
|
|
355
325
|
# Key name to press.
|
|
356
326
|
sig { returns(String) }
|
|
357
|
-
# rubocop:disable Lint/UselessMethodDefinition
|
|
358
|
-
def key = super
|
|
327
|
+
def key = super # rubocop:disable Lint/UselessMethodDefinition
|
|
359
328
|
sig { returns(T::Boolean) }
|
|
360
329
|
def click? = false
|
|
361
330
|
sig { returns(T::Boolean) }
|
|
@@ -386,16 +355,13 @@ module Crawlberg
|
|
|
386
355
|
|
|
387
356
|
# Direction to scroll.
|
|
388
357
|
sig { returns(ScrollDirection) }
|
|
389
|
-
# rubocop:disable Lint/UselessMethodDefinition
|
|
390
|
-
def direction = super
|
|
358
|
+
def direction = super # rubocop:disable Lint/UselessMethodDefinition
|
|
391
359
|
# Optional CSS selector for a scrollable element. Scrolls the page if absent.
|
|
392
360
|
sig { returns(T.nilable(String)) }
|
|
393
|
-
# rubocop:disable Lint/UselessMethodDefinition
|
|
394
|
-
def selector = super
|
|
361
|
+
def selector = super # rubocop:disable Lint/UselessMethodDefinition
|
|
395
362
|
# Optional pixel amount to scroll. Uses a default if absent.
|
|
396
363
|
sig { returns(T.nilable(Integer)) }
|
|
397
|
-
# rubocop:disable Lint/UselessMethodDefinition
|
|
398
|
-
def amount = super
|
|
364
|
+
def amount = super # rubocop:disable Lint/UselessMethodDefinition
|
|
399
365
|
sig { returns(T::Boolean) }
|
|
400
366
|
def click? = false
|
|
401
367
|
sig { returns(T::Boolean) }
|
|
@@ -416,11 +382,7 @@ module Crawlberg
|
|
|
416
382
|
# @return [self]
|
|
417
383
|
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
418
384
|
def self.from_hash(hash)
|
|
419
|
-
new(
|
|
420
|
-
direction: hash[:direction] || hash["direction"],
|
|
421
|
-
selector: hash[:selector] || hash["selector"],
|
|
422
|
-
amount: hash[:amount] || hash["amount"]
|
|
423
|
-
)
|
|
385
|
+
new(direction: hash[:direction] || hash["direction"], selector: hash[:selector] || hash["selector"], amount: hash[:amount] || hash["amount"])
|
|
424
386
|
end
|
|
425
387
|
end
|
|
426
388
|
## Wait for a duration or for an element to appear.
|
|
@@ -430,12 +392,10 @@ module Crawlberg
|
|
|
430
392
|
|
|
431
393
|
# Milliseconds to wait. Ignored if `selector` is provided.
|
|
432
394
|
sig { returns(T.nilable(Integer)) }
|
|
433
|
-
# rubocop:disable Lint/UselessMethodDefinition
|
|
434
|
-
def milliseconds = super
|
|
395
|
+
def milliseconds = super # rubocop:disable Lint/UselessMethodDefinition
|
|
435
396
|
# CSS selector to wait for.
|
|
436
397
|
sig { returns(T.nilable(String)) }
|
|
437
|
-
# rubocop:disable Lint/UselessMethodDefinition
|
|
438
|
-
def selector = super
|
|
398
|
+
def selector = super # rubocop:disable Lint/UselessMethodDefinition
|
|
439
399
|
sig { returns(T::Boolean) }
|
|
440
400
|
def click? = false
|
|
441
401
|
sig { returns(T::Boolean) }
|
|
@@ -470,8 +430,7 @@ module Crawlberg
|
|
|
470
430
|
# `full_page` (snake_case) alias so language bindings and fixtures can
|
|
471
431
|
# use either convention without error.
|
|
472
432
|
sig { returns(T.nilable(T::Boolean)) }
|
|
473
|
-
# rubocop:disable Lint/UselessMethodDefinition
|
|
474
|
-
def full_page = super
|
|
433
|
+
def full_page = super # rubocop:disable Lint/UselessMethodDefinition
|
|
475
434
|
sig { returns(T::Boolean) }
|
|
476
435
|
def click? = false
|
|
477
436
|
sig { returns(T::Boolean) }
|
|
@@ -507,8 +466,7 @@ module Crawlberg
|
|
|
507
466
|
|
|
508
467
|
# JavaScript source code to execute. Max 1 MB.
|
|
509
468
|
sig { returns(String) }
|
|
510
|
-
# rubocop:disable Lint/UselessMethodDefinition
|
|
511
|
-
def script = super
|
|
469
|
+
def script = super # rubocop:disable Lint/UselessMethodDefinition
|
|
512
470
|
sig { returns(T::Boolean) }
|
|
513
471
|
def click? = false
|
|
514
472
|
sig { returns(T::Boolean) }
|
|
@@ -591,14 +549,10 @@ module Crawlberg
|
|
|
591
549
|
def self.from_hash(hash)
|
|
592
550
|
discriminator = hash[:type] || hash["type"]
|
|
593
551
|
case discriminator
|
|
594
|
-
when "exact"
|
|
595
|
-
|
|
596
|
-
when "
|
|
597
|
-
|
|
598
|
-
when "cidr"
|
|
599
|
-
HostMatcherCidr.from_hash(hash)
|
|
600
|
-
else
|
|
601
|
-
raise "Unknown discriminator: #{discriminator}"
|
|
552
|
+
when "exact" then HostMatcherExact.from_hash(hash)
|
|
553
|
+
when "suffix" then HostMatcherSuffix.from_hash(hash)
|
|
554
|
+
when "cidr" then HostMatcherCidr.from_hash(hash)
|
|
555
|
+
else raise "Unknown discriminator: #{discriminator}"
|
|
602
556
|
end
|
|
603
557
|
end
|
|
604
558
|
end
|
|
@@ -609,8 +563,7 @@ module Crawlberg
|
|
|
609
563
|
|
|
610
564
|
# The hostname to match.
|
|
611
565
|
sig { returns(String) }
|
|
612
|
-
# rubocop:disable Lint/UselessMethodDefinition
|
|
613
|
-
def value = super
|
|
566
|
+
def value = super # rubocop:disable Lint/UselessMethodDefinition
|
|
614
567
|
sig { returns(T::Boolean) }
|
|
615
568
|
def exact? = true
|
|
616
569
|
sig { returns(T::Boolean) }
|
|
@@ -631,8 +584,7 @@ module Crawlberg
|
|
|
631
584
|
|
|
632
585
|
# The dot-prefixed suffix to match. A leading dot is optional.
|
|
633
586
|
sig { returns(String) }
|
|
634
|
-
# rubocop:disable Lint/UselessMethodDefinition
|
|
635
|
-
def value = super
|
|
587
|
+
def value = super # rubocop:disable Lint/UselessMethodDefinition
|
|
636
588
|
sig { returns(T::Boolean) }
|
|
637
589
|
def exact? = false
|
|
638
590
|
sig { returns(T::Boolean) }
|
|
@@ -654,8 +606,7 @@ module Crawlberg
|
|
|
654
606
|
# The CIDR block. Validated when built through [`HostMatcher::cidr`] or
|
|
655
607
|
# deserialization.
|
|
656
608
|
sig { returns(String) }
|
|
657
|
-
# rubocop:disable Lint/UselessMethodDefinition
|
|
658
|
-
def value = super
|
|
609
|
+
def value = super # rubocop:disable Lint/UselessMethodDefinition
|
|
659
610
|
sig { returns(T::Boolean) }
|
|
660
611
|
def exact? = false
|
|
661
612
|
sig { returns(T::Boolean) }
|
data/lib/crawlberg/version.rb
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
# This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
-
# alef:hash:
|
|
2
|
+
# alef:hash:9716830c37f9c6de3924a6ebdc2f8f077d296794b443399aa1d4c4f0d483169a
|
|
3
3
|
# To regenerate: alef generate
|
|
4
|
-
# To verify freshness: alef verify
|
|
4
|
+
# To verify freshness: alef verify
|
|
5
5
|
# frozen_string_literal: true
|
|
6
6
|
|
|
7
7
|
module Crawlberg
|
|
8
8
|
## The version string for this package.
|
|
9
|
-
VERSION = "1.
|
|
9
|
+
VERSION = "1.3.3"
|
|
10
10
|
end
|
data/lib/crawlberg.rb
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
-
# alef:hash:
|
|
2
|
+
# alef:hash:5f14772be973afec2e94293f27da4258e3399457f1e279dc68d4d55a011f6b3a
|
|
3
3
|
# To regenerate: alef generate
|
|
4
|
-
# To verify freshness: alef verify
|
|
4
|
+
# To verify freshness: alef verify
|
|
5
5
|
# frozen_string_literal: true
|
|
6
6
|
|
|
7
7
|
require_relative "crawlberg/version"
|