crawlberg 1.3.3 → 1.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,5 @@
1
1
  # This file is auto-generated by alef — DO NOT EDIT.
2
- # alef:hash:468e05d35782efa941278f63a0e1f24595cf4a534fa469e7747a18fe91938bbf
2
+ # alef:hash:925532ba2f606ce36a0d3444d5c96407ca56a1c55bcd256372a1ea2a383d11e5
3
3
  # To regenerate: alef generate
4
4
  # To verify freshness: alef verify
5
5
  # frozen_string_literal: true
@@ -18,7 +18,7 @@ library_candidates = dlexts.flat_map do |dlext|
18
18
  File.join(__dir__, "..", extension_name, ruby_abi, "#{extension_name}.#{dlext}"),
19
19
  File.join(__dir__, "..", extension_name, ruby_abi, "lib#{extension_name}.#{dlext}"),
20
20
  File.join(__dir__, "..", "#{extension_name}.#{dlext}"),
21
- File.join(__dir__, "..", "lib#{extension_name}.#{dlext}"),
21
+ File.join(__dir__, "..", "lib#{extension_name}.#{dlext}")
22
22
  ]
23
23
  end
24
24
 
@@ -34,6 +34,7 @@ unless native_extension
34
34
  end
35
35
 
36
36
  require native_extension
37
+
37
38
  module Crawlberg
38
39
  # Authentication configuration.
39
40
  module AuthConfig
@@ -49,10 +50,14 @@ module Crawlberg
49
50
  def self.from_hash(hash)
50
51
  discriminator = hash[:type] || hash["type"]
51
52
  case discriminator
52
- when "basic" then AuthConfigBasic.from_hash(hash)
53
- when "bearer" then AuthConfigBearer.from_hash(hash)
54
- when "header" then AuthConfigHeader.from_hash(hash)
55
- else raise "Unknown discriminator: #{discriminator}"
53
+ when "basic"
54
+ AuthConfigBasic.from_hash(hash)
55
+ when "bearer"
56
+ AuthConfigBearer.from_hash(hash)
57
+ when "header"
58
+ AuthConfigHeader.from_hash(hash)
59
+ else
60
+ raise "Unknown discriminator: #{discriminator}"
56
61
  end
57
62
  end
58
63
  end
@@ -63,10 +68,12 @@ module Crawlberg
63
68
 
64
69
  # Username sent in the `Authorization: Basic` header.
65
70
  sig { returns(String) }
66
- def username = super # rubocop:disable Lint/UselessMethodDefinition
71
+ # rubocop:disable Lint/UselessMethodDefinition
72
+ def username = super
67
73
  # Password sent in the `Authorization: Basic` header.
68
74
  sig { returns(String) }
69
- def password = super # rubocop:disable Lint/UselessMethodDefinition
75
+ # rubocop:disable Lint/UselessMethodDefinition
76
+ def password = super
70
77
  sig { returns(T::Boolean) }
71
78
  def basic? = true
72
79
  sig { returns(T::Boolean) }
@@ -87,7 +94,8 @@ module Crawlberg
87
94
 
88
95
  # Token sent in the `Authorization: Bearer` header.
89
96
  sig { returns(String) }
90
- def token = super # rubocop:disable Lint/UselessMethodDefinition
97
+ # rubocop:disable Lint/UselessMethodDefinition
98
+ def token = super
91
99
  sig { returns(T::Boolean) }
92
100
  def basic? = false
93
101
  sig { returns(T::Boolean) }
@@ -108,10 +116,12 @@ module Crawlberg
108
116
 
109
117
  # HTTP header name to set on each request.
110
118
  sig { returns(String) }
111
- def name = super # rubocop:disable Lint/UselessMethodDefinition
119
+ # rubocop:disable Lint/UselessMethodDefinition
120
+ def name = super
112
121
  # HTTP header value to send.
113
122
  sig { returns(String) }
114
- def value = super # rubocop:disable Lint/UselessMethodDefinition
123
+ # rubocop:disable Lint/UselessMethodDefinition
124
+ def value = super
115
125
  sig { returns(T::Boolean) }
116
126
  def basic? = false
117
127
  sig { returns(T::Boolean) }
@@ -147,10 +157,14 @@ module Crawlberg
147
157
  def self.from_hash(hash)
148
158
  discriminator = hash[:type] || hash["type"]
149
159
  case discriminator
150
- when "page" then CrawlEventPage.from_hash(hash)
151
- when "error" then CrawlEventError.from_hash(hash)
152
- when "complete" then CrawlEventComplete.from_hash(hash)
153
- else raise "Unknown discriminator: #{discriminator}"
160
+ when "page"
161
+ CrawlEventPage.from_hash(hash)
162
+ when "error"
163
+ CrawlEventError.from_hash(hash)
164
+ when "complete"
165
+ CrawlEventComplete.from_hash(hash)
166
+ else
167
+ raise "Unknown discriminator: #{discriminator}"
154
168
  end
155
169
  end
156
170
  end
@@ -161,7 +175,8 @@ module Crawlberg
161
175
 
162
176
  # The crawled page result.
163
177
  sig { returns(CrawlPageResult) }
164
- def result = super # rubocop:disable Lint/UselessMethodDefinition
178
+ # rubocop:disable Lint/UselessMethodDefinition
179
+ def result = super
165
180
  sig { returns(T::Boolean) }
166
181
  def page? = true
167
182
  sig { returns(T::Boolean) }
@@ -182,10 +197,12 @@ module Crawlberg
182
197
 
183
198
  # The URL that failed.
184
199
  sig { returns(String) }
185
- def url = super # rubocop:disable Lint/UselessMethodDefinition
200
+ # rubocop:disable Lint/UselessMethodDefinition
201
+ def url = super
186
202
  # The error message.
187
203
  sig { returns(String) }
188
- def error = super # rubocop:disable Lint/UselessMethodDefinition
204
+ # rubocop:disable Lint/UselessMethodDefinition
205
+ def error = super
189
206
  sig { returns(T::Boolean) }
190
207
  def page? = false
191
208
  sig { returns(T::Boolean) }
@@ -206,7 +223,8 @@ module Crawlberg
206
223
 
207
224
  # Total number of pages crawled.
208
225
  sig { returns(Integer) }
209
- def pages_crawled = super # rubocop:disable Lint/UselessMethodDefinition
226
+ # rubocop:disable Lint/UselessMethodDefinition
227
+ def pages_crawled = super
210
228
  sig { returns(T::Boolean) }
211
229
  def page? = false
212
230
  sig { returns(T::Boolean) }
@@ -222,6 +240,115 @@ module Crawlberg
222
240
  end
223
241
  end
224
242
 
243
+ module Crawlberg
244
+ # Hostname/IP allowlist matcher for SSRF policy.
245
+ #
246
+ # Serializes as an internally-tagged object so each variant is distinguishable on the
247
+ # wire and round-trips losslessly:
248
+ #
249
+ # ```json
250
+ # {"type": "exact", "value": "api.example.com"}
251
+ # {"type": "suffix", "value": ".example.com"}
252
+ # {"type": "cidr", "value": "10.0.0.0/8"}
253
+ # ```
254
+ #
255
+ # A bare JSON string is still accepted on deserialization and resolves to [`Exact`],
256
+ # preserving configs written against the previous untagged representation.
257
+ #
258
+ # [`Exact`]: HostMatcher::Exact
259
+ module HostMatcher
260
+ extend T::Helpers
261
+ extend T::Sig
262
+
263
+ interface!
264
+
265
+ # Dispatch from a Hash to the appropriate variant constructor.
266
+ # @param hash [Hash] with discriminator field and variant-specific fields
267
+ # @return [variant_class] an instance of the appropriate variant
268
+ sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.untyped) }
269
+ def self.from_hash(hash)
270
+ discriminator = hash[:type] || hash["type"]
271
+ case discriminator
272
+ when "exact"
273
+ HostMatcherExact.from_hash(hash)
274
+ when "suffix"
275
+ HostMatcherSuffix.from_hash(hash)
276
+ when "cidr"
277
+ HostMatcherCidr.from_hash(hash)
278
+ else
279
+ raise "Unknown discriminator: #{discriminator}"
280
+ end
281
+ end
282
+ end
283
+ ## Exact hostname match (case-insensitive).
284
+ HostMatcherExact = Data.define(:value) do
285
+ include HostMatcher
286
+ extend T::Sig
287
+
288
+ # The hostname to match.
289
+ sig { returns(String) }
290
+ # rubocop:disable Lint/UselessMethodDefinition
291
+ def value = super
292
+ sig { returns(T::Boolean) }
293
+ def exact? = true
294
+ sig { returns(T::Boolean) }
295
+ def suffix? = false
296
+ sig { returns(T::Boolean) }
297
+ def cidr? = false
298
+ # @param hash [Hash] deserialized from the native extension
299
+ # @return [self]
300
+ sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
301
+ def self.from_hash(hash)
302
+ new(value: hash[:value] || hash["value"])
303
+ end
304
+ end
305
+ ## Suffix match: ".xberg.io" matches "api.xberg.io" and "xberg.io".
306
+ HostMatcherSuffix = Data.define(:value) do
307
+ include HostMatcher
308
+ extend T::Sig
309
+
310
+ # The dot-prefixed suffix to match. A leading dot is optional.
311
+ sig { returns(String) }
312
+ # rubocop:disable Lint/UselessMethodDefinition
313
+ def value = super
314
+ sig { returns(T::Boolean) }
315
+ def exact? = false
316
+ sig { returns(T::Boolean) }
317
+ def suffix? = true
318
+ sig { returns(T::Boolean) }
319
+ def cidr? = false
320
+ # @param hash [Hash] deserialized from the native extension
321
+ # @return [self]
322
+ sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
323
+ def self.from_hash(hash)
324
+ new(value: hash[:value] || hash["value"])
325
+ end
326
+ end
327
+ ## CIDR match: "10.0.0.0/8" matches IP addresses in that range.
328
+ HostMatcherCidr = Data.define(:value) do
329
+ include HostMatcher
330
+ extend T::Sig
331
+
332
+ # The CIDR block. Validated when built through [`HostMatcher::cidr`] or
333
+ # deserialization.
334
+ sig { returns(String) }
335
+ # rubocop:disable Lint/UselessMethodDefinition
336
+ def value = super
337
+ sig { returns(T::Boolean) }
338
+ def exact? = false
339
+ sig { returns(T::Boolean) }
340
+ def suffix? = false
341
+ sig { returns(T::Boolean) }
342
+ def cidr? = true
343
+ # @param hash [Hash] deserialized from the native extension
344
+ # @return [self]
345
+ sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
346
+ def self.from_hash(hash)
347
+ new(value: hash[:value] || hash["value"])
348
+ end
349
+ end
350
+ end
351
+
225
352
  module Crawlberg
226
353
  # A single page interaction action.
227
354
  #
@@ -240,15 +367,24 @@ module Crawlberg
240
367
  def self.from_hash(hash)
241
368
  discriminator = hash[:type] || hash["type"]
242
369
  case discriminator
243
- when "click" then PageActionClick.from_hash(hash)
244
- when "type" then PageActionTypeText.from_hash(hash)
245
- when "press" then PageActionPress.from_hash(hash)
246
- when "scroll" then PageActionScroll.from_hash(hash)
247
- when "wait" then PageActionWait.from_hash(hash)
248
- when "screenshot" then PageActionScreenshot.from_hash(hash)
249
- when "executeJs" then PageActionExecuteJs.from_hash(hash)
250
- when "scrape" then PageActionScrape.from_hash(hash)
251
- else raise "Unknown discriminator: #{discriminator}"
370
+ when "click"
371
+ PageActionClick.from_hash(hash)
372
+ when "type"
373
+ PageActionTypeText.from_hash(hash)
374
+ when "press"
375
+ PageActionPress.from_hash(hash)
376
+ when "scroll"
377
+ PageActionScroll.from_hash(hash)
378
+ when "wait"
379
+ PageActionWait.from_hash(hash)
380
+ when "screenshot"
381
+ PageActionScreenshot.from_hash(hash)
382
+ when "executeJs"
383
+ PageActionExecuteJs.from_hash(hash)
384
+ when "scrape"
385
+ PageActionScrape.from_hash(hash)
386
+ else
387
+ raise "Unknown discriminator: #{discriminator}"
252
388
  end
253
389
  end
254
390
  end
@@ -259,7 +395,8 @@ module Crawlberg
259
395
 
260
396
  # CSS selector for the element to click.
261
397
  sig { returns(String) }
262
- def selector = super # rubocop:disable Lint/UselessMethodDefinition
398
+ # rubocop:disable Lint/UselessMethodDefinition
399
+ def selector = super
263
400
  sig { returns(T::Boolean) }
264
401
  def click? = true
265
402
  sig { returns(T::Boolean) }
@@ -290,10 +427,12 @@ module Crawlberg
290
427
 
291
428
  # CSS selector for the input element.
292
429
  sig { returns(String) }
293
- def selector = super # rubocop:disable Lint/UselessMethodDefinition
430
+ # rubocop:disable Lint/UselessMethodDefinition
431
+ def selector = super
294
432
  # Text to type into the element.
295
433
  sig { returns(String) }
296
- def text = super # rubocop:disable Lint/UselessMethodDefinition
434
+ # rubocop:disable Lint/UselessMethodDefinition
435
+ def text = super
297
436
  sig { returns(T::Boolean) }
298
437
  def click? = false
299
438
  sig { returns(T::Boolean) }
@@ -324,7 +463,8 @@ module Crawlberg
324
463
 
325
464
  # Key name to press.
326
465
  sig { returns(String) }
327
- def key = super # rubocop:disable Lint/UselessMethodDefinition
466
+ # rubocop:disable Lint/UselessMethodDefinition
467
+ def key = super
328
468
  sig { returns(T::Boolean) }
329
469
  def click? = false
330
470
  sig { returns(T::Boolean) }
@@ -355,13 +495,16 @@ module Crawlberg
355
495
 
356
496
  # Direction to scroll.
357
497
  sig { returns(ScrollDirection) }
358
- def direction = super # rubocop:disable Lint/UselessMethodDefinition
498
+ # rubocop:disable Lint/UselessMethodDefinition
499
+ def direction = super
359
500
  # Optional CSS selector for a scrollable element. Scrolls the page if absent.
360
501
  sig { returns(T.nilable(String)) }
361
- def selector = super # rubocop:disable Lint/UselessMethodDefinition
502
+ # rubocop:disable Lint/UselessMethodDefinition
503
+ def selector = super
362
504
  # Optional pixel amount to scroll. Uses a default if absent.
363
505
  sig { returns(T.nilable(Integer)) }
364
- def amount = super # rubocop:disable Lint/UselessMethodDefinition
506
+ # rubocop:disable Lint/UselessMethodDefinition
507
+ def amount = super
365
508
  sig { returns(T::Boolean) }
366
509
  def click? = false
367
510
  sig { returns(T::Boolean) }
@@ -382,7 +525,11 @@ module Crawlberg
382
525
  # @return [self]
383
526
  sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
384
527
  def self.from_hash(hash)
385
- new(direction: hash[:direction] || hash["direction"], selector: hash[:selector] || hash["selector"], amount: hash[:amount] || hash["amount"])
528
+ new(
529
+ direction: hash[:direction] || hash["direction"],
530
+ selector: hash[:selector] || hash["selector"],
531
+ amount: hash[:amount] || hash["amount"]
532
+ )
386
533
  end
387
534
  end
388
535
  ## Wait for a duration or for an element to appear.
@@ -392,10 +539,12 @@ module Crawlberg
392
539
 
393
540
  # Milliseconds to wait. Ignored if `selector` is provided.
394
541
  sig { returns(T.nilable(Integer)) }
395
- def milliseconds = super # rubocop:disable Lint/UselessMethodDefinition
542
+ # rubocop:disable Lint/UselessMethodDefinition
543
+ def milliseconds = super
396
544
  # CSS selector to wait for.
397
545
  sig { returns(T.nilable(String)) }
398
- def selector = super # rubocop:disable Lint/UselessMethodDefinition
546
+ # rubocop:disable Lint/UselessMethodDefinition
547
+ def selector = super
399
548
  sig { returns(T::Boolean) }
400
549
  def click? = false
401
550
  sig { returns(T::Boolean) }
@@ -430,7 +579,8 @@ module Crawlberg
430
579
  # `full_page` (snake_case) alias so language bindings and fixtures can
431
580
  # use either convention without error.
432
581
  sig { returns(T.nilable(T::Boolean)) }
433
- def full_page = super # rubocop:disable Lint/UselessMethodDefinition
582
+ # rubocop:disable Lint/UselessMethodDefinition
583
+ def full_page = super
434
584
  sig { returns(T::Boolean) }
435
585
  def click? = false
436
586
  sig { returns(T::Boolean) }
@@ -466,7 +616,8 @@ module Crawlberg
466
616
 
467
617
  # JavaScript source code to execute. Max 1 MB.
468
618
  sig { returns(String) }
469
- def script = super # rubocop:disable Lint/UselessMethodDefinition
619
+ # rubocop:disable Lint/UselessMethodDefinition
620
+ def script = super
470
621
  sig { returns(T::Boolean) }
471
622
  def click? = false
472
623
  sig { returns(T::Boolean) }
@@ -519,105 +670,3 @@ module Crawlberg
519
670
  end
520
671
  end
521
672
  end
522
-
523
- module Crawlberg
524
- # Hostname/IP allowlist matcher for SSRF policy.
525
- #
526
- # Serializes as an internally-tagged object so each variant is distinguishable on the
527
- # wire and round-trips losslessly:
528
- #
529
- # ```json
530
- # {"type": "exact", "value": "api.example.com"}
531
- # {"type": "suffix", "value": ".example.com"}
532
- # {"type": "cidr", "value": "10.0.0.0/8"}
533
- # ```
534
- #
535
- # A bare JSON string is still accepted on deserialization and resolves to [`Exact`],
536
- # preserving configs written against the previous untagged representation.
537
- #
538
- # [`Exact`]: HostMatcher::Exact
539
- module HostMatcher
540
- extend T::Helpers
541
- extend T::Sig
542
-
543
- interface!
544
-
545
- # Dispatch from a Hash to the appropriate variant constructor.
546
- # @param hash [Hash] with discriminator field and variant-specific fields
547
- # @return [variant_class] an instance of the appropriate variant
548
- sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.untyped) }
549
- def self.from_hash(hash)
550
- discriminator = hash[:type] || hash["type"]
551
- case discriminator
552
- when "exact" then HostMatcherExact.from_hash(hash)
553
- when "suffix" then HostMatcherSuffix.from_hash(hash)
554
- when "cidr" then HostMatcherCidr.from_hash(hash)
555
- else raise "Unknown discriminator: #{discriminator}"
556
- end
557
- end
558
- end
559
- ## Exact hostname match (case-insensitive).
560
- HostMatcherExact = Data.define(:value) do
561
- include HostMatcher
562
- extend T::Sig
563
-
564
- # The hostname to match.
565
- sig { returns(String) }
566
- def value = super # rubocop:disable Lint/UselessMethodDefinition
567
- sig { returns(T::Boolean) }
568
- def exact? = true
569
- sig { returns(T::Boolean) }
570
- def suffix? = false
571
- sig { returns(T::Boolean) }
572
- def cidr? = false
573
- # @param hash [Hash] deserialized from the native extension
574
- # @return [self]
575
- sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
576
- def self.from_hash(hash)
577
- new(value: hash[:value] || hash["value"])
578
- end
579
- end
580
- ## Suffix match: ".xberg.io" matches "api.xberg.io" and "xberg.io".
581
- HostMatcherSuffix = Data.define(:value) do
582
- include HostMatcher
583
- extend T::Sig
584
-
585
- # The dot-prefixed suffix to match. A leading dot is optional.
586
- sig { returns(String) }
587
- def value = super # rubocop:disable Lint/UselessMethodDefinition
588
- sig { returns(T::Boolean) }
589
- def exact? = false
590
- sig { returns(T::Boolean) }
591
- def suffix? = true
592
- sig { returns(T::Boolean) }
593
- def cidr? = false
594
- # @param hash [Hash] deserialized from the native extension
595
- # @return [self]
596
- sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
597
- def self.from_hash(hash)
598
- new(value: hash[:value] || hash["value"])
599
- end
600
- end
601
- ## CIDR match: "10.0.0.0/8" matches IP addresses in that range.
602
- HostMatcherCidr = Data.define(:value) do
603
- include HostMatcher
604
- extend T::Sig
605
-
606
- # The CIDR block. Validated when built through [`HostMatcher::cidr`] or
607
- # deserialization.
608
- sig { returns(String) }
609
- def value = super # rubocop:disable Lint/UselessMethodDefinition
610
- sig { returns(T::Boolean) }
611
- def exact? = false
612
- sig { returns(T::Boolean) }
613
- def suffix? = false
614
- sig { returns(T::Boolean) }
615
- def cidr? = true
616
- # @param hash [Hash] deserialized from the native extension
617
- # @return [self]
618
- sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
619
- def self.from_hash(hash)
620
- new(value: hash[:value] || hash["value"])
621
- end
622
- end
623
- end
@@ -1,10 +1,10 @@
1
1
  # This file is auto-generated by alef — DO NOT EDIT.
2
- # alef:hash:9716830c37f9c6de3924a6ebdc2f8f077d296794b443399aa1d4c4f0d483169a
2
+ # alef:hash:9ea9a14d259e4988466100a061689d975727c16c521c140aef2205c9b97bf831
3
3
  # To regenerate: alef generate
4
4
  # To verify freshness: alef verify
5
5
  # frozen_string_literal: true
6
6
 
7
7
  module Crawlberg
8
8
  ## The version string for this package.
9
- VERSION = "1.3.3"
9
+ VERSION = "1.4.2"
10
10
  end
data/lib/crawlberg.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # This file is auto-generated by alef — DO NOT EDIT.
2
- # alef:hash:5f14772be973afec2e94293f27da4258e3399457f1e279dc68d4d55a011f6b3a
2
+ # alef:hash:ed4c2274eb4834de21c62fc789eebe8acea49536e10970e2634ab6608966ec3d
3
3
  # To regenerate: alef generate
4
4
  # To verify freshness: alef verify
5
5
  # frozen_string_literal: true