crawlberg 1.10.3 → 1.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +5 -5
- data/ext/crawlberg_rb/native/Cargo.lock +113 -181
- data/ext/crawlberg_rb/native/Cargo.toml +7 -8
- data/ext/crawlberg_rb/src/lib.rs +176 -3
- data/lib/crawlberg/version.rb +2 -2
- data/sig/types.rbs +13 -4
- metadata +8 -5
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 67ea86fd053651445f70e913780834e4c2a660960ce4cf31316e9a87a05669e6
|
|
4
|
+
data.tar.gz: a6494f969d6cb25801ee2b63d1a64f415d24722ce7611c8569b3da2c34097532
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 5da6db4d33dbe01952a45649bba8d820f0222305aaa6902603ebcc54a74acbc59d946a4211fc63a53af14c4174415bcaf68ee3d43a133d125851633f8ff32213
|
|
7
|
+
data.tar.gz: 89b023052617d86df08df9416318babb35240ef2a6e79cb9c40a3488894b89ab8bf551c3df2b437e1021d1d37567e174807f5e72ea9082791bc653d41aa63700
|
data/README.md
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
<!-- This file is auto-generated by alef — DO NOT EDIT. -->
|
|
2
|
-
<!-- alef:hash:
|
|
2
|
+
<!-- alef:hash:fcb23a304253e2719260285609af11e93960b21fb56072d6816533f452223e9c -->
|
|
3
3
|
<!-- To regenerate: alef readme -->
|
|
4
4
|
<!-- To verify freshness: alef verify -->
|
|
5
5
|
|
|
@@ -100,14 +100,14 @@ gem install crawlberg
|
|
|
100
100
|
|
|
101
101
|
## Agent plugin
|
|
102
102
|
|
|
103
|
-
The `crawlberg` plugin is available via the `xberg-io/
|
|
103
|
+
The `crawlberg` plugin is available via the `xberg-io/crawlberg` marketplace.
|
|
104
104
|
|
|
105
105
|
```text
|
|
106
|
-
/plugin marketplace add xberg-io/
|
|
107
|
-
/plugin install crawlberg@
|
|
106
|
+
/plugin marketplace add xberg-io/crawlberg
|
|
107
|
+
/plugin install crawlberg@crawlberg
|
|
108
108
|
```
|
|
109
109
|
|
|
110
|
-
|
|
110
|
+
See [the plugin guide](https://github.com/xberg-io/crawlberg/blob/main/plugin/README.md) for all supported runtimes, CLI prerequisites and standalone public skills.
|
|
111
111
|
|
|
112
112
|
## Quick Start
|
|
113
113
|
|
|
@@ -105,14 +105,14 @@ checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667"
|
|
|
105
105
|
dependencies = [
|
|
106
106
|
"proc-macro2",
|
|
107
107
|
"quote",
|
|
108
|
-
"syn 3.0.
|
|
108
|
+
"syn 3.0.9",
|
|
109
109
|
]
|
|
110
110
|
|
|
111
111
|
[[package]]
|
|
112
112
|
name = "async-tungstenite"
|
|
113
|
-
version = "0.
|
|
113
|
+
version = "0.35.0"
|
|
114
114
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
115
|
-
checksum = "
|
|
115
|
+
checksum = "d6ab5b7664abf9ccae07576888c72bc93750c7090c1ff4ffee9317cc05d11618"
|
|
116
116
|
dependencies = [
|
|
117
117
|
"atomic-waker",
|
|
118
118
|
"futures-core",
|
|
@@ -139,9 +139,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
|
|
139
139
|
|
|
140
140
|
[[package]]
|
|
141
141
|
name = "aws-lc-rs"
|
|
142
|
-
version = "1.18.
|
|
142
|
+
version = "1.18.2"
|
|
143
143
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
144
|
-
checksum = "
|
|
144
|
+
checksum = "faac5829c2b74c28f830747e7818ccfb684261b5f48a1118b1e2a13d36dfab13"
|
|
145
145
|
dependencies = [
|
|
146
146
|
"aws-lc-sys",
|
|
147
147
|
"zeroize",
|
|
@@ -149,9 +149,9 @@ dependencies = [
|
|
|
149
149
|
|
|
150
150
|
[[package]]
|
|
151
151
|
name = "aws-lc-sys"
|
|
152
|
-
version = "0.
|
|
152
|
+
version = "0.46.0"
|
|
153
153
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
154
|
-
checksum = "
|
|
154
|
+
checksum = "f1622d8446a2d4b2ce0c7eefc73dd43a99779028d5ee5c2dd8073a658ba8a2bc"
|
|
155
155
|
dependencies = [
|
|
156
156
|
"cc",
|
|
157
157
|
"cmake",
|
|
@@ -221,16 +221,7 @@ dependencies = [
|
|
|
221
221
|
"cc",
|
|
222
222
|
"cfg-if",
|
|
223
223
|
"constant_time_eq",
|
|
224
|
-
"cpufeatures
|
|
225
|
-
]
|
|
226
|
-
|
|
227
|
-
[[package]]
|
|
228
|
-
name = "block-buffer"
|
|
229
|
-
version = "0.10.4"
|
|
230
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
231
|
-
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
|
232
|
-
dependencies = [
|
|
233
|
-
"generic-array",
|
|
224
|
+
"cpufeatures",
|
|
234
225
|
]
|
|
235
226
|
|
|
236
227
|
[[package]]
|
|
@@ -289,9 +280,9 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
|
|
|
289
280
|
|
|
290
281
|
[[package]]
|
|
291
282
|
name = "cc"
|
|
292
|
-
version = "1.
|
|
283
|
+
version = "1.8.0"
|
|
293
284
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
294
|
-
checksum = "
|
|
285
|
+
checksum = "6651c9ed80effdc7db0ff72512157f901af5e3549e341e24b1dd4887d836d838"
|
|
295
286
|
dependencies = [
|
|
296
287
|
"find-msvc-tools",
|
|
297
288
|
"jobserver",
|
|
@@ -327,8 +318,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
|
327
318
|
checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06"
|
|
328
319
|
dependencies = [
|
|
329
320
|
"cfg-if",
|
|
330
|
-
"cpufeatures
|
|
331
|
-
"rand_core
|
|
321
|
+
"cpufeatures",
|
|
322
|
+
"rand_core",
|
|
323
|
+
]
|
|
324
|
+
|
|
325
|
+
[[package]]
|
|
326
|
+
name = "chardetng"
|
|
327
|
+
version = "1.0.0"
|
|
328
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
329
|
+
checksum = "13de944a44b5064ee5d3a5ceccc49a41bfec50f2580e66f82e87703acdb88b53"
|
|
330
|
+
dependencies = [
|
|
331
|
+
"cfg-if",
|
|
332
|
+
"encoding_rs",
|
|
333
|
+
"memchr",
|
|
332
334
|
]
|
|
333
335
|
|
|
334
336
|
[[package]]
|
|
@@ -530,15 +532,6 @@ version = "1.0.0"
|
|
|
530
532
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
531
533
|
checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48"
|
|
532
534
|
|
|
533
|
-
[[package]]
|
|
534
|
-
name = "cpufeatures"
|
|
535
|
-
version = "0.2.17"
|
|
536
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
537
|
-
checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
|
|
538
|
-
dependencies = [
|
|
539
|
-
"libc",
|
|
540
|
-
]
|
|
541
|
-
|
|
542
535
|
[[package]]
|
|
543
536
|
name = "cpufeatures"
|
|
544
537
|
version = "0.3.1"
|
|
@@ -550,9 +543,9 @@ dependencies = [
|
|
|
550
543
|
|
|
551
544
|
[[package]]
|
|
552
545
|
name = "crawlberg"
|
|
553
|
-
version = "1.
|
|
546
|
+
version = "1.11.0"
|
|
554
547
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
555
|
-
checksum = "
|
|
548
|
+
checksum = "56da73bf80601b5f5b163f981d8002c611146533bccb890763f93d1dc91bbc12"
|
|
556
549
|
dependencies = [
|
|
557
550
|
"ahash",
|
|
558
551
|
"aho-corasick",
|
|
@@ -563,7 +556,9 @@ dependencies = [
|
|
|
563
556
|
"base64 0.23.1",
|
|
564
557
|
"blake3",
|
|
565
558
|
"bytes",
|
|
559
|
+
"chardetng",
|
|
566
560
|
"chrono",
|
|
561
|
+
"cookie_store",
|
|
567
562
|
"crawlberg-chromiumoxide",
|
|
568
563
|
"crawlberg-robots",
|
|
569
564
|
"dashmap",
|
|
@@ -591,6 +586,7 @@ dependencies = [
|
|
|
591
586
|
"opentelemetry",
|
|
592
587
|
"opentelemetry-semantic-conventions",
|
|
593
588
|
"percent-encoding",
|
|
589
|
+
"psl",
|
|
594
590
|
"quick-xml",
|
|
595
591
|
"regex",
|
|
596
592
|
"regex-syntax",
|
|
@@ -617,9 +613,9 @@ dependencies = [
|
|
|
617
613
|
|
|
618
614
|
[[package]]
|
|
619
615
|
name = "crawlberg-chromiumoxide"
|
|
620
|
-
version = "0.9.
|
|
616
|
+
version = "0.9.3"
|
|
621
617
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
622
|
-
checksum = "
|
|
618
|
+
checksum = "8add9abf085c5f95d7dfbde84b67e6b6f2fb95a0ca288c98a4e8ceae169d2d3f"
|
|
623
619
|
dependencies = [
|
|
624
620
|
"async-tungstenite",
|
|
625
621
|
"base64 0.22.1",
|
|
@@ -629,6 +625,7 @@ dependencies = [
|
|
|
629
625
|
"fnv",
|
|
630
626
|
"futures",
|
|
631
627
|
"futures-timer",
|
|
628
|
+
"libc",
|
|
632
629
|
"pin-project-lite",
|
|
633
630
|
"reqwest",
|
|
634
631
|
"serde",
|
|
@@ -639,11 +636,12 @@ dependencies = [
|
|
|
639
636
|
"url",
|
|
640
637
|
"which",
|
|
641
638
|
"windows-registry",
|
|
639
|
+
"windows-sys 0.61.2",
|
|
642
640
|
]
|
|
643
641
|
|
|
644
642
|
[[package]]
|
|
645
643
|
name = "crawlberg-rb"
|
|
646
|
-
version = "1.
|
|
644
|
+
version = "1.11.0"
|
|
647
645
|
dependencies = [
|
|
648
646
|
"crawlberg",
|
|
649
647
|
"futures",
|
|
@@ -656,9 +654,9 @@ dependencies = [
|
|
|
656
654
|
|
|
657
655
|
[[package]]
|
|
658
656
|
name = "crawlberg-robots"
|
|
659
|
-
version = "1.
|
|
657
|
+
version = "1.11.0"
|
|
660
658
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
661
|
-
checksum = "
|
|
659
|
+
checksum = "403b147a1d91d3f3d5d20e1f4f8a6c92c34db944e322ed0f986e79162133eb3d"
|
|
662
660
|
|
|
663
661
|
[[package]]
|
|
664
662
|
name = "crc32fast"
|
|
@@ -681,16 +679,6 @@ version = "0.2.4"
|
|
|
681
679
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
682
680
|
checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
|
|
683
681
|
|
|
684
|
-
[[package]]
|
|
685
|
-
name = "crypto-common"
|
|
686
|
-
version = "0.1.7"
|
|
687
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
688
|
-
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
|
689
|
-
dependencies = [
|
|
690
|
-
"generic-array",
|
|
691
|
-
"typenum",
|
|
692
|
-
]
|
|
693
|
-
|
|
694
682
|
[[package]]
|
|
695
683
|
name = "crypto-common"
|
|
696
684
|
version = "0.2.2"
|
|
@@ -722,19 +710,9 @@ checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06"
|
|
|
722
710
|
|
|
723
711
|
[[package]]
|
|
724
712
|
name = "deranged"
|
|
725
|
-
version = "0.5.
|
|
713
|
+
version = "0.5.9"
|
|
726
714
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
727
|
-
checksum = "
|
|
728
|
-
|
|
729
|
-
[[package]]
|
|
730
|
-
name = "digest"
|
|
731
|
-
version = "0.10.7"
|
|
732
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
733
|
-
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
|
734
|
-
dependencies = [
|
|
735
|
-
"block-buffer 0.10.4",
|
|
736
|
-
"crypto-common 0.1.7",
|
|
737
|
-
]
|
|
715
|
+
checksum = "9e9de72ce2ad1f90dc62fa25f0f430ef85eb4b0d8fa0be4f30373bc40a21d28e"
|
|
738
716
|
|
|
739
717
|
[[package]]
|
|
740
718
|
name = "digest"
|
|
@@ -742,9 +720,9 @@ version = "0.11.3"
|
|
|
742
720
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
743
721
|
checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2"
|
|
744
722
|
dependencies = [
|
|
745
|
-
"block-buffer
|
|
723
|
+
"block-buffer",
|
|
746
724
|
"const-oid",
|
|
747
|
-
"crypto-common
|
|
725
|
+
"crypto-common",
|
|
748
726
|
]
|
|
749
727
|
|
|
750
728
|
[[package]]
|
|
@@ -776,7 +754,7 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8"
|
|
|
776
754
|
dependencies = [
|
|
777
755
|
"proc-macro2",
|
|
778
756
|
"quote",
|
|
779
|
-
"syn 3.0.
|
|
757
|
+
"syn 3.0.9",
|
|
780
758
|
]
|
|
781
759
|
|
|
782
760
|
[[package]]
|
|
@@ -816,9 +794,9 @@ dependencies = [
|
|
|
816
794
|
|
|
817
795
|
[[package]]
|
|
818
796
|
name = "equivalent"
|
|
819
|
-
version = "1.0.
|
|
797
|
+
version = "1.0.3"
|
|
820
798
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
821
|
-
checksum = "
|
|
799
|
+
checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826"
|
|
822
800
|
|
|
823
801
|
[[package]]
|
|
824
802
|
name = "errno"
|
|
@@ -974,7 +952,7 @@ checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44"
|
|
|
974
952
|
dependencies = [
|
|
975
953
|
"proc-macro2",
|
|
976
954
|
"quote",
|
|
977
|
-
"syn 3.0.
|
|
955
|
+
"syn 3.0.9",
|
|
978
956
|
]
|
|
979
957
|
|
|
980
958
|
[[package]]
|
|
@@ -1012,16 +990,6 @@ dependencies = [
|
|
|
1012
990
|
"slab",
|
|
1013
991
|
]
|
|
1014
992
|
|
|
1015
|
-
[[package]]
|
|
1016
|
-
name = "generic-array"
|
|
1017
|
-
version = "0.14.7"
|
|
1018
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1019
|
-
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
|
1020
|
-
dependencies = [
|
|
1021
|
-
"typenum",
|
|
1022
|
-
"version_check",
|
|
1023
|
-
]
|
|
1024
|
-
|
|
1025
993
|
[[package]]
|
|
1026
994
|
name = "getrandom"
|
|
1027
995
|
version = "0.2.17"
|
|
@@ -1059,7 +1027,7 @@ dependencies = [
|
|
|
1059
1027
|
"js-sys",
|
|
1060
1028
|
"libc",
|
|
1061
1029
|
"r-efi 6.0.0",
|
|
1062
|
-
"rand_core
|
|
1030
|
+
"rand_core",
|
|
1063
1031
|
"wasm-bindgen",
|
|
1064
1032
|
]
|
|
1065
1033
|
|
|
@@ -1118,15 +1086,15 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
|
|
1118
1086
|
|
|
1119
1087
|
[[package]]
|
|
1120
1088
|
name = "html-escape"
|
|
1121
|
-
version = "0.
|
|
1089
|
+
version = "0.3.0"
|
|
1122
1090
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1123
|
-
checksum = "
|
|
1091
|
+
checksum = "491194d601b4fa9fac3c9571e014070a0ad1bc3392d1bc10419695196a8b3cc3"
|
|
1124
1092
|
|
|
1125
1093
|
[[package]]
|
|
1126
1094
|
name = "html-to-markdown-rs"
|
|
1127
|
-
version = "3.
|
|
1095
|
+
version = "3.18.1"
|
|
1128
1096
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1129
|
-
checksum = "
|
|
1097
|
+
checksum = "36135ae2471a090d086837db2f4794dca3bf8ca0ba288b799a0bb85dd3f0acf0"
|
|
1130
1098
|
dependencies = [
|
|
1131
1099
|
"ahash",
|
|
1132
1100
|
"astral-tl",
|
|
@@ -1642,9 +1610,9 @@ checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae"
|
|
|
1642
1610
|
|
|
1643
1611
|
[[package]]
|
|
1644
1612
|
name = "litrs"
|
|
1645
|
-
version = "1.0.
|
|
1613
|
+
version = "1.0.1"
|
|
1646
1614
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1647
|
-
checksum = "
|
|
1615
|
+
checksum = "4744e383959f0db86ede514b809b1c53251889093803c05267acc7d4e7030d70"
|
|
1648
1616
|
|
|
1649
1617
|
[[package]]
|
|
1650
1618
|
name = "lock_api"
|
|
@@ -1867,9 +1835,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
|
|
|
1867
1835
|
|
|
1868
1836
|
[[package]]
|
|
1869
1837
|
name = "opentelemetry"
|
|
1870
|
-
version = "0.33.
|
|
1838
|
+
version = "0.33.1"
|
|
1871
1839
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1872
|
-
checksum = "
|
|
1840
|
+
checksum = "8c1f1a6ac9c875a4941e4c7c085b61e8f4daa326de03b4701768c3491a6ad9ee"
|
|
1873
1841
|
dependencies = [
|
|
1874
1842
|
"futures-core",
|
|
1875
1843
|
"futures-sink",
|
|
@@ -1881,9 +1849,9 @@ dependencies = [
|
|
|
1881
1849
|
|
|
1882
1850
|
[[package]]
|
|
1883
1851
|
name = "opentelemetry-semantic-conventions"
|
|
1884
|
-
version = "0.33.
|
|
1852
|
+
version = "0.33.1"
|
|
1885
1853
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1886
|
-
checksum = "
|
|
1854
|
+
checksum = "7a3cf639fefd3e852869114f9925b578da6dd8e99281a2089bb0e773ead46962"
|
|
1887
1855
|
|
|
1888
1856
|
[[package]]
|
|
1889
1857
|
name = "option-ext"
|
|
@@ -2015,15 +1983,6 @@ version = "0.2.1"
|
|
|
2015
1983
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
2016
1984
|
checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958"
|
|
2017
1985
|
|
|
2018
|
-
[[package]]
|
|
2019
|
-
name = "ppv-lite86"
|
|
2020
|
-
version = "0.2.21"
|
|
2021
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
2022
|
-
checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
|
|
2023
|
-
dependencies = [
|
|
2024
|
-
"zerocopy",
|
|
2025
|
-
]
|
|
2026
|
-
|
|
2027
1986
|
[[package]]
|
|
2028
1987
|
name = "precomputed-hash"
|
|
2029
1988
|
version = "0.1.1"
|
|
@@ -2039,6 +1998,15 @@ dependencies = [
|
|
|
2039
1998
|
"unicode-ident",
|
|
2040
1999
|
]
|
|
2041
2000
|
|
|
2001
|
+
[[package]]
|
|
2002
|
+
name = "psl"
|
|
2003
|
+
version = "2.1.241"
|
|
2004
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
2005
|
+
checksum = "c7319b480e204d67e759cad7886178335d26324c7c8573c126cc9dfa7350d064"
|
|
2006
|
+
dependencies = [
|
|
2007
|
+
"psl-types",
|
|
2008
|
+
]
|
|
2009
|
+
|
|
2042
2010
|
[[package]]
|
|
2043
2011
|
name = "psl-types"
|
|
2044
2012
|
version = "2.0.11"
|
|
@@ -2106,7 +2074,7 @@ dependencies = [
|
|
|
2106
2074
|
"bytes",
|
|
2107
2075
|
"getrandom 0.4.3",
|
|
2108
2076
|
"lru-slab",
|
|
2109
|
-
"rand
|
|
2077
|
+
"rand",
|
|
2110
2078
|
"rand_pcg",
|
|
2111
2079
|
"ring",
|
|
2112
2080
|
"rustc-hash",
|
|
@@ -2154,16 +2122,6 @@ version = "6.0.0"
|
|
|
2154
2122
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
2155
2123
|
checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
|
|
2156
2124
|
|
|
2157
|
-
[[package]]
|
|
2158
|
-
name = "rand"
|
|
2159
|
-
version = "0.9.5"
|
|
2160
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
2161
|
-
checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41"
|
|
2162
|
-
dependencies = [
|
|
2163
|
-
"rand_chacha",
|
|
2164
|
-
"rand_core 0.9.5",
|
|
2165
|
-
]
|
|
2166
|
-
|
|
2167
2125
|
[[package]]
|
|
2168
2126
|
name = "rand"
|
|
2169
2127
|
version = "0.10.3"
|
|
@@ -2172,26 +2130,7 @@ checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af"
|
|
|
2172
2130
|
dependencies = [
|
|
2173
2131
|
"chacha20",
|
|
2174
2132
|
"getrandom 0.4.3",
|
|
2175
|
-
"rand_core
|
|
2176
|
-
]
|
|
2177
|
-
|
|
2178
|
-
[[package]]
|
|
2179
|
-
name = "rand_chacha"
|
|
2180
|
-
version = "0.9.0"
|
|
2181
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
2182
|
-
checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
|
|
2183
|
-
dependencies = [
|
|
2184
|
-
"ppv-lite86",
|
|
2185
|
-
"rand_core 0.9.5",
|
|
2186
|
-
]
|
|
2187
|
-
|
|
2188
|
-
[[package]]
|
|
2189
|
-
name = "rand_core"
|
|
2190
|
-
version = "0.9.5"
|
|
2191
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
2192
|
-
checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c"
|
|
2193
|
-
dependencies = [
|
|
2194
|
-
"getrandom 0.3.4",
|
|
2133
|
+
"rand_core",
|
|
2195
2134
|
]
|
|
2196
2135
|
|
|
2197
2136
|
[[package]]
|
|
@@ -2206,7 +2145,7 @@ version = "0.10.2"
|
|
|
2206
2145
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
2207
2146
|
checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a"
|
|
2208
2147
|
dependencies = [
|
|
2209
|
-
"rand_core
|
|
2148
|
+
"rand_core",
|
|
2210
2149
|
]
|
|
2211
2150
|
|
|
2212
2151
|
[[package]]
|
|
@@ -2371,9 +2310,9 @@ dependencies = [
|
|
|
2371
2310
|
|
|
2372
2311
|
[[package]]
|
|
2373
2312
|
name = "rustls"
|
|
2374
|
-
version = "0.23.
|
|
2313
|
+
version = "0.23.46"
|
|
2375
2314
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
2376
|
-
checksum = "
|
|
2315
|
+
checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf"
|
|
2377
2316
|
dependencies = [
|
|
2378
2317
|
"aws-lc-rs",
|
|
2379
2318
|
"once_cell",
|
|
@@ -2385,9 +2324,9 @@ dependencies = [
|
|
|
2385
2324
|
|
|
2386
2325
|
[[package]]
|
|
2387
2326
|
name = "rustls-native-certs"
|
|
2388
|
-
version = "0.8.
|
|
2327
|
+
version = "0.8.5"
|
|
2389
2328
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
2390
|
-
checksum = "
|
|
2329
|
+
checksum = "4a073f5dc7469f984c52ad2752b63b0807745133b6de880b7b64c1ac4c48aec4"
|
|
2391
2330
|
dependencies = [
|
|
2392
2331
|
"openssl-probe",
|
|
2393
2332
|
"rustls-pki-types",
|
|
@@ -2536,14 +2475,14 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
|
|
2536
2475
|
dependencies = [
|
|
2537
2476
|
"proc-macro2",
|
|
2538
2477
|
"quote",
|
|
2539
|
-
"syn 3.0.
|
|
2478
|
+
"syn 3.0.9",
|
|
2540
2479
|
]
|
|
2541
2480
|
|
|
2542
2481
|
[[package]]
|
|
2543
2482
|
name = "serde_json"
|
|
2544
|
-
version = "1.0.
|
|
2483
|
+
version = "1.0.154"
|
|
2545
2484
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
2546
|
-
checksum = "
|
|
2485
|
+
checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6"
|
|
2547
2486
|
dependencies = [
|
|
2548
2487
|
"itoa",
|
|
2549
2488
|
"memchr",
|
|
@@ -2563,24 +2502,24 @@ dependencies = [
|
|
|
2563
2502
|
|
|
2564
2503
|
[[package]]
|
|
2565
2504
|
name = "sha1"
|
|
2566
|
-
version = "0.
|
|
2505
|
+
version = "0.11.0"
|
|
2567
2506
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
2568
|
-
checksum = "
|
|
2507
|
+
checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214"
|
|
2569
2508
|
dependencies = [
|
|
2570
2509
|
"cfg-if",
|
|
2571
|
-
"cpufeatures
|
|
2572
|
-
"digest
|
|
2510
|
+
"cpufeatures",
|
|
2511
|
+
"digest",
|
|
2573
2512
|
]
|
|
2574
2513
|
|
|
2575
2514
|
[[package]]
|
|
2576
2515
|
name = "sha2"
|
|
2577
|
-
version = "0.11.
|
|
2516
|
+
version = "0.11.1"
|
|
2578
2517
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
2579
|
-
checksum = "
|
|
2518
|
+
checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19"
|
|
2580
2519
|
dependencies = [
|
|
2581
2520
|
"cfg-if",
|
|
2582
|
-
"cpufeatures
|
|
2583
|
-
"digest
|
|
2521
|
+
"cpufeatures",
|
|
2522
|
+
"digest",
|
|
2584
2523
|
]
|
|
2585
2524
|
|
|
2586
2525
|
[[package]]
|
|
@@ -2647,9 +2586,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
|
|
|
2647
2586
|
|
|
2648
2587
|
[[package]]
|
|
2649
2588
|
name = "smallvec"
|
|
2650
|
-
version = "1.16.
|
|
2589
|
+
version = "1.16.3"
|
|
2651
2590
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
2652
|
-
checksum = "
|
|
2591
|
+
checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b"
|
|
2653
2592
|
|
|
2654
2593
|
[[package]]
|
|
2655
2594
|
name = "socket2"
|
|
@@ -2720,9 +2659,9 @@ dependencies = [
|
|
|
2720
2659
|
|
|
2721
2660
|
[[package]]
|
|
2722
2661
|
name = "syn"
|
|
2723
|
-
version = "3.0.
|
|
2662
|
+
version = "3.0.9"
|
|
2724
2663
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
2725
|
-
checksum = "
|
|
2664
|
+
checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b"
|
|
2726
2665
|
dependencies = [
|
|
2727
2666
|
"proc-macro2",
|
|
2728
2667
|
"quote",
|
|
@@ -2746,14 +2685,14 @@ checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02"
|
|
|
2746
2685
|
dependencies = [
|
|
2747
2686
|
"proc-macro2",
|
|
2748
2687
|
"quote",
|
|
2749
|
-
"syn 3.0.
|
|
2688
|
+
"syn 3.0.9",
|
|
2750
2689
|
]
|
|
2751
2690
|
|
|
2752
2691
|
[[package]]
|
|
2753
2692
|
name = "sysinfo"
|
|
2754
|
-
version = "0.
|
|
2693
|
+
version = "0.39.6"
|
|
2755
2694
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
2756
|
-
checksum = "
|
|
2695
|
+
checksum = "d2071df9448915b71c4fe6d25deaf1c22f12bd234f01540b77312bb8e41361e6"
|
|
2757
2696
|
dependencies = [
|
|
2758
2697
|
"libc",
|
|
2759
2698
|
"memchr",
|
|
@@ -2843,7 +2782,7 @@ checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524"
|
|
|
2843
2782
|
dependencies = [
|
|
2844
2783
|
"proc-macro2",
|
|
2845
2784
|
"quote",
|
|
2846
|
-
"syn 3.0.
|
|
2785
|
+
"syn 3.0.9",
|
|
2847
2786
|
]
|
|
2848
2787
|
|
|
2849
2788
|
[[package]]
|
|
@@ -2904,9 +2843,9 @@ checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee"
|
|
|
2904
2843
|
|
|
2905
2844
|
[[package]]
|
|
2906
2845
|
name = "tokio"
|
|
2907
|
-
version = "1.53.
|
|
2846
|
+
version = "1.53.3"
|
|
2908
2847
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
2909
|
-
checksum = "
|
|
2848
|
+
checksum = "ce3335fa71841cda333a58d7615b03901380ecf09d59b3296d21f8bbac0dde4e"
|
|
2910
2849
|
dependencies = [
|
|
2911
2850
|
"bytes",
|
|
2912
2851
|
"libc",
|
|
@@ -2926,7 +2865,7 @@ checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e"
|
|
|
2926
2865
|
dependencies = [
|
|
2927
2866
|
"proc-macro2",
|
|
2928
2867
|
"quote",
|
|
2929
|
-
"syn 3.0.
|
|
2868
|
+
"syn 3.0.9",
|
|
2930
2869
|
]
|
|
2931
2870
|
|
|
2932
2871
|
[[package]]
|
|
@@ -2964,9 +2903,9 @@ dependencies = [
|
|
|
2964
2903
|
|
|
2965
2904
|
[[package]]
|
|
2966
2905
|
name = "tokio-util"
|
|
2967
|
-
version = "0.7.
|
|
2906
|
+
version = "0.7.20"
|
|
2968
2907
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
2969
|
-
checksum = "
|
|
2908
|
+
checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b"
|
|
2970
2909
|
dependencies = [
|
|
2971
2910
|
"bytes",
|
|
2972
2911
|
"futures-core",
|
|
@@ -2978,9 +2917,9 @@ dependencies = [
|
|
|
2978
2917
|
|
|
2979
2918
|
[[package]]
|
|
2980
2919
|
name = "toml"
|
|
2981
|
-
version = "1.1.
|
|
2920
|
+
version = "1.1.8+spec-1.1.0"
|
|
2982
2921
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
2983
|
-
checksum = "
|
|
2922
|
+
checksum = "20489e00e4d8741d6be680764cc12e270655e375a20d1011e844a9c3379e678d"
|
|
2984
2923
|
dependencies = [
|
|
2985
2924
|
"indexmap",
|
|
2986
2925
|
"serde_core",
|
|
@@ -3002,9 +2941,9 @@ dependencies = [
|
|
|
3002
2941
|
|
|
3003
2942
|
[[package]]
|
|
3004
2943
|
name = "toml_parser"
|
|
3005
|
-
version = "1.1.
|
|
2944
|
+
version = "1.1.5+spec-1.1.0"
|
|
3006
2945
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
3007
|
-
checksum = "
|
|
2946
|
+
checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c"
|
|
3008
2947
|
dependencies = [
|
|
3009
2948
|
"winnow",
|
|
3010
2949
|
]
|
|
@@ -3106,19 +3045,18 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
|
|
|
3106
3045
|
|
|
3107
3046
|
[[package]]
|
|
3108
3047
|
name = "tungstenite"
|
|
3109
|
-
version = "0.
|
|
3048
|
+
version = "0.30.0"
|
|
3110
3049
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
3111
|
-
checksum = "
|
|
3050
|
+
checksum = "e48ac77174b19c110a50ab2128b24215ac9cb40e0e12e093fb602d175c569d22"
|
|
3112
3051
|
dependencies = [
|
|
3113
3052
|
"bytes",
|
|
3114
3053
|
"data-encoding",
|
|
3115
3054
|
"http",
|
|
3116
3055
|
"httparse",
|
|
3117
3056
|
"log",
|
|
3118
|
-
"rand
|
|
3057
|
+
"rand",
|
|
3119
3058
|
"sha1",
|
|
3120
3059
|
"thiserror 2.0.21",
|
|
3121
|
-
"utf-8",
|
|
3122
3060
|
]
|
|
3123
3061
|
|
|
3124
3062
|
[[package]]
|
|
@@ -3129,9 +3067,9 @@ checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
|
|
|
3129
3067
|
|
|
3130
3068
|
[[package]]
|
|
3131
3069
|
name = "unicode-ident"
|
|
3132
|
-
version = "1.0.
|
|
3070
|
+
version = "1.0.27"
|
|
3133
3071
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
3134
|
-
checksum = "
|
|
3072
|
+
checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
|
|
3135
3073
|
|
|
3136
3074
|
[[package]]
|
|
3137
3075
|
name = "untrusted"
|
|
@@ -3151,12 +3089,6 @@ dependencies = [
|
|
|
3151
3089
|
"serde",
|
|
3152
3090
|
]
|
|
3153
3091
|
|
|
3154
|
-
[[package]]
|
|
3155
|
-
name = "utf-8"
|
|
3156
|
-
version = "0.7.6"
|
|
3157
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
3158
|
-
checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
|
|
3159
|
-
|
|
3160
3092
|
[[package]]
|
|
3161
3093
|
name = "utf8_iter"
|
|
3162
3094
|
version = "1.0.4"
|
|
@@ -3246,7 +3178,7 @@ dependencies = [
|
|
|
3246
3178
|
"bumpalo",
|
|
3247
3179
|
"proc-macro2",
|
|
3248
3180
|
"quote",
|
|
3249
|
-
"syn 3.0.
|
|
3181
|
+
"syn 3.0.9",
|
|
3250
3182
|
"wasm-bindgen-shared",
|
|
3251
3183
|
]
|
|
3252
3184
|
|
|
@@ -3660,24 +3592,24 @@ checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71"
|
|
|
3660
3592
|
dependencies = [
|
|
3661
3593
|
"proc-macro2",
|
|
3662
3594
|
"quote",
|
|
3663
|
-
"syn 3.0.
|
|
3595
|
+
"syn 3.0.9",
|
|
3664
3596
|
"synstructure",
|
|
3665
3597
|
]
|
|
3666
3598
|
|
|
3667
3599
|
[[package]]
|
|
3668
3600
|
name = "zerocopy"
|
|
3669
|
-
version = "0.8.
|
|
3601
|
+
version = "0.8.63"
|
|
3670
3602
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
3671
|
-
checksum = "
|
|
3603
|
+
checksum = "e5fe1f8f1b06191a00962174c61aa5005e0bb391a6d80d07e24d115c01a92ed8"
|
|
3672
3604
|
dependencies = [
|
|
3673
3605
|
"zerocopy-derive",
|
|
3674
3606
|
]
|
|
3675
3607
|
|
|
3676
3608
|
[[package]]
|
|
3677
3609
|
name = "zerocopy-derive"
|
|
3678
|
-
version = "0.8.
|
|
3610
|
+
version = "0.8.63"
|
|
3679
3611
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
3680
|
-
checksum = "
|
|
3612
|
+
checksum = "863ad3ac83293fb4d740aedbfdc9240dd8d1a50c1099acd76ce80ce7c7230c7f"
|
|
3681
3613
|
dependencies = [
|
|
3682
3614
|
"proc-macro2",
|
|
3683
3615
|
"quote",
|
|
@@ -3701,15 +3633,15 @@ checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a"
|
|
|
3701
3633
|
dependencies = [
|
|
3702
3634
|
"proc-macro2",
|
|
3703
3635
|
"quote",
|
|
3704
|
-
"syn 3.0.
|
|
3636
|
+
"syn 3.0.9",
|
|
3705
3637
|
"synstructure",
|
|
3706
3638
|
]
|
|
3707
3639
|
|
|
3708
3640
|
[[package]]
|
|
3709
3641
|
name = "zeroize"
|
|
3710
|
-
version = "1.9.
|
|
3642
|
+
version = "1.9.1"
|
|
3711
3643
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
3712
|
-
checksum = "
|
|
3644
|
+
checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879"
|
|
3713
3645
|
|
|
3714
3646
|
[[package]]
|
|
3715
3647
|
name = "zerotrie"
|
|
@@ -3743,7 +3675,7 @@ checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da"
|
|
|
3743
3675
|
dependencies = [
|
|
3744
3676
|
"proc-macro2",
|
|
3745
3677
|
"quote",
|
|
3746
|
-
"syn 3.0.
|
|
3678
|
+
"syn 3.0.9",
|
|
3747
3679
|
]
|
|
3748
3680
|
|
|
3749
3681
|
[[package]]
|
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
# This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
-
# alef:hash:
|
|
2
|
+
# alef:hash:250b27ae33dca5fc53a5d4ba444b1759ca0977e9dbf24546d07b20cc13b55aee
|
|
3
3
|
# To regenerate: alef generate
|
|
4
4
|
# To verify freshness: alef verify
|
|
5
5
|
|
|
6
6
|
[package]
|
|
7
7
|
name = "crawlberg-rb"
|
|
8
|
-
version = "1.
|
|
8
|
+
version = "1.11.0"
|
|
9
9
|
edition = "2024"
|
|
10
|
+
authors = ["Na'aman Hirschfeld <naaman@xberg.io>", "Tobias Perlstein <tobias@xberg.io>"]
|
|
10
11
|
license = "MIT"
|
|
11
12
|
description = "High-performance web crawling engine"
|
|
12
13
|
readme = false
|
|
@@ -22,7 +23,7 @@ path = "../src/lib.rs"
|
|
|
22
23
|
crate-type = ["cdylib"]
|
|
23
24
|
|
|
24
25
|
[dependencies]
|
|
25
|
-
crawlberg = { version = "1.
|
|
26
|
+
crawlberg = { version = "1.11.0", features = ["interact", "browser-chromiumoxide"] }
|
|
26
27
|
futures = "0.3"
|
|
27
28
|
magnus = "0.9"
|
|
28
29
|
rb-sys = ">=0.9.130, <0.10"
|
|
@@ -31,11 +32,9 @@ serde_json = "1"
|
|
|
31
32
|
tokio = { version = "1", features = ["rt-multi-thread"] }
|
|
32
33
|
|
|
33
34
|
# This crate deliberately does not use `[lints]` / `workspace = true`: its C-ABI /
|
|
34
|
-
# PyO3 / napi / ext-php-rs / NIF boundary requires `unsafe` code
|
|
35
|
-
#
|
|
36
|
-
#
|
|
37
|
-
# instead carries the subset of the workspace's deny-by-default lint policy this crate
|
|
38
|
-
# can actually satisfy. ~keep
|
|
35
|
+
# PyO3 / napi / ext-php-rs / NIF boundary requires `unsafe` code. Local lint tables
|
|
36
|
+
# avoid inheriting workspace policies that could reject these boundaries, and select
|
|
37
|
+
# the lints this binding can actually satisfy. ~keep
|
|
39
38
|
[lints.clippy]
|
|
40
39
|
all = { level = "warn", priority = -1 }
|
|
41
40
|
dbg_macro = "deny"
|
data/ext/crawlberg_rb/src/lib.rs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// This file is auto-generated by alef. DO NOT EDIT.
|
|
2
|
-
// alef:hash:
|
|
2
|
+
// alef:hash:2ec1fdc93f72b06b0d045fd1f6b6288fad256babb9754b0e89759ebc3fd4b1f8
|
|
3
3
|
// Re-generate with: alef generate
|
|
4
4
|
#![allow(dead_code, unused_imports, unused_variables)]
|
|
5
5
|
#![allow(
|
|
@@ -1437,6 +1437,13 @@ pub struct ContentConfig {
|
|
|
1437
1437
|
wrap_width: usize,
|
|
1438
1438
|
include_document_structure: bool,
|
|
1439
1439
|
extract_metadata: bool,
|
|
1440
|
+
strip_newlines: bool,
|
|
1441
|
+
bullets: String,
|
|
1442
|
+
list_indent_width: usize,
|
|
1443
|
+
strong_em_symbol: String,
|
|
1444
|
+
compact_tables: bool,
|
|
1445
|
+
keep_inline_images_in: Vec<String>,
|
|
1446
|
+
keep_hidden: bool,
|
|
1440
1447
|
}
|
|
1441
1448
|
|
|
1442
1449
|
unsafe impl IntoValueFromNative for ContentConfig {}
|
|
@@ -1605,6 +1612,69 @@ impl ContentConfig {
|
|
|
1605
1612
|
})?,
|
|
1606
1613
|
None => true,
|
|
1607
1614
|
},
|
|
1615
|
+
strip_newlines: match kwargs.get(ruby.to_symbol("strip_newlines")) {
|
|
1616
|
+
Some(v) => bool::try_convert(v).map_err(|e| {
|
|
1617
|
+
magnus::Error::new(
|
|
1618
|
+
unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
|
|
1619
|
+
format!("invalid value for `strip_newlines`: {}", e),
|
|
1620
|
+
)
|
|
1621
|
+
})?,
|
|
1622
|
+
None => false,
|
|
1623
|
+
},
|
|
1624
|
+
bullets: match kwargs.get(ruby.to_symbol("bullets")) {
|
|
1625
|
+
Some(v) => String::try_convert(v).map_err(|e| {
|
|
1626
|
+
magnus::Error::new(
|
|
1627
|
+
unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
|
|
1628
|
+
format!("invalid value for `bullets`: {}", e),
|
|
1629
|
+
)
|
|
1630
|
+
})?,
|
|
1631
|
+
None => "-*+".to_string(),
|
|
1632
|
+
},
|
|
1633
|
+
list_indent_width: match kwargs.get(ruby.to_symbol("list_indent_width")) {
|
|
1634
|
+
Some(v) => usize::try_convert(v).map_err(|e| {
|
|
1635
|
+
magnus::Error::new(
|
|
1636
|
+
unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
|
|
1637
|
+
format!("invalid value for `list_indent_width`: {}", e),
|
|
1638
|
+
)
|
|
1639
|
+
})?,
|
|
1640
|
+
None => 2,
|
|
1641
|
+
},
|
|
1642
|
+
strong_em_symbol: match kwargs.get(ruby.to_symbol("strong_em_symbol")) {
|
|
1643
|
+
Some(v) => String::try_convert(v).map_err(|e| {
|
|
1644
|
+
magnus::Error::new(
|
|
1645
|
+
unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
|
|
1646
|
+
format!("invalid value for `strong_em_symbol`: {}", e),
|
|
1647
|
+
)
|
|
1648
|
+
})?,
|
|
1649
|
+
None => "*".to_string(),
|
|
1650
|
+
},
|
|
1651
|
+
compact_tables: match kwargs.get(ruby.to_symbol("compact_tables")) {
|
|
1652
|
+
Some(v) => bool::try_convert(v).map_err(|e| {
|
|
1653
|
+
magnus::Error::new(
|
|
1654
|
+
unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
|
|
1655
|
+
format!("invalid value for `compact_tables`: {}", e),
|
|
1656
|
+
)
|
|
1657
|
+
})?,
|
|
1658
|
+
None => false,
|
|
1659
|
+
},
|
|
1660
|
+
keep_inline_images_in: match kwargs.get(ruby.to_symbol("keep_inline_images_in")) {
|
|
1661
|
+
Some(v) => <Vec<String>>::try_convert(v).map_err(|e| {
|
|
1662
|
+
magnus::Error::new(
|
|
1663
|
+
unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
|
|
1664
|
+
format!("invalid value for `keep_inline_images_in`: {}", e),
|
|
1665
|
+
)
|
|
1666
|
+
})?,
|
|
1667
|
+
None => Default::default(),
|
|
1668
|
+
},
|
|
1669
|
+
keep_hidden: match kwargs.get(ruby.to_symbol("keep_hidden")) {
|
|
1670
|
+
Some(v) => bool::try_convert(v).map_err(|e| {
|
|
1671
|
+
magnus::Error::new(
|
|
1672
|
+
unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
|
|
1673
|
+
format!("invalid value for `keep_hidden`: {}", e),
|
|
1674
|
+
)
|
|
1675
|
+
})?,
|
|
1676
|
+
None => false,
|
|
1677
|
+
},
|
|
1608
1678
|
})
|
|
1609
1679
|
}
|
|
1610
1680
|
|
|
@@ -1659,6 +1729,34 @@ impl ContentConfig {
|
|
|
1659
1729
|
fn extract_metadata(&self) -> bool {
|
|
1660
1730
|
self.extract_metadata
|
|
1661
1731
|
}
|
|
1732
|
+
|
|
1733
|
+
fn strip_newlines(&self) -> bool {
|
|
1734
|
+
self.strip_newlines
|
|
1735
|
+
}
|
|
1736
|
+
|
|
1737
|
+
fn bullets(&self) -> String {
|
|
1738
|
+
self.bullets.clone()
|
|
1739
|
+
}
|
|
1740
|
+
|
|
1741
|
+
fn list_indent_width(&self) -> usize {
|
|
1742
|
+
self.list_indent_width
|
|
1743
|
+
}
|
|
1744
|
+
|
|
1745
|
+
fn strong_em_symbol(&self) -> String {
|
|
1746
|
+
self.strong_em_symbol.clone()
|
|
1747
|
+
}
|
|
1748
|
+
|
|
1749
|
+
fn compact_tables(&self) -> bool {
|
|
1750
|
+
self.compact_tables
|
|
1751
|
+
}
|
|
1752
|
+
|
|
1753
|
+
fn keep_inline_images_in(&self) -> Vec<String> {
|
|
1754
|
+
self.keep_inline_images_in.clone()
|
|
1755
|
+
}
|
|
1756
|
+
|
|
1757
|
+
fn keep_hidden(&self) -> bool {
|
|
1758
|
+
self.keep_hidden
|
|
1759
|
+
}
|
|
1662
1760
|
}
|
|
1663
1761
|
|
|
1664
1762
|
#[derive(Clone, serde::Serialize, serde::Deserialize)]
|
|
@@ -2102,7 +2200,7 @@ impl CrawlConfig {
|
|
|
2102
2200
|
format!("invalid value for `dedup_include_query`: {}", e),
|
|
2103
2201
|
)
|
|
2104
2202
|
})?,
|
|
2105
|
-
None =>
|
|
2203
|
+
None => true,
|
|
2106
2204
|
},
|
|
2107
2205
|
strip_tracking_params: match kwargs.get(ruby.to_symbol("strip_tracking_params")) {
|
|
2108
2206
|
Some(v) => bool::try_convert(v).map_err(|e| {
|
|
@@ -2946,6 +3044,7 @@ pub struct CrawlPageResult {
|
|
|
2946
3044
|
noindex_detected: bool,
|
|
2947
3045
|
nofollow_detected: bool,
|
|
2948
3046
|
ssrf_refused_urls: Vec<String>,
|
|
3047
|
+
original_url: String,
|
|
2949
3048
|
}
|
|
2950
3049
|
|
|
2951
3050
|
unsafe impl IntoValueFromNative for CrawlPageResult {}
|
|
@@ -3234,6 +3333,15 @@ impl CrawlPageResult {
|
|
|
3234
3333
|
})?,
|
|
3235
3334
|
None => Default::default(),
|
|
3236
3335
|
},
|
|
3336
|
+
original_url: match kwargs.get(ruby.to_symbol("original_url")) {
|
|
3337
|
+
Some(v) => String::try_convert(v).map_err(|e| {
|
|
3338
|
+
magnus::Error::new(
|
|
3339
|
+
unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
|
|
3340
|
+
format!("invalid value for `original_url`: {}", e),
|
|
3341
|
+
)
|
|
3342
|
+
})?,
|
|
3343
|
+
None => Default::default(),
|
|
3344
|
+
},
|
|
3237
3345
|
})
|
|
3238
3346
|
}
|
|
3239
3347
|
|
|
@@ -3340,6 +3448,10 @@ impl CrawlPageResult {
|
|
|
3340
3448
|
fn ssrf_refused_urls(&self) -> Vec<String> {
|
|
3341
3449
|
self.ssrf_refused_urls.clone()
|
|
3342
3450
|
}
|
|
3451
|
+
|
|
3452
|
+
fn original_url(&self) -> String {
|
|
3453
|
+
self.original_url.clone()
|
|
3454
|
+
}
|
|
3343
3455
|
}
|
|
3344
3456
|
|
|
3345
3457
|
#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
|
|
@@ -4757,7 +4869,7 @@ impl JsonLdEntry {
|
|
|
4757
4869
|
}
|
|
4758
4870
|
}
|
|
4759
4871
|
|
|
4760
|
-
#[derive(Clone, Debug, serde::Serialize
|
|
4872
|
+
#[derive(Clone, Debug, serde::Serialize)]
|
|
4761
4873
|
#[serde(default)]
|
|
4762
4874
|
#[magnus::wrap(class = "Crawlberg::LinkInfo")]
|
|
4763
4875
|
pub struct LinkInfo {
|
|
@@ -4766,6 +4878,7 @@ pub struct LinkInfo {
|
|
|
4766
4878
|
link_type: LinkType,
|
|
4767
4879
|
rel: Option<String>,
|
|
4768
4880
|
nofollow: bool,
|
|
4881
|
+
original_url: String,
|
|
4769
4882
|
}
|
|
4770
4883
|
|
|
4771
4884
|
unsafe impl IntoValueFromNative for LinkInfo {}
|
|
@@ -4795,6 +4908,14 @@ impl magnus::TryConvert for LinkInfo {
|
|
|
4795
4908
|
}
|
|
4796
4909
|
|
|
4797
4910
|
unsafe impl TryConvertOwned for LinkInfo {}
|
|
4911
|
+
impl<'de> serde::Deserialize<'de> for LinkInfo {
|
|
4912
|
+
fn deserialize<D>(deserializer: D) -> ::core::result::Result<Self, D::Error>
|
|
4913
|
+
where
|
|
4914
|
+
D: serde::Deserializer<'de>,
|
|
4915
|
+
{
|
|
4916
|
+
<crawlberg::LinkInfo as serde::Deserialize>::deserialize(deserializer).map(Into::into)
|
|
4917
|
+
}
|
|
4918
|
+
}
|
|
4798
4919
|
|
|
4799
4920
|
impl Default for LinkInfo {
|
|
4800
4921
|
fn default() -> Self {
|
|
@@ -4854,6 +4975,15 @@ impl LinkInfo {
|
|
|
4854
4975
|
})?,
|
|
4855
4976
|
None => Default::default(),
|
|
4856
4977
|
},
|
|
4978
|
+
original_url: match kwargs.get(ruby.to_symbol("original_url")) {
|
|
4979
|
+
Some(v) => String::try_convert(v).map_err(|e| {
|
|
4980
|
+
magnus::Error::new(
|
|
4981
|
+
unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
|
|
4982
|
+
format!("invalid value for `original_url`: {}", e),
|
|
4983
|
+
)
|
|
4984
|
+
})?,
|
|
4985
|
+
None => Default::default(),
|
|
4986
|
+
},
|
|
4857
4987
|
})
|
|
4858
4988
|
}
|
|
4859
4989
|
|
|
@@ -4876,6 +5006,10 @@ impl LinkInfo {
|
|
|
4876
5006
|
fn nofollow(&self) -> bool {
|
|
4877
5007
|
self.nofollow
|
|
4878
5008
|
}
|
|
5009
|
+
|
|
5010
|
+
fn original_url(&self) -> String {
|
|
5011
|
+
self.original_url.clone()
|
|
5012
|
+
}
|
|
4879
5013
|
}
|
|
4880
5014
|
|
|
4881
5015
|
#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
|
|
@@ -8117,6 +8251,13 @@ impl From<ContentConfig> for crawlberg::ContentConfig {
|
|
|
8117
8251
|
wrap_width: val.wrap_width,
|
|
8118
8252
|
include_document_structure: val.include_document_structure,
|
|
8119
8253
|
extract_metadata: val.extract_metadata,
|
|
8254
|
+
strip_newlines: val.strip_newlines,
|
|
8255
|
+
bullets: val.bullets,
|
|
8256
|
+
list_indent_width: val.list_indent_width,
|
|
8257
|
+
strong_em_symbol: val.strong_em_symbol,
|
|
8258
|
+
compact_tables: val.compact_tables,
|
|
8259
|
+
keep_inline_images_in: val.keep_inline_images_in.into_iter().collect(),
|
|
8260
|
+
keep_hidden: val.keep_hidden,
|
|
8120
8261
|
..Default::default()
|
|
8121
8262
|
}
|
|
8122
8263
|
}
|
|
@@ -8138,6 +8279,13 @@ impl From<crawlberg::ContentConfig> for ContentConfig {
|
|
|
8138
8279
|
wrap_width: val.wrap_width,
|
|
8139
8280
|
include_document_structure: val.include_document_structure,
|
|
8140
8281
|
extract_metadata: val.extract_metadata,
|
|
8282
|
+
strip_newlines: val.strip_newlines,
|
|
8283
|
+
bullets: val.bullets.to_string(),
|
|
8284
|
+
list_indent_width: val.list_indent_width,
|
|
8285
|
+
strong_em_symbol: val.strong_em_symbol.to_string(),
|
|
8286
|
+
compact_tables: val.compact_tables,
|
|
8287
|
+
keep_inline_images_in: val.keep_inline_images_in.into_iter().collect(),
|
|
8288
|
+
keep_hidden: val.keep_hidden,
|
|
8141
8289
|
}
|
|
8142
8290
|
}
|
|
8143
8291
|
}
|
|
@@ -8334,6 +8482,7 @@ impl From<CrawlPageResult> for crawlberg::CrawlPageResult {
|
|
|
8334
8482
|
noindex_detected: val.noindex_detected,
|
|
8335
8483
|
nofollow_detected: val.nofollow_detected,
|
|
8336
8484
|
ssrf_refused_urls: val.ssrf_refused_urls.into_iter().collect(),
|
|
8485
|
+
original_url: val.original_url,
|
|
8337
8486
|
..Default::default()
|
|
8338
8487
|
}
|
|
8339
8488
|
}
|
|
@@ -8369,6 +8518,7 @@ impl From<crawlberg::CrawlPageResult> for CrawlPageResult {
|
|
|
8369
8518
|
noindex_detected: val.noindex_detected,
|
|
8370
8519
|
nofollow_detected: val.nofollow_detected,
|
|
8371
8520
|
ssrf_refused_urls: val.ssrf_refused_urls.into_iter().collect(),
|
|
8521
|
+
original_url: val.original_url.to_string(),
|
|
8372
8522
|
}
|
|
8373
8523
|
}
|
|
8374
8524
|
}
|
|
@@ -8698,6 +8848,7 @@ impl From<LinkInfo> for crawlberg::LinkInfo {
|
|
|
8698
8848
|
link_type: val.link_type.into(),
|
|
8699
8849
|
rel: val.rel,
|
|
8700
8850
|
nofollow: val.nofollow,
|
|
8851
|
+
original_url: val.original_url,
|
|
8701
8852
|
..Default::default()
|
|
8702
8853
|
}
|
|
8703
8854
|
}
|
|
@@ -8712,6 +8863,7 @@ impl From<crawlberg::LinkInfo> for LinkInfo {
|
|
|
8712
8863
|
link_type: val.link_type.into(),
|
|
8713
8864
|
rel: val.rel.map(|v| v.to_string()),
|
|
8714
8865
|
nofollow: val.nofollow,
|
|
8866
|
+
original_url: val.original_url.to_string(),
|
|
8715
8867
|
}
|
|
8716
8868
|
}
|
|
8717
8869
|
}
|
|
@@ -9787,6 +9939,23 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
|
|
|
9787
9939
|
|
|
9788
9940
|
class.define_method("extract_metadata", method!(ContentConfig::extract_metadata, 0))?;
|
|
9789
9941
|
|
|
9942
|
+
class.define_method("strip_newlines", method!(ContentConfig::strip_newlines, 0))?;
|
|
9943
|
+
|
|
9944
|
+
class.define_method("bullets", method!(ContentConfig::bullets, 0))?;
|
|
9945
|
+
|
|
9946
|
+
class.define_method("list_indent_width", method!(ContentConfig::list_indent_width, 0))?;
|
|
9947
|
+
|
|
9948
|
+
class.define_method("strong_em_symbol", method!(ContentConfig::strong_em_symbol, 0))?;
|
|
9949
|
+
|
|
9950
|
+
class.define_method("compact_tables", method!(ContentConfig::compact_tables, 0))?;
|
|
9951
|
+
|
|
9952
|
+
class.define_method(
|
|
9953
|
+
"keep_inline_images_in",
|
|
9954
|
+
method!(ContentConfig::keep_inline_images_in, 0),
|
|
9955
|
+
)?;
|
|
9956
|
+
|
|
9957
|
+
class.define_method("keep_hidden", method!(ContentConfig::keep_hidden, 0))?;
|
|
9958
|
+
|
|
9790
9959
|
let class = module.define_class("CookieInfo", ruby.class_object())?;
|
|
9791
9960
|
|
|
9792
9961
|
class.define_singleton_method("new", function!(CookieInfo::new, -1))?;
|
|
@@ -9994,6 +10163,8 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
|
|
|
9994
10163
|
|
|
9995
10164
|
class.define_method("ssrf_refused_urls", method!(CrawlPageResult::ssrf_refused_urls, 0))?;
|
|
9996
10165
|
|
|
10166
|
+
class.define_method("original_url", method!(CrawlPageResult::original_url, 0))?;
|
|
10167
|
+
|
|
9997
10168
|
let class = module.define_class("CrawlResult", ruby.class_object())?;
|
|
9998
10169
|
|
|
9999
10170
|
class.define_singleton_method("new", function!(CrawlResult::new, -1))?;
|
|
@@ -10169,6 +10340,8 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
|
|
|
10169
10340
|
|
|
10170
10341
|
class.define_method("nofollow", method!(LinkInfo::nofollow, 0))?;
|
|
10171
10342
|
|
|
10343
|
+
class.define_method("original_url", method!(LinkInfo::original_url, 0))?;
|
|
10344
|
+
|
|
10172
10345
|
let class = module.define_class("MapResult", ruby.class_object())?;
|
|
10173
10346
|
|
|
10174
10347
|
class.define_singleton_method("new", function!(MapResult::new, -1))?;
|
data/lib/crawlberg/version.rb
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
# This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
-
# alef:hash:
|
|
2
|
+
# alef:hash:93297a4dfefbe27a3aa25aae988b939f292098b0ce5ca75dcbafac1118e3cdd6
|
|
3
3
|
# To regenerate: alef generate
|
|
4
4
|
# To verify freshness: alef verify
|
|
5
5
|
# frozen_string_literal: true
|
|
6
6
|
|
|
7
7
|
module Crawlberg
|
|
8
8
|
## The version string for this package.
|
|
9
|
-
VERSION = "1.
|
|
9
|
+
VERSION = "1.11.0"
|
|
10
10
|
end
|
data/sig/types.rbs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
-
# alef:hash:
|
|
2
|
+
# alef:hash:ad470f62b746634fbd629f163e7bae4a88fb011bbde28d998e38ade4877adf3f
|
|
3
3
|
# To regenerate: alef generate
|
|
4
4
|
# To verify freshness: alef verify
|
|
5
5
|
|
|
@@ -128,8 +128,15 @@ def initialize: (?content: String, ?references: Array[CitationReference]) -> voi
|
|
|
128
128
|
attr_reader wrap_width: Integer
|
|
129
129
|
attr_reader include_document_structure: bool
|
|
130
130
|
attr_reader extract_metadata: bool
|
|
131
|
+
attr_reader strip_newlines: bool
|
|
132
|
+
attr_reader bullets: String
|
|
133
|
+
attr_reader list_indent_width: Integer
|
|
134
|
+
attr_reader strong_em_symbol: String
|
|
135
|
+
attr_reader compact_tables: bool
|
|
136
|
+
attr_reader keep_inline_images_in: Array[String]
|
|
137
|
+
attr_reader keep_hidden: bool
|
|
131
138
|
|
|
132
|
-
def initialize: (?output_format: String, ?preprocessing_preset: String, ?remove_navigation: bool, ?remove_forms: bool, ?strip_tags: Array[String], ?preserve_tags: Array[String], ?exclude_selectors: Array[String], ?skip_images: bool, ?max_depth: Integer, ?wrap: bool, ?wrap_width: Integer, ?include_document_structure: bool, ?extract_metadata: bool) -> void
|
|
139
|
+
def initialize: (?output_format: String, ?preprocessing_preset: String, ?remove_navigation: bool, ?remove_forms: bool, ?strip_tags: Array[String], ?preserve_tags: Array[String], ?exclude_selectors: Array[String], ?skip_images: bool, ?max_depth: Integer, ?wrap: bool, ?wrap_width: Integer, ?include_document_structure: bool, ?extract_metadata: bool, ?strip_newlines: bool, ?bullets: String, ?list_indent_width: Integer, ?strong_em_symbol: String, ?compact_tables: bool, ?keep_inline_images_in: Array[String], ?keep_hidden: bool) -> void
|
|
133
140
|
end
|
|
134
141
|
|
|
135
142
|
class CookieInfo
|
|
@@ -234,8 +241,9 @@ def batch_crawl_stream: (BatchCrawlStreamRequest req) -> Enumerator[CrawlEvent]
|
|
|
234
241
|
attr_reader noindex_detected: bool
|
|
235
242
|
attr_reader nofollow_detected: bool
|
|
236
243
|
attr_reader ssrf_refused_urls: Array[String]
|
|
244
|
+
attr_reader original_url: String
|
|
237
245
|
|
|
238
|
-
def initialize: (?url: String, ?normalized_url: String, ?status_code: Integer, ?content_type: String, ?html: String, ?body_size: Integer, ?metadata: PageMetadata, ?links: Array[LinkInfo], ?images: Array[ImageInfo], ?feeds: Array[FeedInfo], ?json_ld: Array[JsonLdEntry], ?depth: Integer, ?stayed_on_domain: bool, ?was_skipped: bool, ?is_pdf: bool, ?detected_charset: String, ?markdown: MarkdownResult, ?extracted_data: String, ?extraction_meta: ExtractionMeta, ?downloaded_document: DownloadedDocument, ?browser_used: bool, ?final_url: String, ?redirect_count: Integer, ?noindex_detected: bool, ?nofollow_detected: bool, ?ssrf_refused_urls: Array[String]) -> void
|
|
246
|
+
def initialize: (?url: String, ?normalized_url: String, ?status_code: Integer, ?content_type: String, ?html: String, ?body_size: Integer, ?metadata: PageMetadata, ?links: Array[LinkInfo], ?images: Array[ImageInfo], ?feeds: Array[FeedInfo], ?json_ld: Array[JsonLdEntry], ?depth: Integer, ?stayed_on_domain: bool, ?was_skipped: bool, ?is_pdf: bool, ?detected_charset: String, ?markdown: MarkdownResult, ?extracted_data: String, ?extraction_meta: ExtractionMeta, ?downloaded_document: DownloadedDocument, ?browser_used: bool, ?final_url: String, ?redirect_count: Integer, ?noindex_detected: bool, ?nofollow_detected: bool, ?ssrf_refused_urls: Array[String], ?original_url: String) -> void
|
|
239
247
|
end
|
|
240
248
|
|
|
241
249
|
class CrawlResult
|
|
@@ -358,8 +366,9 @@ def initialize: (?action_results: Array[ActionResult], ?final_html: String, ?fin
|
|
|
358
366
|
attr_reader link_type: enum_LinkType
|
|
359
367
|
attr_reader rel: String?
|
|
360
368
|
attr_reader nofollow: bool
|
|
369
|
+
attr_reader original_url: String
|
|
361
370
|
|
|
362
|
-
def initialize: (?url: String, ?text: String, ?link_type: enum_LinkType, ?rel: String, ?nofollow: bool) -> void
|
|
371
|
+
def initialize: (?url: String, ?text: String, ?link_type: enum_LinkType, ?rel: String, ?nofollow: bool, ?original_url: String) -> void
|
|
363
372
|
end
|
|
364
373
|
|
|
365
374
|
class MapResult
|
metadata
CHANGED
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: crawlberg
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.11.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
|
-
-
|
|
7
|
+
- Na'aman Hirschfeld
|
|
8
|
+
- Tobias Perlstein
|
|
8
9
|
autorequire:
|
|
9
10
|
bindir: bin
|
|
10
11
|
cert_chain: []
|
|
11
|
-
date: 2026-10-
|
|
12
|
+
date: 2026-10-15 00:00:00.000000000 Z
|
|
12
13
|
dependencies:
|
|
13
14
|
- !ruby/object:Gem::Dependency
|
|
14
15
|
name: rb_sys
|
|
@@ -36,16 +37,18 @@ dependencies:
|
|
|
36
37
|
requirements:
|
|
37
38
|
- - "~>"
|
|
38
39
|
- !ruby/object:Gem::Version
|
|
39
|
-
version: '0.
|
|
40
|
+
version: '0.6'
|
|
40
41
|
type: :runtime
|
|
41
42
|
prerelease: false
|
|
42
43
|
version_requirements: !ruby/object:Gem::Requirement
|
|
43
44
|
requirements:
|
|
44
45
|
- - "~>"
|
|
45
46
|
- !ruby/object:Gem::Version
|
|
46
|
-
version: '0.
|
|
47
|
+
version: '0.6'
|
|
47
48
|
description: High-performance web crawling engine
|
|
48
49
|
email:
|
|
50
|
+
- naaman@xberg.io
|
|
51
|
+
- tobias@xberg.io
|
|
49
52
|
executables: []
|
|
50
53
|
extensions:
|
|
51
54
|
- ext/crawlberg_rb/native/extconf.rb
|