http_crawler 0.3.0.4 → 0.3.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/http_crawler/client.rb +7 -1
- data/lib/http_crawler/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 4deb4d5965b16f6f8edfbeea11855ec7c2af4d3a1d2337b8afba6fd7dfc34b28
|
|
4
|
+
data.tar.gz: 6ad1f67ff98bd61d7a7f105ea51ac2d692b9f81fbab3674404aa41f32978a1a3
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 144d4ed97e3d60c541d5e878b8a7252378effd929e778dc743af7aaf83dfae3e9c228759297d1c7fb4274c6680e920ac4a1fcb3711228d2ca75d99473fde771b
|
|
7
|
+
data.tar.gz: 44f5c97fc062d0a0424154f07bddb09078316df9a0c1aba27723083baeb4aebabed266f652b19d89f0448f08fe7c8db26871121ee6e6bdca9d4c29de7097f292
|
data/lib/http_crawler/client.rb
CHANGED
|
@@ -174,9 +174,15 @@ module HttpCrawler
|
|
|
174
174
|
proxy_ip
|
|
175
175
|
end
|
|
176
176
|
|
|
177
|
+
attr_accessor :error_urls
|
|
178
|
+
|
|
179
|
+
def error_urls
|
|
180
|
+
@error_urls ||= []
|
|
181
|
+
end
|
|
182
|
+
|
|
177
183
|
# 添加错误的url地址，表示这里面的url都是异常地址，存的是正则
|
|
178
184
|
def add_error_url(url_string)
|
|
179
|
-
|
|
185
|
+
self.error_urls << url_string
|
|
180
186
|
end
|
|
181
187
|
|
|
182
188
|
|
data/lib/http_crawler/version.rb
CHANGED