http_crawler 0.2.3.2 → 0.2.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.idea/http_crawler.iml +1 -2
- data/.idea/workspace.xml +31 -26
- data/lib/http_crawler/http.rb +5 -3
- data/lib/http_crawler/net/response.rb +6 -1
- data/lib/http_crawler/version.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
|
-
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 1143552ea7737865d9fe582c48b7fa3f5824da47cbb0e16d8911050bd807dc68
|
|
4
|
+
data.tar.gz: 0e55bdd075e617e1f60e9fb6a54f75366c7cc670801d6a49b48c007e612b00a3
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: e26f0d48ec8318b0d977933d95f091a8e9ce00ef872ef8668d66dcbacc653e984871210b132468b530fd7d30261bb2caae592b93f7d0fa73067432c7b4b38750
|
|
7
|
+
data.tar.gz: 98004bfd606c248e367d18bac5cea84304a5f83854d9c9ac4adedbbf5ee84a6533f6d909ffa73afaf66f0313605093f84da2bde25c78c3e6ed60b86e7e8c71f7
|
data/.idea/http_crawler.iml
CHANGED
|
@@ -7,7 +7,6 @@
|
|
|
7
7
|
<content url="file://$MODULE_DIR$" />
|
|
8
8
|
<orderEntry type="inheritedJdk" />
|
|
9
9
|
<orderEntry type="sourceFolder" forTests="false" />
|
|
10
|
-
<orderEntry type="library" scope="PROVIDED" name="bundler (v1.
|
|
11
|
-
<orderEntry type="library" scope="PROVIDED" name="nokogiri (v1.8.0, RVM: ruby-2.4.1) [gem]" level="application" />
|
|
10
|
+
<orderEntry type="library" scope="PROVIDED" name="bundler (v1.16.6, RVM: ruby-2.4.1) [gem]" level="application" />
|
|
12
11
|
</component>
|
|
13
12
|
</module>
|
data/.idea/workspace.xml
CHANGED
|
@@ -2,7 +2,11 @@
|
|
|
2
2
|
<project version="4">
|
|
3
3
|
<component name="ChangeListManager">
|
|
4
4
|
<list default="true" id="07223dd4-8944-486b-a29b-7461a5c9ec2d" name="Default" comment="">
|
|
5
|
+
<change beforePath="$PROJECT_DIR$/.idea/http_crawler.iml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/http_crawler.iml" afterDir="false" />
|
|
5
6
|
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
|
|
7
|
+
<change beforePath="$PROJECT_DIR$/lib/http_crawler/http.rb" beforeDir="false" afterPath="$PROJECT_DIR$/lib/http_crawler/http.rb" afterDir="false" />
|
|
8
|
+
<change beforePath="$PROJECT_DIR$/lib/http_crawler/net/response.rb" beforeDir="false" afterPath="$PROJECT_DIR$/lib/http_crawler/net/response.rb" afterDir="false" />
|
|
9
|
+
<change beforePath="$PROJECT_DIR$/lib/http_crawler/version.rb" beforeDir="false" afterPath="$PROJECT_DIR$/lib/http_crawler/version.rb" afterDir="false" />
|
|
6
10
|
</list>
|
|
7
11
|
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
|
|
8
12
|
<option name="TRACKING_ENABLED" value="true" />
|
|
@@ -16,8 +20,8 @@
|
|
|
16
20
|
<file leaf-file-name="version.rb" pinned="false" current-in-tab="true">
|
|
17
21
|
<entry file="file://$PROJECT_DIR$/lib/http_crawler/version.rb">
|
|
18
22
|
<provider selected="true" editor-type-id="text-editor">
|
|
19
|
-
<state relative-caret-position="
|
|
20
|
-
<caret line="
|
|
23
|
+
<state relative-caret-position="21">
|
|
24
|
+
<caret line="1" column="21" selection-start-line="1" selection-start-column="21" selection-end-line="1" selection-end-column="21" />
|
|
21
25
|
</state>
|
|
22
26
|
</provider>
|
|
23
27
|
</entry>
|
|
@@ -25,7 +29,7 @@
|
|
|
25
29
|
<file leaf-file-name="web.rb" pinned="false" current-in-tab="false">
|
|
26
30
|
<entry file="file://$PROJECT_DIR$/lib/http_crawler/web.rb">
|
|
27
31
|
<provider selected="true" editor-type-id="text-editor">
|
|
28
|
-
<state relative-caret-position="
|
|
32
|
+
<state relative-caret-position="126">
|
|
29
33
|
<caret line="6" selection-start-line="6" selection-end-line="6" />
|
|
30
34
|
</state>
|
|
31
35
|
</provider>
|
|
@@ -34,8 +38,8 @@
|
|
|
34
38
|
<file leaf-file-name="http.rb" pinned="false" current-in-tab="false">
|
|
35
39
|
<entry file="file://$PROJECT_DIR$/lib/http_crawler/http.rb">
|
|
36
40
|
<provider selected="true" editor-type-id="text-editor">
|
|
37
|
-
<state relative-caret-position="
|
|
38
|
-
<caret line="210" column="
|
|
41
|
+
<state relative-caret-position="4410">
|
|
42
|
+
<caret line="210" column="11" lean-forward="true" selection-start-line="210" selection-start-column="11" selection-end-line="210" selection-end-column="11" />
|
|
39
43
|
</state>
|
|
40
44
|
</provider>
|
|
41
45
|
</entry>
|
|
@@ -43,7 +47,7 @@
|
|
|
43
47
|
<file leaf-file-name="compat.rb" pinned="false" current-in-tab="false">
|
|
44
48
|
<entry file="file://$USER_HOME$/.rvm/rubies/ruby-2.4.1/lib/ruby/2.4.0/webrick/compat.rb">
|
|
45
49
|
<provider selected="true" editor-type-id="text-editor">
|
|
46
|
-
<state relative-caret-position="
|
|
50
|
+
<state relative-caret-position="378">
|
|
47
51
|
<caret line="18" column="1" selection-start-line="18" selection-start-column="1" selection-end-line="18" selection-end-column="1" />
|
|
48
52
|
</state>
|
|
49
53
|
</provider>
|
|
@@ -52,7 +56,7 @@
|
|
|
52
56
|
<file leaf-file-name="errno.rb" pinned="false" current-in-tab="false">
|
|
53
57
|
<entry file="file://$APPLICATION_HOME_DIR$/rubystubs24/errno.rb">
|
|
54
58
|
<provider selected="true" editor-type-id="text-editor">
|
|
55
|
-
<state relative-caret-position="
|
|
59
|
+
<state relative-caret-position="672">
|
|
56
60
|
<caret line="32" column="7" selection-start-line="32" selection-start-column="7" selection-end-line="32" selection-end-column="7" />
|
|
57
61
|
</state>
|
|
58
62
|
</provider>
|
|
@@ -61,7 +65,7 @@
|
|
|
61
65
|
<file leaf-file-name="common.rb" pinned="false" current-in-tab="false">
|
|
62
66
|
<entry file="file://$PROJECT_DIR$/lib/http_crawler/common.rb">
|
|
63
67
|
<provider selected="true" editor-type-id="text-editor">
|
|
64
|
-
<state relative-caret-position="
|
|
68
|
+
<state relative-caret-position="42">
|
|
65
69
|
<caret line="2" selection-start-line="2" selection-end-line="2" />
|
|
66
70
|
</state>
|
|
67
71
|
</provider>
|
|
@@ -70,7 +74,7 @@
|
|
|
70
74
|
<file leaf-file-name="proxy.rb" pinned="false" current-in-tab="false">
|
|
71
75
|
<entry file="file://$PROJECT_DIR$/lib/http_crawler/proxy.rb">
|
|
72
76
|
<provider selected="true" editor-type-id="text-editor">
|
|
73
|
-
<state relative-caret-position="
|
|
77
|
+
<state relative-caret-position="441">
|
|
74
78
|
<caret line="21" selection-start-line="21" selection-end-line="21" />
|
|
75
79
|
</state>
|
|
76
80
|
</provider>
|
|
@@ -79,7 +83,7 @@
|
|
|
79
83
|
<file leaf-file-name="client.rb" pinned="false" current-in-tab="false">
|
|
80
84
|
<entry file="file://$PROJECT_DIR$/lib/http_crawler/web/baidu/client.rb">
|
|
81
85
|
<provider selected="true" editor-type-id="text-editor">
|
|
82
|
-
<state relative-caret-position="
|
|
86
|
+
<state relative-caret-position="21">
|
|
83
87
|
<caret line="1" column="18" selection-start-line="1" selection-start-column="7" selection-end-line="1" selection-end-column="18" />
|
|
84
88
|
</state>
|
|
85
89
|
</provider>
|
|
@@ -88,7 +92,7 @@
|
|
|
88
92
|
<file leaf-file-name="Gemfile" pinned="false" current-in-tab="false">
|
|
89
93
|
<entry file="file://$PROJECT_DIR$/Gemfile">
|
|
90
94
|
<provider selected="true" editor-type-id="text-editor">
|
|
91
|
-
<state relative-caret-position="
|
|
95
|
+
<state relative-caret-position="168">
|
|
92
96
|
<caret line="8" column="23" selection-start-line="8" selection-start-column="23" selection-end-line="8" selection-end-column="23" />
|
|
93
97
|
</state>
|
|
94
98
|
</provider>
|
|
@@ -247,17 +251,18 @@
|
|
|
247
251
|
<workItem from="1546240992243" duration="719000" />
|
|
248
252
|
<workItem from="1546291493927" duration="464000" />
|
|
249
253
|
<workItem from="1546436457874" duration="2443000" />
|
|
254
|
+
<workItem from="1549964225949" duration="5000" />
|
|
250
255
|
</task>
|
|
251
256
|
<servers />
|
|
252
257
|
</component>
|
|
253
258
|
<component name="TimeTrackingManager">
|
|
254
|
-
<option name="totallyTimeSpent" value="
|
|
259
|
+
<option name="totallyTimeSpent" value="23113000" />
|
|
255
260
|
</component>
|
|
256
261
|
<component name="ToolWindowManager">
|
|
257
|
-
<frame x="0" y="0" width="
|
|
262
|
+
<frame x="0" y="0" width="1680" height="1050" extended-state="0" />
|
|
258
263
|
<editor active="true" />
|
|
259
264
|
<layout>
|
|
260
|
-
<window_info
|
|
265
|
+
<window_info content_ui="combo" id="Project" order="0" visible="true" weight="0.34188035" />
|
|
261
266
|
<window_info anchor="bottom" id="TODO" order="6" />
|
|
262
267
|
<window_info anchor="bottom" id="Docker" order="7" show_stripe_button="false" />
|
|
263
268
|
<window_info anchor="bottom" id="Event Log" order="7" side_tool="true" />
|
|
@@ -265,7 +270,7 @@
|
|
|
265
270
|
<window_info anchor="bottom" id="Database Changes" order="7" show_stripe_button="false" />
|
|
266
271
|
<window_info anchor="bottom" id="Version Control" order="7" />
|
|
267
272
|
<window_info id="Structure" order="1" side_tool="true" weight="0.25" />
|
|
268
|
-
<window_info anchor="bottom" id="Terminal" order="7" visible="true" weight="0.
|
|
273
|
+
<window_info active="true" anchor="bottom" id="Terminal" order="7" visible="true" weight="0.45725647" />
|
|
269
274
|
<window_info id="Favorites" order="2" side_tool="true" />
|
|
270
275
|
<window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
|
|
271
276
|
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
|
|
@@ -554,50 +559,50 @@
|
|
|
554
559
|
</entry>
|
|
555
560
|
<entry file="file://$PROJECT_DIR$/lib/http_crawler/common.rb">
|
|
556
561
|
<provider selected="true" editor-type-id="text-editor">
|
|
557
|
-
<state relative-caret-position="
|
|
562
|
+
<state relative-caret-position="42">
|
|
558
563
|
<caret line="2" selection-start-line="2" selection-end-line="2" />
|
|
559
564
|
</state>
|
|
560
565
|
</provider>
|
|
561
566
|
</entry>
|
|
562
567
|
<entry file="file://$PROJECT_DIR$/lib/http_crawler/proxy.rb">
|
|
563
568
|
<provider selected="true" editor-type-id="text-editor">
|
|
564
|
-
<state relative-caret-position="
|
|
569
|
+
<state relative-caret-position="441">
|
|
565
570
|
<caret line="21" selection-start-line="21" selection-end-line="21" />
|
|
566
571
|
</state>
|
|
567
572
|
</provider>
|
|
568
573
|
</entry>
|
|
569
574
|
<entry file="file://$PROJECT_DIR$/lib/http_crawler/web.rb">
|
|
570
575
|
<provider selected="true" editor-type-id="text-editor">
|
|
571
|
-
<state relative-caret-position="
|
|
576
|
+
<state relative-caret-position="126">
|
|
572
577
|
<caret line="6" selection-start-line="6" selection-end-line="6" />
|
|
573
578
|
</state>
|
|
574
579
|
</provider>
|
|
575
580
|
</entry>
|
|
576
581
|
<entry file="file://$PROJECT_DIR$/lib/http_crawler/web/baidu/client.rb">
|
|
577
582
|
<provider selected="true" editor-type-id="text-editor">
|
|
578
|
-
<state relative-caret-position="
|
|
583
|
+
<state relative-caret-position="21">
|
|
579
584
|
<caret line="1" column="18" selection-start-line="1" selection-start-column="7" selection-end-line="1" selection-end-column="18" />
|
|
580
585
|
</state>
|
|
581
586
|
</provider>
|
|
582
587
|
</entry>
|
|
583
588
|
<entry file="file://$PROJECT_DIR$/Gemfile">
|
|
584
589
|
<provider selected="true" editor-type-id="text-editor">
|
|
585
|
-
<state relative-caret-position="
|
|
590
|
+
<state relative-caret-position="168">
|
|
586
591
|
<caret line="8" column="23" selection-start-line="8" selection-start-column="23" selection-end-line="8" selection-end-column="23" />
|
|
587
592
|
</state>
|
|
588
593
|
</provider>
|
|
589
594
|
</entry>
|
|
590
595
|
<entry file="file://$APPLICATION_HOME_DIR$/rubystubs24/errno.rb">
|
|
591
596
|
<provider selected="true" editor-type-id="text-editor">
|
|
592
|
-
<state relative-caret-position="
|
|
597
|
+
<state relative-caret-position="672">
|
|
593
598
|
<caret line="32" column="7" selection-start-line="32" selection-start-column="7" selection-end-line="32" selection-end-column="7" />
|
|
594
599
|
</state>
|
|
595
600
|
</provider>
|
|
596
601
|
</entry>
|
|
597
602
|
<entry file="file://$PROJECT_DIR$/lib/http_crawler/http.rb">
|
|
598
603
|
<provider selected="true" editor-type-id="text-editor">
|
|
599
|
-
<state relative-caret-position="
|
|
600
|
-
<caret line="210" column="
|
|
604
|
+
<state relative-caret-position="4410">
|
|
605
|
+
<caret line="210" column="11" lean-forward="true" selection-start-line="210" selection-start-column="11" selection-end-line="210" selection-end-column="11" />
|
|
601
606
|
</state>
|
|
602
607
|
</provider>
|
|
603
608
|
</entry>
|
|
@@ -610,15 +615,15 @@
|
|
|
610
615
|
</entry>
|
|
611
616
|
<entry file="file://$USER_HOME$/.rvm/rubies/ruby-2.4.1/lib/ruby/2.4.0/webrick/compat.rb">
|
|
612
617
|
<provider selected="true" editor-type-id="text-editor">
|
|
613
|
-
<state relative-caret-position="
|
|
618
|
+
<state relative-caret-position="378">
|
|
614
619
|
<caret line="18" column="1" selection-start-line="18" selection-start-column="1" selection-end-line="18" selection-end-column="1" />
|
|
615
620
|
</state>
|
|
616
621
|
</provider>
|
|
617
622
|
</entry>
|
|
618
623
|
<entry file="file://$PROJECT_DIR$/lib/http_crawler/version.rb">
|
|
619
624
|
<provider selected="true" editor-type-id="text-editor">
|
|
620
|
-
<state relative-caret-position="
|
|
621
|
-
<caret line="
|
|
625
|
+
<state relative-caret-position="21">
|
|
626
|
+
<caret line="1" column="21" selection-start-line="1" selection-start-column="21" selection-end-line="1" selection-end-column="21" />
|
|
622
627
|
</state>
|
|
623
628
|
</provider>
|
|
624
629
|
</entry>
|
data/lib/http_crawler/http.rb
CHANGED
|
@@ -5,6 +5,11 @@ module HttpCrawler
|
|
|
5
5
|
attr_accessor :auto_proxy
|
|
6
6
|
# 代理API的别名 主要关联 HttpCrawler::Proxy中维护的代理API
|
|
7
7
|
attr_accessor :proxy_api
|
|
8
|
+
|
|
9
|
+
def proxy_api
|
|
10
|
+
@proxy_api ||= "my"
|
|
11
|
+
end
|
|
12
|
+
|
|
8
13
|
# 调用自己的代理池所需要的主键 key
|
|
9
14
|
attr_accessor :proxy_key
|
|
10
15
|
# 请求错误后的重复最大请求次数
|
|
@@ -28,9 +33,6 @@ module HttpCrawler
|
|
|
28
33
|
sleep(3)
|
|
29
34
|
end
|
|
30
35
|
|
|
31
|
-
def proxy_api
|
|
32
|
-
@proxy_api ||= "my"
|
|
33
|
-
end
|
|
34
36
|
|
|
35
37
|
@@proxy_list = []
|
|
36
38
|
# 为 @http 重设代理
|
data/lib/http_crawler/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: http_crawler
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.2.3.
|
|
4
|
+
version: 0.2.3.3
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- jagger
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2019-
|
|
11
|
+
date: 2019-02-12 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: rspec
|
|
@@ -175,7 +175,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
175
175
|
version: '0'
|
|
176
176
|
requirements: []
|
|
177
177
|
rubyforge_project:
|
|
178
|
-
rubygems_version: 2.
|
|
178
|
+
rubygems_version: 2.7.8
|
|
179
179
|
signing_key:
|
|
180
180
|
specification_version: 4
|
|
181
181
|
summary: http 爬虫。
|