grell 1.6.10 → 2.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.travis.yml +17 -3
- data/CHANGELOG.md +24 -0
- data/Gemfile +4 -0
- data/README.md +100 -63
- data/grell.gemspec +6 -7
- data/lib/grell.rb +1 -0
- data/lib/grell/capybara_driver.rb +9 -19
- data/lib/grell/crawler.rb +27 -53
- data/lib/grell/crawler_manager.rb +84 -0
- data/lib/grell/page.rb +2 -1
- data/lib/grell/page_collection.rb +9 -1
- data/lib/grell/rawpage.rb +11 -1
- data/lib/grell/version.rb +1 -1
- data/spec/lib/capybara_driver_spec.rb +3 -8
- data/spec/lib/crawler_manager_spec.rb +174 -0
- data/spec/lib/crawler_spec.rb +52 -93
- data/spec/lib/page_spec.rb +11 -0
- data/spec/spec_helper.rb +1 -0
- metadata +15 -42
data/spec/lib/page_spec.rb
CHANGED
@@ -92,6 +92,17 @@ RSpec.describe Grell::Page do
|
|
92
92
|
end
|
93
93
|
end
|
94
94
|
|
95
|
+
describe '#navigate' do
|
96
|
+
before do
|
97
|
+
proxy.stub(url).and_return(body: '', code: 200, headers: {})
|
98
|
+
end
|
99
|
+
|
100
|
+
it 'waits for all ajax requests' do
|
101
|
+
expect_any_instance_of(Grell::RawPage).to receive(:wait_for_all_ajax_requests).with(0, 0.5)
|
102
|
+
page.navigate
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
95
106
|
shared_examples_for 'an errored grell page' do
|
96
107
|
it 'returns empty status 404 page after navigating' do
|
97
108
|
expect(page.status).to eq(404)
|
data/spec/spec_helper.rb
CHANGED
@@ -23,6 +23,7 @@ RSpec.configure do |config|
|
|
23
23
|
# We do not need to wait for pages to return all the data
|
24
24
|
config.before do
|
25
25
|
stub_const("Grell::Page::WAIT_TIME", 0)
|
26
|
+
allow_any_instance_of(Capybara::Session).to receive(:evaluate_script).and_return(nil)
|
26
27
|
end
|
27
28
|
|
28
29
|
config.expect_with :rspec do |expectations|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: grell
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 2.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jordi Polo Carres
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-02-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: capybara
|
@@ -16,42 +16,28 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '2.
|
19
|
+
version: '2.10'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '2.
|
26
|
+
version: '2.10'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: poltergeist
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '1.
|
33
|
+
version: '1.11'
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: '1.
|
41
|
-
- !ruby/object:Gem::Dependency
|
42
|
-
name: bundler
|
43
|
-
requirement: !ruby/object:Gem::Requirement
|
44
|
-
requirements:
|
45
|
-
- - "~>"
|
46
|
-
- !ruby/object:Gem::Version
|
47
|
-
version: '1.6'
|
48
|
-
type: :development
|
49
|
-
prerelease: false
|
50
|
-
version_requirements: !ruby/object:Gem::Requirement
|
51
|
-
requirements:
|
52
|
-
- - "~>"
|
53
|
-
- !ruby/object:Gem::Version
|
54
|
-
version: '1.6'
|
40
|
+
version: '1.11'
|
55
41
|
- !ruby/object:Gem::Dependency
|
56
42
|
name: byebug
|
57
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -114,28 +100,28 @@ dependencies:
|
|
114
100
|
requirements:
|
115
101
|
- - "~>"
|
116
102
|
- !ruby/object:Gem::Version
|
117
|
-
version: '3.
|
103
|
+
version: '3.5'
|
118
104
|
type: :development
|
119
105
|
prerelease: false
|
120
106
|
version_requirements: !ruby/object:Gem::Requirement
|
121
107
|
requirements:
|
122
108
|
- - "~>"
|
123
109
|
- !ruby/object:Gem::Version
|
124
|
-
version: '3.
|
110
|
+
version: '3.5'
|
125
111
|
- !ruby/object:Gem::Dependency
|
126
112
|
name: puffing-billy
|
127
113
|
requirement: !ruby/object:Gem::Requirement
|
128
114
|
requirements:
|
129
115
|
- - "~>"
|
130
116
|
- !ruby/object:Gem::Version
|
131
|
-
version: '0.
|
117
|
+
version: '0.9'
|
132
118
|
type: :development
|
133
119
|
prerelease: false
|
134
120
|
version_requirements: !ruby/object:Gem::Requirement
|
135
121
|
requirements:
|
136
122
|
- - "~>"
|
137
123
|
- !ruby/object:Gem::Version
|
138
|
-
version: '0.
|
124
|
+
version: '0.9'
|
139
125
|
- !ruby/object:Gem::Dependency
|
140
126
|
name: timecop
|
141
127
|
requirement: !ruby/object:Gem::Requirement
|
@@ -150,20 +136,6 @@ dependencies:
|
|
150
136
|
- - "~>"
|
151
137
|
- !ruby/object:Gem::Version
|
152
138
|
version: '0.8'
|
153
|
-
- !ruby/object:Gem::Dependency
|
154
|
-
name: capybara-webkit
|
155
|
-
requirement: !ruby/object:Gem::Requirement
|
156
|
-
requirements:
|
157
|
-
- - "~>"
|
158
|
-
- !ruby/object:Gem::Version
|
159
|
-
version: 1.11.1
|
160
|
-
type: :development
|
161
|
-
prerelease: false
|
162
|
-
version_requirements: !ruby/object:Gem::Requirement
|
163
|
-
requirements:
|
164
|
-
- - "~>"
|
165
|
-
- !ruby/object:Gem::Version
|
166
|
-
version: 1.11.1
|
167
139
|
- !ruby/object:Gem::Dependency
|
168
140
|
name: selenium-webdriver
|
169
141
|
requirement: !ruby/object:Gem::Requirement
|
@@ -196,6 +168,7 @@ files:
|
|
196
168
|
- lib/grell.rb
|
197
169
|
- lib/grell/capybara_driver.rb
|
198
170
|
- lib/grell/crawler.rb
|
171
|
+
- lib/grell/crawler_manager.rb
|
199
172
|
- lib/grell/grell_logger.rb
|
200
173
|
- lib/grell/page.rb
|
201
174
|
- lib/grell/page_collection.rb
|
@@ -203,6 +176,7 @@ files:
|
|
203
176
|
- lib/grell/reader.rb
|
204
177
|
- lib/grell/version.rb
|
205
178
|
- spec/lib/capybara_driver_spec.rb
|
179
|
+
- spec/lib/crawler_manager_spec.rb
|
206
180
|
- spec/lib/crawler_spec.rb
|
207
181
|
- spec/lib/page_collection_spec.rb
|
208
182
|
- spec/lib/page_spec.rb
|
@@ -220,23 +194,22 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
220
194
|
requirements:
|
221
195
|
- - ">="
|
222
196
|
- !ruby/object:Gem::Version
|
223
|
-
version: 1.
|
197
|
+
version: 2.1.8
|
224
198
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
225
199
|
requirements:
|
226
200
|
- - ">="
|
227
201
|
- !ruby/object:Gem::Version
|
228
202
|
version: '0'
|
229
203
|
requirements: []
|
230
|
-
|
231
|
-
rubygems_version: 2.5.1
|
204
|
+
rubygems_version: 3.0.8
|
232
205
|
signing_key:
|
233
206
|
specification_version: 4
|
234
207
|
summary: Ruby web crawler
|
235
208
|
test_files:
|
236
209
|
- spec/lib/capybara_driver_spec.rb
|
210
|
+
- spec/lib/crawler_manager_spec.rb
|
237
211
|
- spec/lib/crawler_spec.rb
|
238
212
|
- spec/lib/page_collection_spec.rb
|
239
213
|
- spec/lib/page_spec.rb
|
240
214
|
- spec/lib/reader_spec.rb
|
241
215
|
- spec/spec_helper.rb
|
242
|
-
has_rdoc:
|