polipus 0.3.7 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. checksums.yaml +8 -8
  2. data/.rspec +1 -1
  3. data/.rubocop.yml +3 -3
  4. data/.rubocop_todo.yml +1 -1
  5. data/.travis.yml +14 -4
  6. data/AUTHORS.md +1 -0
  7. data/CHANGELOG.md +9 -1
  8. data/Gemfile +9 -0
  9. data/README.md +2 -3
  10. data/Rakefile +1 -3
  11. data/examples/basic.rb +8 -1
  12. data/lib/polipus.rb +25 -13
  13. data/lib/polipus/queue_overflow.rb +1 -0
  14. data/lib/polipus/queue_overflow/manager.rb +1 -0
  15. data/lib/polipus/queue_overflow/mongo_queue.rb +1 -1
  16. data/lib/polipus/queue_overflow/worker.rb +24 -0
  17. data/lib/polipus/storage.rb +10 -16
  18. data/lib/polipus/storage/mongo_store.rb +6 -1
  19. data/lib/polipus/storage/rethink_store.rb +90 -0
  20. data/lib/polipus/version.rb +1 -1
  21. data/polipus.gemspec +16 -18
  22. data/spec/{http_spec.rb → polipus/http_spec.rb} +26 -37
  23. data/spec/{page_spec.rb → polipus/page_spec.rb} +7 -11
  24. data/spec/{queue_overflow_manager_spec.rb → polipus/queue_overflow/manager_spec.rb} +22 -29
  25. data/spec/{queue_overflow_spec.rb → polipus/queue_overflow_spec.rb} +14 -20
  26. data/spec/{robotex_spec.rb → polipus/robotex_spec.rb} +10 -11
  27. data/spec/{signal_handler_spec.rb → polipus/signal_handler_spec.rb} +2 -6
  28. data/spec/{storage_memory_spec.rb → polipus/storage/memory_store_spec.rb} +18 -21
  29. data/spec/{storage_mongo_spec.rb → polipus/storage/mongo_store_spec.rb} +23 -25
  30. data/spec/polipus/storage/rethink_store_spec.rb +117 -0
  31. data/spec/{url_tracker_spec.rb → polipus/url_tracker_spec.rb} +4 -4
  32. data/spec/polipus_spec.rb +13 -15
  33. data/spec/spec_helper.rb +13 -12
  34. metadata +76 -154
  35. data/lib/polipus/storage/s3_store.rb +0 -96
  36. data/spec/cassettes/08b228db424a926e1ed6ab63b38d847e.yml +0 -166
  37. data/spec/cassettes/20aa41f181b49f00078c3ca30bad5afe.yml +0 -166
  38. data/spec/cassettes/4640919145753505af2d0f8423de37f3.yml +0 -270
  39. data/spec/cassettes/66aae15a03f4aab8efd15e40d2d7882a.yml +0 -194
  40. data/spec/cassettes/76b7c197c95a5bf9b1e882c567192d72.yml +0 -183
  41. data/spec/cassettes/9b1d523b7f5db7214f8a8bd9272cccba.yml +0 -221
  42. data/spec/cassettes/ab333f89535a2efb284913fede6aa7c7.yml +0 -221
  43. data/spec/cassettes/ae5d7cffde3f53122cdf79f3d1367e8e.yml +0 -221
  44. data/spec/cassettes/ffe3d588b6df4b9de35e5a7ccaf5a81b.yml +0 -695
  45. data/spec/storage_s3_spec.rb +0 -115
data/spec/{signal_handler_spec.rb → polipus/signal_handler_spec.rb} RENAMED
@@ -1,19 +1,15 @@
1
- # encoding: UTF-8
2
1
  require 'spec_helper'
3
2
 
4
3
  describe Polipus::SignalHandler do
5
-
6
4
  context 'signal handler' do
7
-
8
5
  it 'should be enabled by default' do
9
6
  Polipus::PolipusCrawler.new('polipus-rspec', [])
10
- Polipus::SignalHandler.enabled?.should be true
7
+ expect(Polipus::SignalHandler.enabled?).to be true
11
8
  end
12
9
 
13
10
  it 'should be disabled if specified' do
14
11
  Polipus::PolipusCrawler.new('polipus-rspec', [], enable_signal_handler: false)
15
- Polipus::SignalHandler.enabled?.should be false
12
+ expect(Polipus::SignalHandler.enabled?).to be false
16
13
  end
17
-
18
14
  end
19
15
  end
data/spec/{storage_memory_spec.rb → polipus/storage/memory_store_spec.rb} RENAMED
@@ -3,38 +3,37 @@ require 'spec_helper'
3
3
  require 'mongo'
4
4
  require 'polipus/storage/memory_store'
5
5
  describe Polipus::Storage::MemoryStore do
6
-
7
6
  let(:storage) { Polipus::Storage.memory_store }
8
7
 
9
8
  it 'should store a page' do
10
9
  p = page_factory 'http://www.google.com', code: 200, body: '<html></html>'
11
10
  uuid = storage.add p
12
- uuid.should be == 'ed646a3334ca891fd3467db131372140'
13
- storage.count.should be 1
11
+ expect(uuid).to eq('ed646a3334ca891fd3467db131372140')
12
+ expect(storage.count).to be 1
14
13
  p = storage.get p
15
- p.url.to_s.should be == 'http://www.google.com'
16
- p.body.should be == '<html></html>'
14
+ expect(p.url.to_s).to eq('http://www.google.com')
15
+ expect(p.body).to eq('<html></html>')
17
16
  end
18
17
 
19
18
  it 'should update a page' do
20
19
  p = page_factory 'http://www.google.com', code: 301, body: '<html></html>'
21
20
  storage.add p
22
21
  p = storage.get p
23
- p.code.should be == 301
22
+ expect(p.code).to eq(301)
24
23
  end
25
24
 
26
25
  it 'should iterate over stored pages' do
27
26
  storage.each do |k, page|
28
- k.should be == 'ed646a3334ca891fd3467db131372140'
29
- page.url.to_s.should be == 'http://www.google.com'
27
+ expect(k).to eq('ed646a3334ca891fd3467db131372140')
28
+ expect(page.url.to_s).to eq('http://www.google.com')
30
29
  end
31
30
  end
32
31
 
33
32
  it 'should delete a page' do
34
33
  p = page_factory 'http://www.google.com', code: 301, body: '<html></html>'
35
34
  storage.remove p
36
- storage.get(p).should be_nil
37
- storage.count.should be 0
35
+ expect(storage.get(p)).to be_nil
36
+ expect(storage.count).to be 0
38
37
  end
39
38
 
40
39
  it 'should store a page removing a query string from the uuid generation' do
@@ -42,7 +41,7 @@ describe Polipus::Storage::MemoryStore do
42
41
  p_no_query = page_factory 'http://www.asd.com/?asdas=dasda&adsda=1', code: 200, body: '<html></html>'
43
42
  storage.include_query_string_in_uuid = false
44
43
  storage.add p
45
- storage.exists?(p_no_query).should be_true
44
+ expect(storage.exists?(p_no_query)).to be_truthy
46
45
  storage.remove p
47
46
  end
48
47
 
@@ -51,7 +50,7 @@ describe Polipus::Storage::MemoryStore do
51
50
  p_no_query = page_factory 'http://www.asd.com', code: 200, body: '<html></html>'
52
51
  storage.include_query_string_in_uuid = false
53
52
  storage.add p
54
- storage.exists?(p_no_query).should be_true
53
+ expect(storage.exists?(p_no_query)).to be_truthy
55
54
  storage.remove p
56
55
  end
57
56
 
@@ -59,9 +58,9 @@ describe Polipus::Storage::MemoryStore do
59
58
  p = page_factory 'http://www.user.com', code: 200, body: '<html></html>'
60
59
  p.user_data.name = 'Test User Data'
61
60
  storage.add p
62
- storage.exists?(p).should be_true
61
+ expect(storage.exists?(p)).to be_truthy
63
62
  p = storage.get(p)
64
- p.user_data.name.should be == 'Test User Data'
63
+ expect(p.user_data.name).to eq('Test User Data')
65
64
  storage.remove p
66
65
  end
67
66
 
@@ -70,21 +69,19 @@ describe Polipus::Storage::MemoryStore do
70
69
  p = page_factory 'http://www.user-doo.com', code: 200, body: '<html></html>'
71
70
  storage.add p
72
71
  p = storage.get p
73
- p.body.should be_empty
72
+ expect(p.body).to be_empty
74
73
  storage.clear
75
74
  end
76
75
 
77
76
  it 'should return false if a doc not exists' do
78
77
  storage.include_query_string_in_uuid = false
79
78
  p_other = page_factory 'http://www.asdrrrr.com', code: 200, body: '<html></html>'
80
- storage.exists?(p_other).should be_false
79
+ expect(storage.exists?(p_other)).to be_falsey
81
80
  storage.add p_other
82
- storage.exists?(p_other).should be_true
81
+ expect(storage.exists?(p_other)).to be_truthy
83
82
  p_other = page_factory 'http://www.asdrrrr.com?trk=asd-lol', code: 200, body: '<html></html>'
84
- storage.exists?(p_other).should be_true
83
+ expect(storage.exists?(p_other)).to be_truthy
85
84
  storage.include_query_string_in_uuid = true
86
- storage.exists?(p_other).should be_false
87
-
85
+ expect(storage.exists?(p_other)).to be_falsey
88
86
  end
89
-
90
87
  end
data/spec/{storage_mongo_spec.rb → polipus/storage/mongo_store_spec.rb} RENAMED
@@ -20,34 +20,34 @@ describe Polipus::Storage::MongoStore do
20
20
  it 'should store a page' do
21
21
  p = page_factory 'http://www.google.com', code: 200, body: '<html></html>'
22
22
  uuid = @storage.add p
23
- uuid.should be == 'ed646a3334ca891fd3467db131372140'
24
- @storage.count.should be 1
25
- @mongo['_test_pages'].count.should be 1
23
+ expect(uuid).to eq('ed646a3334ca891fd3467db131372140')
24
+ expect(@storage.count).to be 1
25
+ expect(@mongo['_test_pages'].count).to be 1
26
26
  p = @storage.get p
27
- p.url.to_s.should be == 'http://www.google.com'
28
- p.body.should be == '<html></html>'
27
+ expect(p.url.to_s).to eq('http://www.google.com')
28
+ expect(p.body).to eq('<html></html>')
29
29
  end
30
30
 
31
31
  it 'should update a page' do
32
32
  p = page_factory 'http://www.google.com', code: 301, body: '<html></html>'
33
33
  @storage.add p
34
34
  p = @storage.get p
35
- p.code.should be == 301
36
- @mongo['_test_pages'].count.should be 1
35
+ expect(p.code).to eq(301)
36
+ expect(@mongo['_test_pages'].count).to be 1
37
37
  end
38
38
 
39
39
  it 'should iterate over stored pages' do
40
40
  @storage.each do |k, page|
41
- k.should be == 'ed646a3334ca891fd3467db131372140'
42
- page.url.to_s.should be == 'http://www.google.com'
41
+ expect(k).to eq('ed646a3334ca891fd3467db131372140')
42
+ expect(page.url.to_s).to eq('http://www.google.com')
43
43
  end
44
44
  end
45
45
 
46
46
  it 'should delete a page' do
47
47
  p = page_factory 'http://www.google.com', code: 301, body: '<html></html>'
48
48
  @storage.remove p
49
- @storage.get(p).should be_nil
50
- @storage.count.should be 0
49
+ expect(@storage.get(p)).to be_nil
50
+ expect(@storage.count).to be 0
51
51
  end
52
52
 
53
53
  it 'should store a page removing a query string from the uuid generation' do
@@ -55,7 +55,7 @@ describe Polipus::Storage::MongoStore do
55
55
  p_no_query = page_factory 'http://www.asd.com/?asdas=dasda&adsda=1', code: 200, body: '<html></html>'
56
56
  @storage.include_query_string_in_uuid = false
57
57
  @storage.add p
58
- @storage.exists?(p_no_query).should be_true
58
+ expect(@storage.exists?(p_no_query)).to be_truthy
59
59
  @storage.remove p
60
60
  end
61
61
 
@@ -64,7 +64,7 @@ describe Polipus::Storage::MongoStore do
64
64
  p_no_query = page_factory 'http://www.asd.com', code: 200, body: '<html></html>'
65
65
  @storage.include_query_string_in_uuid = false
66
66
  @storage.add p
67
- @storage.exists?(p_no_query).should be_true
67
+ expect(@storage.exists?(p_no_query)).to be_truthy
68
68
  @storage.remove p
69
69
  end
70
70
 
@@ -72,9 +72,9 @@ describe Polipus::Storage::MongoStore do
72
72
  p = page_factory 'http://www.user.com', code: 200, body: '<html></html>'
73
73
  p.user_data.name = 'Test User Data'
74
74
  @storage.add p
75
- @storage.exists?(p).should be_true
75
+ expect(@storage.exists?(p)).to be_truthy
76
76
  p = @storage.get(p)
77
- p.user_data.name.should be == 'Test User Data'
77
+ expect(p.user_data.name).to eq('Test User Data')
78
78
  @storage.remove p
79
79
  end
80
80
 
@@ -83,30 +83,29 @@ describe Polipus::Storage::MongoStore do
83
83
  p = page_factory 'http://www.user-doo.com', code: 200, body: '<html></html>'
84
84
  storage.add p
85
85
  p = storage.get p
86
- p.body.should be_empty
86
+ expect(p.body).to be_empty
87
87
  storage.clear
88
88
  end
89
89
 
90
90
  it 'should return false if a doc not exists' do
91
91
  @storage.include_query_string_in_uuid = false
92
92
  p_other = page_factory 'http://www.asdrrrr.com', code: 200, body: '<html></html>'
93
- @storage.exists?(p_other).should be_false
93
+ expect(@storage.exists?(p_other)).to be_falsey
94
94
  @storage.add p_other
95
- @storage.exists?(p_other).should be_true
95
+ expect(@storage.exists?(p_other)).to be_truthy
96
96
  p_other = page_factory 'http://www.asdrrrr.com?trk=asd-lol', code: 200, body: '<html></html>'
97
- @storage.exists?(p_other).should be_true
97
+ expect(@storage.exists?(p_other)).to be_truthy
98
98
  @storage.include_query_string_in_uuid = true
99
- @storage.exists?(p_other).should be_false
100
-
99
+ expect(@storage.exists?(p_other)).to be_falsey
101
100
  end
102
101
 
103
102
  it 'should set page.fetched_at based on the id creation' do
104
103
  storage = Polipus::Storage.mongo_store(@mongo, '_test_pages')
105
104
  p = page_factory 'http://www.user-doojo.com', code: 200, body: '<html></html>'
106
105
  storage.add p
107
- p.fetched_at.should be_nil
106
+ expect(p.fetched_at).to be_nil
108
107
  p = storage.get p
109
- p.fetched_at.should_not be_nil
108
+ expect(p.fetched_at).not_to be_nil
110
109
  end
111
110
 
112
111
  it 'should NOT set page.fetched_at if already present' do
@@ -115,7 +114,6 @@ describe Polipus::Storage::MongoStore do
115
114
  p.fetched_at = 10
116
115
  storage.add p
117
116
  p = storage.get p
118
- p.fetched_at.should be 10
117
+ expect(p.fetched_at).to be 10
119
118
  end
120
-
121
119
  end
data/spec/polipus/storage/rethink_store_spec.rb ADDED
@@ -0,0 +1,117 @@
1
+ # encoding: UTF-8
2
+ require 'spec_helper'
3
+ require 'polipus/storage/rethink_store'
4
+
5
+ describe Polipus::Storage::RethinkStore do
6
+ before(:all)do
7
+ @r = RethinkDB::RQL.new
8
+ @rethink = @r.connect(host: 'localhost', port: 28_015, db: 'polipus_spec')
9
+ @r.db_create('polipus_spec').run(@rethink) unless @r.db_list.run(@rethink).include?('polipus_spec')
10
+ @table = 'test_pages'
11
+ @storage = Polipus::Storage.rethink_store(@rethink, @table)
12
+ end
13
+
14
+ after(:each) do
15
+ @r.table(@table).delete.run(@rethink)
16
+ end
17
+
18
+ it 'should store a page' do
19
+ page = page_factory 'http://www.google.com', code: 200, body: '<html></html>'
20
+ uuid = @storage.add page
21
+ expect(uuid).to eq('ed646a3334ca891fd3467db131372140')
22
+ expect(@storage.count).to eq(1)
23
+ expect(@r.table(@table).count.run(@rethink)).to eq(1)
24
+ page = @storage.get page
25
+ expect(page.url.to_s).to eq('http://www.google.com')
26
+ expect(page.body).to eq('<html></html>')
27
+ end
28
+
29
+ it 'should update a page' do
30
+ page = page_factory 'http://www.google.com', code: 301, body: '<html></html>'
31
+ @storage.add page
32
+ page = @storage.get page
33
+ expect(page.code).to eq(301)
34
+ expect(@r.table(@table).count.run(@rethink)).to eq(1)
35
+ end
36
+
37
+ it 'should iterate over stored pages' do
38
+ @storage.each do |k, page|
39
+ expect(k).to eq('ed646a3334ca891fd3467db131372140')
40
+ expect(page.url.to_s).to eq('http://www.google.com')
41
+ end
42
+ end
43
+
44
+ it 'should delete a page' do
45
+ page = page_factory 'http://www.google.com', code: 301, body: '<html></html>'
46
+ @storage.remove page
47
+ expect(@storage.get(page)).to be_nil
48
+ expect(@storage.count).to be 0
49
+ end
50
+
51
+ it 'should store a page removing a query string from the uuid generation' do
52
+ page = page_factory 'http://www.asd.com/?asd=lol', code: 200, body: '<html></html>'
53
+ p_no_query = page_factory 'http://www.asd.com/?asdas=dasda&adsda=1', code: 200, body: '<html></html>'
54
+ @storage.include_query_string_in_uuid = false
55
+ @storage.add page
56
+ expect(@storage.exists?(p_no_query)).to be_truthy
57
+ @storage.remove page
58
+ end
59
+
60
+ it 'should store a page removing a query string from the uuid generation no ending slash' do
61
+ page = page_factory 'http://www.asd.com?asd=lol', code: 200, body: '<html></html>'
62
+ p_no_query = page_factory 'http://www.asd.com', code: 200, body: '<html></html>'
63
+ @storage.include_query_string_in_uuid = false
64
+ @storage.add page
65
+ expect(@storage.exists?(p_no_query)).to be_truthy
66
+ @storage.remove page
67
+ end
68
+
69
+ it 'should store a page with user data associated' do
70
+ page = page_factory 'http://www.user.com', code: 200, body: '<html></html>'
71
+ page.user_data.name = 'Test User Data'
72
+ @storage.add page
73
+ expect(@storage.exists?(page)).to be_truthy
74
+ page = @storage.get(page)
75
+ expect(page.user_data.name).to eq('Test User Data')
76
+ @storage.remove page
77
+ end
78
+
79
+ it 'should honor the except parameters' do
80
+ storage = Polipus::Storage.rethink_store(@rethink, @table, ['body'])
81
+ page = page_factory 'http://www.user-doo.com', code: 200, body: '<html></html>'
82
+ storage.add page
83
+ page = storage.get page
84
+ expect(page.body).to be_empty
85
+ storage.clear
86
+ end
87
+
88
+ it 'should return false if a doc not exists' do
89
+ @storage.include_query_string_in_uuid = false
90
+ p_other = page_factory 'http://www.asdrrrr.com', code: 200, body: '<html></html>'
91
+ expect(@storage.exists?(p_other)).to be_falsey
92
+ @storage.add p_other
93
+ expect(@storage.exists?(p_other)).to be_truthy
94
+ p_other = page_factory 'http://www.asdrrrr.com?trk=asd-lol', code: 200, body: '<html></html>'
95
+ expect(@storage.exists?(p_other)).to be_truthy
96
+ @storage.include_query_string_in_uuid = true
97
+ expect(@storage.exists?(p_other)).to be_falsey
98
+ end
99
+
100
+ it 'should set page.fetched_at based on the id creation' do
101
+ storage = Polipus::Storage.rethink_store(@rethink, @table)
102
+ page = page_factory 'http://www.user-doojo.com', code: 200, body: '<html></html>'
103
+ storage.add page
104
+ expect(page.fetched_at).to be_nil
105
+ page = storage.get page
106
+ expect(page.fetched_at).not_to be_nil
107
+ end
108
+
109
+ it 'should NOT set page.fetched_at if already present' do
110
+ storage = Polipus::Storage.rethink_store(@rethink, @table)
111
+ page = page_factory 'http://www.user-doojooo.com', code: 200, body: '<html></html>'
112
+ page.fetched_at = 10
113
+ storage.add page
114
+ page = storage.get page
115
+ expect(page.fetched_at).to be 10
116
+ end
117
+ end
data/spec/{url_tracker_spec.rb → polipus/url_tracker_spec.rb} RENAMED
@@ -16,14 +16,14 @@ describe Polipus::UrlTracker do
16
16
  it 'should work (bf)' do
17
17
  url = 'http://www.asd.com/asd/lol'
18
18
  @bf.visit url
19
- @bf.visited?(url).should be_true
20
- @bf.visited?('http://www.google.com').should be_false
19
+ expect(@bf.visited?(url)).to be_truthy
20
+ expect(@bf.visited?('http://www.google.com')).to be_falsey
21
21
  end
22
22
 
23
23
  it 'should work (redis_set)' do
24
24
  url = 'http://www.asd.com/asd/lol'
25
25
  @set.visit url
26
- @set.visited?(url).should be_true
27
- @set.visited?('http://www.google.com').should be_false
26
+ expect(@set.visited?(url)).to be_truthy
27
+ expect(@set.visited?('http://www.google.com')).to be_falsey
28
28
  end
29
29
  end
data/spec/polipus_spec.rb CHANGED
@@ -26,31 +26,30 @@ describe Polipus::PolipusCrawler do
26
26
  let(:logger) { Logger.new(nil) }
27
27
 
28
28
  context 'polipus' do
29
-
30
29
  it 'should create a polipus instance' do
31
- polipus.should be_an_instance_of Polipus::PolipusCrawler
30
+ expect(polipus).to be_an_instance_of Polipus::PolipusCrawler
32
31
  end
33
32
 
34
33
  it 'should execute a crawling session' do
35
34
  polipus.takeover
36
- polipus.storage.exists?(init_page).should be_true
37
- polipus.storage.get(init_page).links.count.should be polipus.storage.count
35
+ expect(polipus.storage.exists?(init_page)).to be_truthy
36
+ expect(polipus.storage.get(init_page).links.count).to be polipus.storage.count
38
37
  end
39
38
 
40
39
  it 'should filter unwanted urls' do
41
40
  polipus.skip_links_like(/\/pages\//)
42
41
  polipus.takeover
43
- polipus.storage.get(init_page).links
44
- .reject { |e| e.path.to_s =~ /\/pages\// }.count.should be polipus.storage.count
42
+ expect(polipus.storage.get(init_page).links
43
+ .reject { |e| e.path.to_s =~ /\/pages\// }.count).to be polipus.storage.count
45
44
  end
46
45
 
47
46
  it 'should follow only wanted urls' do
48
47
  polipus.follow_links_like(/\/pages\//)
49
48
  polipus.follow_links_like(/\/gems$/)
50
49
  polipus.takeover
51
- polipus.storage.get(init_page).links
50
+ expect(polipus.storage.get(init_page).links
52
51
  .reject { |e| ![/\/pages\//, /\/gems$/].any? { |p| e.path =~ p } }
53
- .count.should be polipus.storage.count
52
+ .count).to be polipus.storage.count
54
53
  end
55
54
 
56
55
  it 'should refresh expired pages' do
@@ -60,9 +59,9 @@ describe Polipus::PolipusCrawler do
60
59
  page.fetched_at = page.fetched_at - 3600
61
60
  polipus.storage.add(page)
62
61
  end
63
- polipus.storage.each { |_id, page| page.expired?(3600).should be_true }
62
+ polipus.storage.each { |_id, page| expect(page.expired?(3600)).to be_truthy }
64
63
  polipus.takeover
65
- polipus.storage.each { |_id, page| page.expired?(3600).should be_false }
64
+ polipus.storage.each { |_id, page| expect(page.expired?(3600)).to be_falsey }
66
65
  end
67
66
 
68
67
  it 'should re-download seeder urls no matter what' do
@@ -74,7 +73,7 @@ describe Polipus::PolipusCrawler do
74
73
  end
75
74
  polipus.takeover
76
75
  polipus.takeover
77
- cache_hit['http://rubygems.org/gems'].should be 2
76
+ expect(cache_hit['http://rubygems.org/gems']).to be 2
78
77
  end
79
78
 
80
79
  it 'should call on_page_error code blocks when a page has error' do
@@ -82,8 +81,8 @@ describe Polipus::PolipusCrawler do
82
81
  a_page = nil
83
82
  p.on_page_error { |page| a_page = page }
84
83
  p.takeover
85
- a_page.should_not be_nil
86
- a_page.error.should_not be_nil
84
+ expect(a_page).not_to be_nil
85
+ expect(a_page.error).not_to be_nil
87
86
  end
88
87
 
89
88
  it 'should obey to the robots.txt file' do
@@ -92,8 +91,7 @@ describe Polipus::PolipusCrawler do
92
91
  polipus = Polipus::PolipusCrawler.new('polipus-rspec', ['https://rubygems.org/gems/polipus'], lopt)
93
92
  polipus.depth_limit = 1
94
93
  polipus.takeover
95
- polipus.storage.each { |_id, page| (page.url.path =~ /$\/downloads\//).should be_false }
94
+ polipus.storage.each { |_id, page| expect(page.url.path =~ /$\/downloads\//).to be_falsey }
96
95
  end
97
-
98
96
  end
99
97
  end