polipus 0.3.7 → 0.4.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (45) hide show
  1. checksums.yaml +8 -8
  2. data/.rspec +1 -1
  3. data/.rubocop.yml +3 -3
  4. data/.rubocop_todo.yml +1 -1
  5. data/.travis.yml +14 -4
  6. data/AUTHORS.md +1 -0
  7. data/CHANGELOG.md +9 -1
  8. data/Gemfile +9 -0
  9. data/README.md +2 -3
  10. data/Rakefile +1 -3
  11. data/examples/basic.rb +8 -1
  12. data/lib/polipus.rb +25 -13
  13. data/lib/polipus/queue_overflow.rb +1 -0
  14. data/lib/polipus/queue_overflow/manager.rb +1 -0
  15. data/lib/polipus/queue_overflow/mongo_queue.rb +1 -1
  16. data/lib/polipus/queue_overflow/worker.rb +24 -0
  17. data/lib/polipus/storage.rb +10 -16
  18. data/lib/polipus/storage/mongo_store.rb +6 -1
  19. data/lib/polipus/storage/rethink_store.rb +90 -0
  20. data/lib/polipus/version.rb +1 -1
  21. data/polipus.gemspec +16 -18
  22. data/spec/{http_spec.rb → polipus/http_spec.rb} +26 -37
  23. data/spec/{page_spec.rb → polipus/page_spec.rb} +7 -11
  24. data/spec/{queue_overflow_manager_spec.rb → polipus/queue_overflow/manager_spec.rb} +22 -29
  25. data/spec/{queue_overflow_spec.rb → polipus/queue_overflow_spec.rb} +14 -20
  26. data/spec/{robotex_spec.rb → polipus/robotex_spec.rb} +10 -11
  27. data/spec/{signal_handler_spec.rb → polipus/signal_handler_spec.rb} +2 -6
  28. data/spec/{storage_memory_spec.rb → polipus/storage/memory_store_spec.rb} +18 -21
  29. data/spec/{storage_mongo_spec.rb → polipus/storage/mongo_store_spec.rb} +23 -25
  30. data/spec/polipus/storage/rethink_store_spec.rb +117 -0
  31. data/spec/{url_tracker_spec.rb → polipus/url_tracker_spec.rb} +4 -4
  32. data/spec/polipus_spec.rb +13 -15
  33. data/spec/spec_helper.rb +13 -12
  34. metadata +76 -154
  35. data/lib/polipus/storage/s3_store.rb +0 -96
  36. data/spec/cassettes/08b228db424a926e1ed6ab63b38d847e.yml +0 -166
  37. data/spec/cassettes/20aa41f181b49f00078c3ca30bad5afe.yml +0 -166
  38. data/spec/cassettes/4640919145753505af2d0f8423de37f3.yml +0 -270
  39. data/spec/cassettes/66aae15a03f4aab8efd15e40d2d7882a.yml +0 -194
  40. data/spec/cassettes/76b7c197c95a5bf9b1e882c567192d72.yml +0 -183
  41. data/spec/cassettes/9b1d523b7f5db7214f8a8bd9272cccba.yml +0 -221
  42. data/spec/cassettes/ab333f89535a2efb284913fede6aa7c7.yml +0 -221
  43. data/spec/cassettes/ae5d7cffde3f53122cdf79f3d1367e8e.yml +0 -221
  44. data/spec/cassettes/ffe3d588b6df4b9de35e5a7ccaf5a81b.yml +0 -695
  45. data/spec/storage_s3_spec.rb +0 -115
data/spec/storage_s3_spec.rb DELETED
@@ -1,115 +0,0 @@
1
- # encoding: UTF-8
2
- require 'spec_helper'
3
- require 'aws/s3'
4
- require 'polipus/storage/s3_store'
5
- describe Polipus::Storage::S3Store do
6
-
7
- before(:each) do
8
- @storage = Polipus::Storage.s3_store(
9
- '_test_pages',
10
-
11
- access_key_id: 'XXXXXXX',
12
- secret_access_key: 'XXXX'
13
-
14
- )
15
- end
16
-
17
- after(:each) { @storage.clear }
18
-
19
- it 'should store a page' do
20
-
21
- p = page_factory 'http://www.google.com', code: 200, body: '<html></html>'
22
- uuid = @storage.add p
23
- uuid.should be == 'ed646a3334ca891fd3467db131372140'
24
- @storage.count.should be 1
25
- p = @storage.get p
26
- p.url.to_s.should be == 'http://www.google.com'
27
- p.body.should be == '<html></html>'
28
- @storage.remove p
29
-
30
- end
31
-
32
- it 'should update a page' do
33
- p = page_factory 'http://www.google.com', code: 301, body: '<html></html>'
34
- @storage.add p
35
- p = @storage.get p
36
- p.code.should be == 301
37
- @storage.count.should be == 1
38
- @storage.remove p
39
- end
40
-
41
- it 'should iterate over stored pages' do
42
- 10.times { |i| @storage.add page_factory("http://www.google.com/p_#{i}", code: 200, body: "<html>#{i}</html>") }
43
- @storage.count.should be 10
44
- @storage.each do |k, _page|
45
- k.should be =~ /[a-f0-9]{32}/
46
- end
47
- end
48
-
49
- it 'should delete a page' do
50
- p = page_factory 'http://www.google.com', code: 301, body: '<html></html>'
51
- @storage.add p
52
- @storage.remove p
53
- @storage.get(p).should be_nil
54
- @storage.count.should be 0
55
- end
56
-
57
- it 'should store a page removing a query string from the uuid generation' do
58
- p = page_factory 'http://www.asd.com/?asd=lol', code: 200, body: '<html></html>'
59
- p_no_query = page_factory 'http://www.asd.com/?asdas=dasda&adsda=1', code: 200, body: '<html></html>'
60
- @storage.include_query_string_in_uuid = false
61
- @storage.add p
62
- @storage.exists?(p_no_query).should be_true
63
- @storage.remove p
64
- end
65
-
66
- it 'should store a page removing a query string from the uuid generation no ending slash' do
67
- p = page_factory 'http://www.asd.com?asd=lol', code: 200, body: '<html></html>'
68
- p_no_query = page_factory 'http://www.asd.com', code: 200, body: '<html></html>'
69
- @storage.include_query_string_in_uuid = false
70
- @storage.add p
71
- @storage.exists?(p_no_query).should be_true
72
- @storage.remove p
73
- end
74
-
75
- it 'should store a page with user data associated' do
76
- p = page_factory 'http://www.user.com', code: 200, body: '<html></html>'
77
- p.user_data.name = 'Test User Data'
78
- @storage.add p
79
- @storage.exists?(p).should be_true
80
- p = @storage.get(p)
81
- p.user_data.name.should be == 'Test User Data'
82
- @storage.remove p
83
- end
84
-
85
- it 'should honor the except parameters' do
86
- storage = Polipus::Storage.s3_store(
87
- '_test_pages',
88
- {
89
- access_key_id: 'XXXXXXX',
90
- secret_access_key: 'XXXX'
91
- },
92
- ['body']
93
- )
94
- p = page_factory 'http://www.user-doo.com', code: 200, body: '<html></html>'
95
- storage.add p
96
- p = storage.get p
97
-
98
- p.body.should be_nil
99
- storage.clear
100
- end
101
-
102
- it 'should return false if a doc not exists' do
103
- @storage.include_query_string_in_uuid = false
104
- p_other = page_factory 'http://www.asdrrrr.com', code: 200, body: '<html></html>'
105
- @storage.exists?(p_other).should be_false
106
- @storage.add p_other
107
- @storage.exists?(p_other).should be_true
108
- p_other = page_factory 'http://www.asdrrrr.com?trk=asd-lol', code: 200, body: '<html></html>'
109
- @storage.exists?(p_other).should be_true
110
- @storage.include_query_string_in_uuid = true
111
- @storage.exists?(p_other).should be_false
112
- @storage.remove p_other
113
- end
114
-
115
- end