polipus 0.3.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +8 -8
  2. data/.rubocop.yml +17 -0
  3. data/.rubocop_todo.yml +37 -0
  4. data/.travis.yml +2 -1
  5. data/CHANGELOG.md +20 -0
  6. data/README.md +10 -0
  7. data/Rakefile +4 -4
  8. data/examples/basic.rb +16 -19
  9. data/examples/incremental.rb +17 -17
  10. data/examples/robots_txt_handling.rb +1 -1
  11. data/examples/survival.rb +3 -3
  12. data/lib/polipus.rb +186 -229
  13. data/lib/polipus/http.rb +41 -42
  14. data/lib/polipus/page.rb +33 -34
  15. data/lib/polipus/plugin.rb +2 -2
  16. data/lib/polipus/plugins/cleaner.rb +7 -8
  17. data/lib/polipus/plugins/sample.rb +6 -9
  18. data/lib/polipus/plugins/sleeper.rb +7 -8
  19. data/lib/polipus/queue_overflow.rb +11 -11
  20. data/lib/polipus/queue_overflow/base.rb +1 -1
  21. data/lib/polipus/queue_overflow/dev_null_queue.rb +9 -9
  22. data/lib/polipus/queue_overflow/manager.rb +28 -25
  23. data/lib/polipus/queue_overflow/mongo_queue.rb +24 -26
  24. data/lib/polipus/queue_overflow/mongo_queue_capped.rb +12 -12
  25. data/lib/polipus/robotex.rb +41 -51
  26. data/lib/polipus/signal_handler.rb +41 -0
  27. data/lib/polipus/storage.rb +11 -11
  28. data/lib/polipus/storage/base.rb +10 -8
  29. data/lib/polipus/storage/dev_null.rb +6 -7
  30. data/lib/polipus/storage/memory_store.rb +21 -22
  31. data/lib/polipus/storage/mongo_store.rb +34 -38
  32. data/lib/polipus/storage/s3_store.rb +33 -38
  33. data/lib/polipus/url_tracker.rb +3 -3
  34. data/lib/polipus/url_tracker/bloomfilter.rb +4 -5
  35. data/lib/polipus/url_tracker/redis_set.rb +3 -4
  36. data/lib/polipus/version.rb +3 -3
  37. data/polipus.gemspec +12 -13
  38. data/spec/clear.rb +3 -3
  39. data/spec/http_spec.rb +27 -28
  40. data/spec/page_spec.rb +16 -16
  41. data/spec/polipus_spec.rb +34 -31
  42. data/spec/queue_overflow_manager_spec.rb +30 -28
  43. data/spec/queue_overflow_spec.rb +15 -15
  44. data/spec/robotex_spec.rb +9 -10
  45. data/spec/signal_handler_spec.rb +18 -0
  46. data/spec/spec_helper.rb +7 -6
  47. data/spec/storage_memory_spec.rb +18 -18
  48. data/spec/storage_mongo_spec.rb +19 -19
  49. data/spec/storage_s3_spec.rb +30 -31
  50. data/spec/url_tracker_spec.rb +7 -7
  51. metadata +7 -2
data/spec/url_tracker_spec.rb CHANGED
@@ -1,5 +1,5 @@
1
- require "spec_helper"
2
- require "polipus/url_tracker"
1
+ require 'spec_helper'
2
+ require 'polipus/url_tracker'
3
3
 
4
4
  describe Polipus::UrlTracker do
5
5
  before(:all) do
@@ -13,16 +13,16 @@ describe Polipus::UrlTracker do
13
13
  end
14
14
 
15
15
  it 'should work (bf)' do
16
- url = "http://www.asd.com/asd/lol"
16
+ url = 'http://www.asd.com/asd/lol'
17
17
  @bf.visit url
18
18
  @bf.visited?(url).should be_true
19
- @bf.visited?("http://www.google.com").should be_false
19
+ @bf.visited?('http://www.google.com').should be_false
20
20
  end
21
21
 
22
22
  it 'should work (redis_set)' do
23
- url = "http://www.asd.com/asd/lol"
23
+ url = 'http://www.asd.com/asd/lol'
24
24
  @set.visit url
25
25
  @set.visited?(url).should be_true
26
- @set.visited?("http://www.google.com").should be_false
26
+ @set.visited?('http://www.google.com').should be_false
27
27
  end
28
- end
28
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: polipus
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.0
4
+ version: 0.3.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Francesco Laurita
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-06-02 00:00:00.000000000 Z
11
+ date: 2014-06-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: redis-bloomfilter
@@ -315,6 +315,8 @@ files:
315
315
  - .document
316
316
  - .gitignore
317
317
  - .rspec
318
+ - .rubocop.yml
319
+ - .rubocop_todo.yml
318
320
  - .travis.yml
319
321
  - AUTHORS.md
320
322
  - CHANGELOG.md
@@ -341,6 +343,7 @@ files:
341
343
  - lib/polipus/queue_overflow/mongo_queue.rb
342
344
  - lib/polipus/queue_overflow/mongo_queue_capped.rb
343
345
  - lib/polipus/robotex.rb
346
+ - lib/polipus/signal_handler.rb
344
347
  - lib/polipus/storage.rb
345
348
  - lib/polipus/storage/base.rb
346
349
  - lib/polipus/storage/dev_null.rb
@@ -381,6 +384,7 @@ files:
381
384
  - spec/queue_overflow_manager_spec.rb
382
385
  - spec/queue_overflow_spec.rb
383
386
  - spec/robotex_spec.rb
387
+ - spec/signal_handler_spec.rb
384
388
  - spec/spec_helper.rb
385
389
  - spec/storage_memory_spec.rb
386
390
  - spec/storage_mongo_spec.rb
@@ -440,6 +444,7 @@ test_files:
440
444
  - spec/queue_overflow_manager_spec.rb
441
445
  - spec/queue_overflow_spec.rb
442
446
  - spec/robotex_spec.rb
447
+ - spec/signal_handler_spec.rb
443
448
  - spec/spec_helper.rb
444
449
  - spec/storage_memory_spec.rb
445
450
  - spec/storage_mongo_spec.rb