polipus 0.3.2 → 0.3.3

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NmQ5ZTBhZThlMDNlZGEzNzQ4ZDU0MGQ3NzBkODY0YjZlOGZmZTFmOA==
+    MGZhM2Q1OWE5ZTkxZWE3Y2JhMDJkNzZkMWRjMWZjYjI4YmRhODJmYw==
   data.tar.gz: !binary |-
-    NTc3NTI5YmMyYzk4MzNkNDFjZTY4N2JiNGE2MzI4ZTRiZDZmZjY4Yw==
+    ODg4NTk5ZjhjNDM0ZjBhN2M2OWJlOWQyOTE4OWFiZmY1ZmQxMTZiYQ==
 SHA512:
   metadata.gz: !binary |-
-    ZjczYmUzNjdhOWY0NjYzMzNhZDE4NmMwMWI5YTVhZTg1NGY5ZGJjNWJkN2I5
-    NDYwZjRiNDM1ZGZlYjFmZTAxNWNiNmQ0MGM5MzJmNDAzZTAxYWUyNTc3OTlh
-    ZWFhMWZkZGI4MTlhZmQ3NmM3NmQwZTk0YmM1YWU4NDJkYzk4NGQ=
+    NTdmYWZhMmMzYTQxOWY5OTBlNGE1ZWQzNWYwMThjMjAwNzQ4NmU4MDY4OWRi
+    OWVkNzEwNmYwNTY3OGI4NmFkZGJiNzNhN2I3M2ZjMDUxNjAzMDAyZjEyYmZj
+    ZGFkODliNDc5ZTYzYWE2ODNiMzdkMWFjZGExYjgzNTQyNmM4MDc=
   data.tar.gz: !binary |-
-    ZmI4YzFjMzNiNTYwNWFkZmNiY2VmZTcyY2I1YTI3ZWQ2NWI2MjYwYTJiNDY3
-    YjhmZjE3MGJkNWUwNmZkMWNhMTExZTJhYmE4YTI4OThlNmRkYWZhODkxZTY1
-    MWZiZjA0M2E2MjcwOGZiNDViZmUyNTg4YmFkNmVjM2Y4ZjIzZDU=
+    NGYzNDc2ODQ3OTBiMzE5N2M5YmM3ZDliN2IyYWRmNjBiZDI3ZTYyYjkxNjBk
+    MDFhZjA0NDE5YjFkYzdiZTQxMjg0ZGEwODA3Nzk0YWZiNDRmY2I1MmE2ZWI4
+    MjQxNTU4NjgwOTM1ZDdlYjM4Mjg0NjhjY2M2OGU5MDA1YjNjMjY=

data/.rubocop_todo.yml

@@ -17,10 +17,6 @@ Style/CyclomaticComplexity:
 Style/Documentation:
   Enabled: false
 
-# Offense count: 38
-# Configuration parameters: EnforcedStyle, SupportedStyles.
-Style/Encoding:
-  Enabled: false
 
 # Offense count: 2
 # Configuration parameters: EnforcedStyle, SupportedStyles.

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 0.3.3 (2015-06-26)
+
+[Compare changes in gem](https://github.com/taganaka/polipus/compare/0.3.2...0.3.3)
+
+* BugFix: Better compatibility for mongo 2.6.x on index creation
+
 ## 0.3.2 (2015-06-17)
 
 [Compare changes in gem](https://github.com/taganaka/polipus/compare/0.3.1...0.3.2)

data/Rakefile

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 require 'bundler/gem_tasks'
 require 'rspec/core/rake_task'
 

data/examples/basic.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 require 'polipus'
 require 'mongo'
 require 'polipus/plugins/cleaner'

data/examples/error_handling.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 require 'polipus'
 
 Polipus.crawler('rubygems', 'http://rubygems.org/') do |crawler|

data/examples/incremental.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 require 'polipus'
 require 'mongo'
 

data/examples/robots_txt_handling.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 require 'polipus'
 
 options = {

data/examples/survival.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 require 'polipus'
 
 Polipus.crawler('rubygems', 'http://rubygems.org/') do |crawler|

data/lib/polipus/http.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 require 'net/https'
 require 'polipus/page'
 require 'zlib'

data/lib/polipus/page.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 require 'nokogiri'
 require 'json'
 require 'ostruct'

data/lib/polipus/plugin.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 module Polipus
   module Plugin
     @@plugins = {}

data/lib/polipus/plugins/cleaner.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 module Polipus
   module Plugin
     class Cleaner
@@ -8,7 +9,7 @@ module Polipus
       def on_initialize(crawler)
         crawler.logger.info { 'Cleaner plugin loaded' }
         unless @reset
-          crawler.logger.info { 'Cleaner plugin is disable, add :reset => true to the plugin if you really know what you are doing' }
+          crawler.logger.info { 'Cleaner plugin is disabled, add :reset => true to the plugin if you really know what you are doing' }
           return nil
         end
         crawler.logger.info { 'Cleaning all: url_tracker, storage, queue' }

data/lib/polipus/plugins/sample.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 module Polipus
   module Plugin
     class Sample

data/lib/polipus/plugins/sleeper.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 module Polipus
   module Plugin
     class Sleeper

data/lib/polipus/queue_overflow.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 require 'polipus/queue_overflow/manager'
 module Polipus
   module QueueOverflow

data/lib/polipus/queue_overflow/base.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 module Polipus
   module QueueOverflow
     class Base

data/lib/polipus/queue_overflow/dev_null_queue.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 require 'thread'
 module Polipus
   module QueueOverflow

data/lib/polipus/queue_overflow/manager.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 module Polipus
   module QueueOverflow
     class Manager

data/lib/polipus/queue_overflow/mongo_queue.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 require 'thread'
 require 'mongo'
 module Polipus

data/lib/polipus/queue_overflow/mongo_queue_capped.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 require 'polipus/queue_overflow/mongo_queue'
 module Polipus
   module QueueOverflow

data/lib/polipus/robotex.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 require 'open-uri'
 require 'uri'
 require 'timeout'

data/lib/polipus/signal_handler.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 require 'singleton'
 module Polipus
   class SignalHandler

data/lib/polipus/storage.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 require 'polipus/storage/base'
 module Polipus
   module Storage

data/lib/polipus/storage/base.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 require 'uri'
 
 module Polipus

data/lib/polipus/storage/dev_null.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 module Polipus
   module Storage
     class DevNull < Base

data/lib/polipus/storage/memory_store.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 require 'thread'
 module Polipus
   module Storage

data/lib/polipus/storage/mongo_store.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 require 'mongo'
 require 'zlib'
 require 'thread'
@@ -9,7 +10,7 @@ module Polipus
         @mongo      = options[:mongo]
         @collection = options[:collection]
         @mongo.create_collection(@collection)
-        @mongo[@collection].ensure_index(:uuid, unique: true, drop_dups: true, background: true)
+        @mongo[@collection].ensure_index(:uuid, unique: true, dropDups: true, background: true)
         @compress_body = options[:compress_body] ||= true
         @except = options[:except] ||= []
         @semaphore = Mutex.new

data/lib/polipus/storage/s3_store.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 require 'aws/s3'
 require 'zlib'
 require 'thread'

data/lib/polipus/url_tracker.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 module Polipus
   module UrlTracker
     def self.bloomfilter(options = {})

data/lib/polipus/url_tracker/bloomfilter.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 require 'redis-bloomfilter'
 module Polipus
   module UrlTracker

data/lib/polipus/url_tracker/redis_set.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 module Polipus
   module UrlTracker
     class RedisSet

data/lib/polipus/version.rb

@@ -1,4 +1,5 @@
+# encoding: UTF-8
 module Polipus
-  VERSION = '0.3.2'
+  VERSION = '0.3.3'
   HOMEPAGE = 'https://github.com/taganaka/polipus'
 end

data/spec/clear.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 require 'yaml'
 Dir.glob('./cassettes/*.yml').each do|f|
   next unless f =~ /[a-f0-9]{32}/

data/spec/http_spec.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 require 'spec_helper'
 require 'mongo'
 require 'polipus/http'

data/spec/page_spec.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 require 'spec_helper'
 require 'polipus/page'
 

data/spec/polipus_spec.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 require 'spec_helper'
 
 describe Polipus::PolipusCrawler do

data/spec/queue_overflow_manager_spec.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 require 'spec_helper'
 require 'mongo'
 require 'polipus/queue_overflow'

data/spec/queue_overflow_spec.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 require 'spec_helper'
 require 'polipus/queue_overflow'
 

data/spec/robotex_spec.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 require 'spec_helper'
 require 'polipus/robotex'
 describe Polipus::Robotex do

data/spec/signal_handler_spec.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 require 'spec_helper'
 
 describe Polipus::SignalHandler do

data/spec/spec_helper.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 # This file was generated by the `rspec --init` command. Conventionally, all
 # specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
 # Require this file using `require "spec_helper"` to ensure that it is only

data/spec/storage_memory_spec.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 require 'spec_helper'
 require 'mongo'
 require 'polipus/storage/memory_store'

data/spec/storage_mongo_spec.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 require 'spec_helper'
 require 'mongo'
 require 'polipus/storage/mongo_store'

data/spec/storage_s3_spec.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 require 'spec_helper'
 require 'aws/s3'
 require 'polipus/storage/s3_store'

data/spec/url_tracker_spec.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 require 'spec_helper'
 require 'polipus/url_tracker'
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: polipus
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.3.3
 platform: ruby
 authors:
 - Francesco Laurita
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-18 00:00:00.000000000 Z
+date: 2014-06-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis-bloomfilter