polipus 0.0.1

data/lib/polipus/url_tracker.rb

@@ -0,0 +1,20 @@
+module Polipus
+  module UrlTracker
+    def self.bloomfilter(options = {})
+      require "polipus/url_tracker/bloomfilter"
+      options[:size]       ||= 1_000_000
+      options[:error_rate] ||= 0.01
+      options[:key_name]   ||= 'polipus-bloomfilter'
+      options[:redis]      ||= Redis.current
+      options[:driver]     ||= 'lua'
+      self::Bloomfilter.new options
+    end
+
+    def self.redis_set(options = {})
+      require "polipus/url_tracker/redis_set"
+      options[:redis]      ||= Redis.current
+      options[:key_name]   ||= 'polipus-set'
+      self::RedisSet.new options
+    end
+  end
+end

data/lib/polipus/url_tracker/bloomfilter.rb

@@ -0,0 +1,27 @@
+require "redis-bloomfilter"
+module Polipus
+  module UrlTracker
+    class Bloomfilter
+      def initialize(options = {})
+        @bf = Redis::Bloomfilter.new options
+      end
+
+      def visited?(url)
+        @bf.include?(url)
+      end
+
+      def visit url
+        @bf.insert url
+      end
+
+      def remove url
+        @bf.remove url
+      end
+
+      def clear
+        @bf.clear
+      end
+
+    end
+  end
+end

data/lib/polipus/url_tracker/redis_set.rb

@@ -0,0 +1,27 @@
+module Polipus
+  module UrlTracker
+    class RedisSet
+      
+      def initialize(options = {})
+        @redis    = options[:redis] || Redis.current
+        @set_name = options[:key_name]
+      end
+
+      def visited?(url)
+        @redis.sismember(@set_name,url)
+      end
+
+      def visit url
+        @redis.sadd(@set_name, url)
+      end
+
+      def remove
+        @redis.srem(@set_name, url, 0)
+      end
+
+      def clear
+        @redis.del @set_name
+      end
+    end
+  end
+end

data/lib/polipus/version.rb

@@ -0,0 +1,4 @@
+module Polipus
+  VERSION = "0.0.1"
+  HOMEPAGE = "https://github.com/taganaka/polipus"
+end

data/polipus.gemspec

@@ -0,0 +1,39 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "polipus/version"
+
+Gem::Specification.new do |s|
+  s.name        = "polipus"
+  s.version     = Polipus::VERSION
+  s.authors     = ["Francesco Laurita"]
+  s.email       = ["francesco.laurita@gmail.com"]
+  s.homepage    = "https://github.com/taganaka/polipus"
+  s.summary     = %q{Polipus distributed web-crawler framework}
+  s.description = %q{
+    An easy to use distributed web-crawler framework based on Redis
+  }
+
+  s.rubyforge_project = "polipus"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency "redis-bloomfilter", "~> 0.0.1"
+  s.add_dependency "redis-queue",       "~> 0.0.3"
+  s.add_dependency "nokogiri",          "~> 1.6.0"
+  s.add_dependency "hiredis",           "~> 0.4.5"
+  s.add_dependency "redis",             "~> 3.0.4"
+  s.add_dependency "mongo",             "~> 1.8.6"
+  s.add_dependency "bson_ext",          "~> 1.8.6"
+  s.add_dependency "json",              "~> 1.8.0"
+  s.add_dependency "aws-s3",            "~> 0.6.3"
+  s.add_dependency "http-cookie",       "~> 1.0.1"
+  
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "vcr", "~> 2.5.0"
+  s.add_development_dependency "webmock"
+  s.add_development_dependency "flexmock", "~> 1.3.2"
+
+end

data/spec/cassettes/08b228db424a926e1ed6ab63b38d847e.yml

@@ -0,0 +1,166 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: http://s3.amazonaws.com/com.polipus.pages._test_pages
+    body:
+      encoding: US-ASCII
+      string: ''
+  response:
+    status:
+      code: 200
+      message: OK
+    body:
+      encoding: US-ASCII
+      string: ! '<?xml version="1.0" encoding="UTF-8"?>
+
+        <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Name>com.polipus.pages._test_pages</Name><Prefix></Prefix><Marker></Marker><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated></ListBucketResult>'
+    http_version: 
+  recorded_at: Thu, 18 Jul 2013 11:04:53 GMT
+- request:
+    method: put
+    uri: http://s3.amazonaws.com/com.polipus.pages._test_pages/8b86345aba1588c6b0b0c1729e077c7b
+    body:
+      encoding: ASCII-8BIT
+      string: !binary |-
+        eJxVTUtugzAQvcusaTBpAtRJ21Uj9QyhQv5MZFSDkRkLRSh37xTURVfvO28W
+        SNGDBEc0yjyf53mnJrszoX9nfPXBQwYOlcU4ca1JQohDo5eVlJ/N6hSFCQPh
+        QE90H3GL5AYfl+u/qhCbPG3iwus62DtPnx31/u2cr8C274Zvfnn9ysAEiyD3
+        QmRgcSQHklnEG0aMfAm/wnYRDbUU/oxpDMOELXU93w7J+wxuSMahBUkxYQZp
+        wthaRQrk8mCZOo6g1nX5fDgqrYpjXZtSCy1MUe1fUFSVqTQ8fgD3C19t
+  response:
+    status:
+      code: 200
+      message: OK
+    body:
+      encoding: US-ASCII
+      string: ''
+    http_version: 
+  recorded_at: Thu, 18 Jul 2013 11:04:53 GMT
+- request:
+    method: head
+    uri: http://s3.amazonaws.com/com.polipus.pages._test_pages/8b86345aba1588c6b0b0c1729e077c7b
+    body:
+      encoding: US-ASCII
+      string: ''
+  response:
+    status:
+      code: 200
+      message: OK
+    body:
+      encoding: US-ASCII
+      string: ''
+    http_version: 
+  recorded_at: Thu, 18 Jul 2013 11:04:54 GMT
+- request:
+    method: head
+    uri: http://s3.amazonaws.com/com.polipus.pages._test_pages/8b86345aba1588c6b0b0c1729e077c7b
+    body:
+      encoding: US-ASCII
+      string: ''
+  response:
+    status:
+      code: 200
+      message: OK
+    body:
+      encoding: US-ASCII
+      string: ''
+    http_version: 
+  recorded_at: Thu, 18 Jul 2013 11:04:54 GMT
+- request:
+    method: delete
+    uri: http://s3.amazonaws.com/com.polipus.pages._test_pages/8b86345aba1588c6b0b0c1729e077c7b
+    body:
+      encoding: US-ASCII
+      string: ''
+  response:
+    status:
+      code: 204
+      message: No Content
+    body:
+      encoding: US-ASCII
+      string: ''
+    http_version: 
+  recorded_at: Thu, 18 Jul 2013 11:04:54 GMT
+- request:
+    method: get
+    uri: http://s3.amazonaws.com/com.polipus.pages._test_pages
+    body:
+      encoding: US-ASCII
+      string: ''
+  response:
+    status:
+      code: 200
+      message: OK
+    body:
+      encoding: US-ASCII
+      string: ! '<?xml version="1.0" encoding="UTF-8"?>
+
+        <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Name>com.polipus.pages._test_pages</Name><Prefix></Prefix><Marker></Marker><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated></ListBucketResult>'
+    http_version: 
+  recorded_at: Thu, 18 Jul 2013 11:04:54 GMT
+- request:
+    method: get
+    uri: http://s3.amazonaws.com/com.polipus.pages._test_pages
+    body:
+      encoding: US-ASCII
+      string: ''
+  response:
+    status:
+      code: 200
+      message: OK
+    body:
+      encoding: US-ASCII
+      string: ! '<?xml version="1.0" encoding="UTF-8"?>
+
+        <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Name>com.polipus.pages._test_pages</Name><Prefix></Prefix><Marker></Marker><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated></ListBucketResult>'
+    http_version: 
+  recorded_at: Thu, 18 Jul 2013 11:04:55 GMT
+- request:
+    method: delete
+    uri: http://s3.amazonaws.com/com.polipus.pages._test_pages
+    body:
+      encoding: US-ASCII
+      string: ''
+  response:
+    status:
+      code: 204
+      message: No Content
+    body:
+      encoding: US-ASCII
+      string: ''
+    http_version: 
+  recorded_at: Thu, 18 Jul 2013 11:04:55 GMT
+- request:
+    method: put
+    uri: http://s3.amazonaws.com/com.polipus.pages._test_pages
+    body:
+      encoding: US-ASCII
+      string: ''
+  response:
+    status:
+      code: 200
+      message: OK
+    body:
+      encoding: US-ASCII
+      string: ''
+    http_version: 
+  recorded_at: Thu, 18 Jul 2013 11:04:55 GMT
+- request:
+    method: get
+    uri: http://s3.amazonaws.com/com.polipus.pages._test_pages
+    body:
+      encoding: US-ASCII
+      string: ''
+  response:
+    status:
+      code: 200
+      message: OK
+    body:
+      encoding: US-ASCII
+      string: ! '<?xml version="1.0" encoding="UTF-8"?>
+
+        <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Name>com.polipus.pages._test_pages</Name><Prefix></Prefix><Marker></Marker><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated></ListBucketResult>'
+    http_version: 
+  recorded_at: Thu, 18 Jul 2013 11:04:56 GMT
+recorded_with: VCR 2.5.0

data/spec/cassettes/20aa41f181b49f00078c3ca30bad5afe.yml

@@ -0,0 +1,166 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: http://s3.amazonaws.com/com.polipus.pages._test_pages
+    body:
+      encoding: US-ASCII
+      string: ''
+  response:
+    status:
+      code: 200
+      message: OK
+    body:
+      encoding: US-ASCII
+      string: ! '<?xml version="1.0" encoding="UTF-8"?>
+
+        <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Name>com.polipus.pages._test_pages</Name><Prefix></Prefix><Marker></Marker><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated></ListBucketResult>'
+    http_version: 
+  recorded_at: Thu, 18 Jul 2013 11:04:38 GMT
+- request:
+    method: put
+    uri: http://s3.amazonaws.com/com.polipus.pages._test_pages/ef727a04499928b997ca7f26fcedd1b4
+    body:
+      encoding: ASCII-8BIT
+      string: !binary |-
+        eJxVTc1qhDAQfpc5p2sUWWu6bU9d6DOsRWIyojQaSSbIIvvunSo99PT9zjcb
+        pOBAwUC0qCxb1/Wkoz0ZP2XvTF6ddyBgQG0xRO41SUpZNt22k/Nnszt5bvxM
+        ONMT3Rc8InXAx/X2ryrlIV8OceX1zts7T18GmtzbJduBbTfO3/zy9iXAeIug
+        CikFWFxoAMUsYI8BA1/Cr7BjQEMt+T8jLn6O2NI48e2cnBPQI5kBLSgKCQWk
+        iKG1mjSo7cEyjRwB9lVRaVmWdV0Xz11dV0ZXfXHuDVqbdyU8fgA35WAW
+  response:
+    status:
+      code: 200
+      message: OK
+    body:
+      encoding: US-ASCII
+      string: ''
+    http_version: 
+  recorded_at: Thu, 18 Jul 2013 11:04:38 GMT
+- request:
+    method: head
+    uri: http://s3.amazonaws.com/com.polipus.pages._test_pages/ef727a04499928b997ca7f26fcedd1b4
+    body:
+      encoding: US-ASCII
+      string: ''
+  response:
+    status:
+      code: 200
+      message: OK
+    body:
+      encoding: US-ASCII
+      string: ''
+    http_version: 
+  recorded_at: Thu, 18 Jul 2013 11:04:39 GMT
+- request:
+    method: head
+    uri: http://s3.amazonaws.com/com.polipus.pages._test_pages/ef727a04499928b997ca7f26fcedd1b4
+    body:
+      encoding: US-ASCII
+      string: ''
+  response:
+    status:
+      code: 200
+      message: OK
+    body:
+      encoding: US-ASCII
+      string: ''
+    http_version: 
+  recorded_at: Thu, 18 Jul 2013 11:04:39 GMT
+- request:
+    method: delete
+    uri: http://s3.amazonaws.com/com.polipus.pages._test_pages/ef727a04499928b997ca7f26fcedd1b4
+    body:
+      encoding: US-ASCII
+      string: ''
+  response:
+    status:
+      code: 204
+      message: No Content
+    body:
+      encoding: US-ASCII
+      string: ''
+    http_version: 
+  recorded_at: Thu, 18 Jul 2013 11:04:39 GMT
+- request:
+    method: get
+    uri: http://s3.amazonaws.com/com.polipus.pages._test_pages
+    body:
+      encoding: US-ASCII
+      string: ''
+  response:
+    status:
+      code: 200
+      message: OK
+    body:
+      encoding: US-ASCII
+      string: ! '<?xml version="1.0" encoding="UTF-8"?>
+
+        <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Name>com.polipus.pages._test_pages</Name><Prefix></Prefix><Marker></Marker><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated></ListBucketResult>'
+    http_version: 
+  recorded_at: Thu, 18 Jul 2013 11:04:39 GMT
+- request:
+    method: get
+    uri: http://s3.amazonaws.com/com.polipus.pages._test_pages
+    body:
+      encoding: US-ASCII
+      string: ''
+  response:
+    status:
+      code: 200
+      message: OK
+    body:
+      encoding: US-ASCII
+      string: ! '<?xml version="1.0" encoding="UTF-8"?>
+
+        <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Name>com.polipus.pages._test_pages</Name><Prefix></Prefix><Marker></Marker><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated></ListBucketResult>'
+    http_version: 
+  recorded_at: Thu, 18 Jul 2013 11:04:40 GMT
+- request:
+    method: delete
+    uri: http://s3.amazonaws.com/com.polipus.pages._test_pages
+    body:
+      encoding: US-ASCII
+      string: ''
+  response:
+    status:
+      code: 204
+      message: No Content
+    body:
+      encoding: US-ASCII
+      string: ''
+    http_version: 
+  recorded_at: Thu, 18 Jul 2013 11:04:40 GMT
+- request:
+    method: put
+    uri: http://s3.amazonaws.com/com.polipus.pages._test_pages
+    body:
+      encoding: US-ASCII
+      string: ''
+  response:
+    status:
+      code: 200
+      message: OK
+    body:
+      encoding: US-ASCII
+      string: ''
+    http_version: 
+  recorded_at: Thu, 18 Jul 2013 11:04:40 GMT
+- request:
+    method: get
+    uri: http://s3.amazonaws.com/com.polipus.pages._test_pages
+    body:
+      encoding: US-ASCII
+      string: ''
+  response:
+    status:
+      code: 200
+      message: OK
+    body:
+      encoding: US-ASCII
+      string: ! '<?xml version="1.0" encoding="UTF-8"?>
+
+        <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Name>com.polipus.pages._test_pages</Name><Prefix></Prefix><Marker></Marker><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated></ListBucketResult>'
+    http_version: 
+  recorded_at: Thu, 18 Jul 2013 11:04:41 GMT
+recorded_with: VCR 2.5.0