spider 0.5.1 → 0.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ae84ef471855de9c49c3499eb658d961342a4338
-  data.tar.gz: cf80d52423709af79478ff051f212dea0b8021a2
+  metadata.gz: 1303a4df64456b14c15c76ef68959829f83458f4
+  data.tar.gz: a178f2d2d961b175eded73c24c96eea5ad763502
 SHA512:
-  metadata.gz: 72785824697410005b8738a32e74d71a78d4623522e7f3f4bb56318349c12d693cee8a8a003766c33afec0b70bb195ce079117413543d9a004fe7adfae03b9d8
-  data.tar.gz: 8bdd202aa793c3f39984e3394e915c915c2fd7d68b7f475c0d317103d31c4a61bdc3ceaa87cfea02a77ec2cdbf2e23cc245a4e080c211c03322ce729b5551bb8
+  metadata.gz: 6c556fbf8dc472250d9740b216983a6844263388b8fa18956858052606003685c2f84da8ac2815621295db086d78331181a353676600785df5bfdec7a39dae40
+  data.tar.gz: 9680ebca3b44c55c8f952e7f2903526f0267013967a2d38eab524970f1b702817515470cd48a04ba3ad3fbb46ccaf92ff47e911df4c62e1589ef5482e28a8b33

data/AUTHORS

@@ -4,9 +4,11 @@ the following kind souls:
 Brian Campbell
 Henri Cook
 James Edward Gray II
+Jeremy Evans
 Joao Eriberto Mota Filho
 John Buckley
 John Nagro
 Matt Horan
+Marc (@brigriffin)
 Mike Burns (original author)
 Sander van der Vliet

data/README.md

@@ -72,6 +72,16 @@ scraping, collecting, and looping so that you can just handle the data._
  end
 ```
 
+### Use Redis to track cycles
+
+```ruby
+ require 'spider'
+ require 'spider/included_in_redis'
+ Spider.start_at('http://cashcats.biz/') do |s|
+   s.check_already_seen_with IncludedInRedis.new(host: '127.0.0.1', port: 6379)
+ end
+```
+
 ### Track cycles with a custom object
 
 ```ruby

data/lib/spider.rb

@@ -4,7 +4,7 @@ require File.dirname(__FILE__)+'/spider/spider_instance'
 # links, and doing it all over again.
 class Spider
 
-  VERSION_INFO = [0, 5, 1] unless defined?(self::VERSION_INFO)
+  VERSION_INFO = [0, 5, 2] unless defined?(self::VERSION_INFO)
   VERSION = VERSION_INFO.map(&:to_s).join('.') unless defined?(self::VERSION)
 
   def self.version

data/lib/spider/included_in_redis.rb

@@ -0,0 +1,31 @@
+# Use Redis to track cycles.
+
+require 'redis'
+require 'json'
+
+# A specialized class using Redis to track items stored. It supports
+# three operations: new, <<, and include? . Together these can be used to
+# add items to Redis, then determine whether the item has been added.
+#
+# To use it with Spider use the check_already_seen_with method:
+#
+#  Spider.start_at('http://example.com/') do |s|
+#    s.check_already_seen_with IncludedInRedis.new(host: '127.0.0.1', port: 6379)
+#  end
+class IncludedInRedis
+  # Construct a new IncludedInRedis instance. All arguments here are
+  # passed to Redis (part of the redis gem).
+  def initialize(*a)
+    @c = Redis.new(*a)
+  end
+
+  # Add an item to Redis
+  def <<(v)
+    @c.set(v.to_s, v.to_json)
+  end
+
+  # True if the item is in Redis
+  def include?(v)
+    @c.get(v.to_s) == v.to_json
+  end
+end

data/lib/spider/spider_instance.rb

@@ -123,7 +123,7 @@ class SpiderInstance
   def on(code, p = nil, &block)
     f = p ? p : block
     case code
-    when Fixnum
+    when Integer
       @callbacks[code] = f
     else
       @callbacks[code.to_sym] = f

data/spec/spider/included_in_redis_spec.rb

@@ -0,0 +1,43 @@
+require File.dirname(__FILE__)+'/../spec_helper'
+
+def before_specing_redis
+  local_require 'spider/included_in_redis'
+  system('redis-server 127.0.0.1:6379')
+end
+
+def after_specing_redis
+  system('kill -KILL `pidof redis-server`')
+end
+
+Spec::Runner.configure { |c| c.mock_with :mocha }
+
+describe 'Object to halt cycles' do
+  before do
+    before_specing_redis
+  end
+
+  it 'should understand <<' do
+    c = IncludedInRedis.new(host: 'localhost', port: 6379)
+    c.should respond_to(:<<)
+  end
+
+  it 'should understand included?' do
+    c = IncludedInRedis.new(host: 'localhost', port: 6379)
+    c.should respond_to(:include?)
+  end
+
+  it 'should produce false if the object is not included' do
+    c = IncludedInRedis.new(host: 'localhost', port: 6379)
+    c.include?('a').should be_false
+  end
+
+  it 'should produce true if the object is included' do
+    c = IncludedInRedis.new(host: 'localhost', port: 6379)
+    c << 'a'
+    c.include?('a').should be_true
+  end
+  
+  after do
+    after_specing_redis
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spider
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.5.2
 platform: ruby
 authors:
 - John Nagro
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-04 00:00:00.000000000 Z
+date: 2018-04-23 00:00:00.000000000 Z
 dependencies: []
 description: |
   A Web spidering library: handles robots.txt, scraping, finding more
@@ -24,11 +24,13 @@ files:
 - README.md
 - lib/spider.rb
 - lib/spider/included_in_memcached.rb
+- lib/spider/included_in_redis.rb
 - lib/spider/next_urls_in_sqs.rb
 - lib/spider/robot_rules.rb
 - lib/spider/spider_instance.rb
 - spec/spec_helper.rb
 - spec/spider/included_in_memcached_spec.rb
+- spec/spider/included_in_redis_spec.rb
 - spec/spider/spider_instance_spec.rb
 - spec/spider_spec.rb
 - spider.gemspec
@@ -52,7 +54,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: spider
-rubygems_version: 2.6.6
+rubygems_version: 2.5.2.1
 signing_key: 
 specification_version: 4
 summary: A Web spidering library