spider 0.5.3 → 0.5.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 60310268052e7e2cfe120fdd313e314b28124c5a
-  data.tar.gz: 8747d828232c89f674e765e005df722c170a7839
+SHA256:
+  metadata.gz: b87ed979c115546fa802f888fea0baf322f458be6e50e40f8ea8fd9ee392c8ac
+  data.tar.gz: eabe506949614a5622afa2def1da954d352d737dccadd0622d060be13c061115
 SHA512:
-  metadata.gz: a58cc23eb4d464b41957ca39ac17a3d2c91d86ac71140163b2072bedc317e3b529430fd47630980c0e87668df509031078a6d0a7d63d62878d5a31300b34bc51
-  data.tar.gz: a2391c2b8e64dd4dd5f895e9530312bd515d00981c85d857ff8261d28bdaab2a1669e1d9fc9e15fa43f62f4eba99666465cbc0bdd6378e6610daa8e043e5f156
+  metadata.gz: ab52efe227f19067dd52efb0333d5901d3725f4e88fa0b86942c12bd702efa6b3bf4ed72b03d5ce58b32a22c574b0ee31764fa5e37df07a57132c250bf6b0658
+  data.tar.gz: ede92b88eb09867c41c1f7fcca58c99a55e63f02bb54fa094ece40e8875240734aa9abdcaf043db003120a44b239198dc6d063779543927febb493116732b7f3

data/AUTHORS

@@ -1,6 +1,7 @@
 The Ruby Spider Gem would not be what it is today without the help of
 the following kind souls:
 
+Alexandre Rousseau
 Brian Campbell
 Henri Cook
 James Edward Gray II
@@ -11,4 +12,6 @@ John Nagro
 Matt Horan
 Marc (@brigriffin)
 Mike Burns (original author)
+Olle Jonsson
 Sander van der Vliet
+Stuart Yamartino

data/README.md

@@ -82,6 +82,16 @@ scraping, collecting, and looping so that you can just handle the data._
  end
 ```
 
+### Use Plain text to track cycles
+
+```ruby
+ require 'spider'
+ require 'spider/included_in_redis'
+ Spider.start_at('http://cashcats.biz/') do |s|
+   s.check_already_seen_with IncludedInFile.new('/tmp/cashcats_crawl.txt')
+ end
+```
+
 ### Track cycles with a custom object
 
 ```ruby
@@ -116,7 +126,7 @@ scraping, collecting, and looping so that you can just handle the data._
  require 'spider'
  class MyArray < Array
    def pop
-	super
+     super
    end
 
    def push(a_msg)
@@ -150,7 +160,7 @@ scraping, collecting, and looping so that you can just handle the data._
  require 'net/http_configuration'
  require 'spider'
  http_conf = Net::HTTP::Configuration.new(:proxy_host => '7proxies.org',
-                                          :proxy_port => 8881)  
+                                          :proxy_port => 8881)
  http_conf.apply do
    Spider.start_at('http://img.4chan.org/b/') do |s|
      s.on(:success) do |a_url, resp, prior_url|

data/lib/spider.rb

@@ -4,7 +4,7 @@ require File.dirname(__FILE__)+'/spider/spider_instance'
 # links, and doing it all over again.
 class Spider
 
-  VERSION_INFO = [0, 5, 3] unless defined?(self::VERSION_INFO)
+  VERSION_INFO = [0, 5, 4] unless defined?(self::VERSION_INFO)
   VERSION = VERSION_INFO.map(&:to_s).join('.') unless defined?(self::VERSION)
 
   def self.version

data/lib/spider/included_in_file.rb

@@ -0,0 +1,32 @@
+# Use plain text file to track cycles.
+
+# A specialized class using a plain text to track items stored. It supports
+# three operations: new, <<, and include? . Together these can be used to
+# add items to the text file, then determine whether the item has been added.
+#
+# To use it with Spider use the check_already_seen_with method:
+#
+#  Spider.start_at('http://example.com/') do |s|
+#    s.check_already_seen_with IncludedInFile.new('/tmp/crawled.log')
+#  end
+class IncludedInFile
+  # Construct a new IncludedInFile instance.
+  # @param filepath [String] as path of file to store crawled URL
+  def initialize(filepath)
+    @filepath = filepath
+    # create file if not exists
+    File.write(@filepath, '') unless File.file?(@filepath)
+    @urls = File.readlines(@filepath).map(&:chomp)
+  end
+
+  # Add an item to the file & array of URL.
+  def <<(v)
+    @urls << v.to_s
+    File.write(@filepath, "#{v}\r\n", File.size(@filepath), mode: 'a')
+  end
+
+  # True if the item is in the file.
+  def include?(v)
+    @urls.include? v.to_s
+  end
+end

data/spider.gemspec

@@ -9,7 +9,6 @@ spec = Gem::Specification.new do |s|
   s.has_rdoc = true
   s.homepage = 'https://github.com/johnnagro/spider'
   s.name = 'spider'
-  s.rubyforge_project = 'spider'
   s.summary = 'A Web spidering library'
   s.files = Dir['**/*'].delete_if { |f| f =~ /(cvs|gem|svn)$/i }
   s.require_path = 'lib'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spider
 version: !ruby/object:Gem::Version
-  version: 0.5.3
+  version: 0.5.4
 platform: ruby
 authors:
 - John Nagro
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-04-23 00:00:00.000000000 Z
+date: 2020-06-04 00:00:00.000000000 Z
 dependencies: []
 description: |
   A Web spidering library: handles robots.txt, scraping, finding more
@@ -23,6 +23,7 @@ files:
 - LICENSE
 - README.md
 - lib/spider.rb
+- lib/spider/included_in_file.rb
 - lib/spider/included_in_memcached.rb
 - lib/spider/included_in_redis.rb
 - lib/spider/next_urls_in_sqs.rb
@@ -53,8 +54,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: spider
-rubygems_version: 2.5.2.1
+rubyforge_project: 
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: A Web spidering library