sutch-anemone 0.7.2

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    YWExNTI1ZGRiMjA0NTZhNzk3Y2YyMDYzNDcxMDQwNTk4NDJkMGI2NQ==
+  data.tar.gz: !binary |-
+    YWE2MDg1OTQzNTEyOTZiZTJjMTUzNWZiMzgxNTBjMDJmMzBkYjYzZQ==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    YWY1MjliMGJjMzRhZGQ1OTVmYWVlYzI3YmU3YTI1ZGVjMDk5NjAwZGEwZmJh
+    MzA0Y2Q4ZWMwODM3MzgyMTU5ZTk2NTE3ZDFhNDc4MDBmOWZjNWViOWQ1Nzlk
+    ZDU4OTA4Y2VkNWI1MDA1ZjUyNWQ2YzJkMDA3YmJiNGQwZTczMGM=
+  data.tar.gz: !binary |-
+    MjYzODE4ZTIwYjM5OTljMDY2NTdkNzRiY2FlOWJkOGMxNmZjNzE4M2JkN2Fk
+    YjQ4MjE5NjM2NjllZmJhODc4M2UzYjYwYTZhY2ZhZWRiYzgwZjk4MDZmYzEy
+    N2FkMjZiZjdiMmI2NWI2N2I3MDUyNWM2YmI0YTIyODAzZmQ1Yzg=

data/CHANGELOG.rdoc

@@ -0,0 +1,136 @@
+* Enhancements
+
+== sutch's branch
+
+  * Added Anemone::Resource to provide for spidering of resources other than HTML pages
+
+== 0.7.2 / 2012-05-30
+
+* Bug fixes
+
+  * Fix bug causing anchor links to have '#' converted to '%23'
+
+== 0.7.1 / 2012-01-20
+
+* Minor enhancements
+
+  * Switch from robots gem (which people reported problems with) to new robotex gem
+
+* Bug fixes
+
+  * Fix incorrect default file extension for KyotoCabinet
+
+== 0.7.0 / 2012-01-19
+
+* Major enhancements
+
+  * Added support for SQLite3 and Kyoto Cabinet storage
+
+* Minor enhancements
+
+  * Added Page#base to use base HTML element
+  * Use bundler for development dependencies
+
+* Bug fixes
+
+  * Encode characters in URLs
+  * Fix specs to run under rake
+  * Fix handling of redirect_to in storage adapters
+
+== 0.6.1 / 2011-02-24
+
+* Bug fixes
+
+  * Fix a bug preventing SSL connections from working
+
+== 0.6.0 / 2011-02-17
+
+* Major enhancements
+
+  * Added support for HTTP Basic Auth with URLs containing a username and password
+  * Added support for anonymous HTTP proxies
+
+* Minor enhancements
+
+  * Added read_timeout option to set the HTTP request timeout in seconds
+
+* Bug fixes
+
+  * Don't fatal error if a page request times out
+  * Fix double encoding of links containing %20
+
+== 0.5.0 / 2010-09-01
+
+* Major enhancements
+
+  * Added page storage engines for MongoDB and Redis
+
+* Minor enhancements
+
+  * Use xpath for link parsing instead of CSS (faster) (Marc Seeger)
+  * Added skip_query_strings option to skip links with query strings (Joost Baaij)
+
+* Bug fixes
+
+  * Only consider status code 300..307 a redirect (Marc Seeger)
+  * Canonicalize redirect links (Marc Seeger)
+
+== 0.4.0 / 2010-04-08
+
+* Major enchancements
+
+  * Cookies can be accepted and sent with each HTTP request.
+
+== 0.3.2 / 2010-02-04
+
+* Bug fixes
+
+  * Fixed issue that allowed following redirects off the original domain
+
+== 0.3.1 / 2010-01-22
+
+* Minor enhancements
+
+  * Added an attr_accessor to Page for the HTTP response body
+
+* Bug fixes
+
+  * Fixed incorrect method calls in CLI scripts
+
+== 0.3.0 / 2009-12-15
+
+* Major enchancements
+
+  * Option for persistent storage of pages during crawl with TokyoCabinet or PStore
+
+* Minor enhancements
+
+  * Options can be set via methods on the Core object in the crawl block
+
+== 0.2.3 / 2009-11-01
+
+* Minor enhancements
+
+  * Options are now applied per-crawl, rather than module-wide.
+
+* Bug fixes
+
+  * Fixed a bug which caused deadlock if an exception occurred when crawling the last page in the queue.
+
+== 0.2.2 / 2009-10-26
+
+* Minor enhancements
+
+  * When the :verbose option is set to true, exception backtraces are printed to aid debugging.
+
+== 0.2.1 / 2009-10-24
+
+* Major enhancements
+
+  * Added HTTPS support.
+  * CLI program 'anemone', which is a frontend for several tasks.
+
+* Minor enhancements
+
+  * HTTP request response time recorded in Page.
+  * Use of persistent HTTP connections.

data/LICENSE.txt

@@ -0,0 +1,19 @@
+Copyright (c) 2009 Vertive, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,38 @@
+= Anemone
+
+Anemone is a web spider framework that can spider a domain and collect useful
+information about the pages it visits. It is versatile, allowing you to
+write your own specialized spider tasks quickly and easily.
+
+See http://anemone.rubyforge.org for more information.
+
+== Features
+* Multi-threaded design for high performance
+* Tracks 301 HTTP redirects
+* Built-in BFS algorithm for determining page depth
+* Allows exclusion of URLs based on regular expressions
+* Choose the links to follow on each page with focus_crawl()
+* HTTPS support
+* Records response time for each page
+* CLI program can list all pages in a domain, calculate page depths, and more
+* Obey robots.txt
+* In-memory or persistent storage of pages during crawl, using TokyoCabinet, SQLite3, MongoDB, or Redis
+
+== Examples
+See the scripts under the <tt>lib/anemone/cli</tt> directory for examples of several useful Anemone tasks.
+
+== Requirements
+* nokogiri
+* robots
+
+== Development
+To test and develop this gem, additional requirements are:
+* rspec
+* fakeweb
+* tokyocabinet
+* kyotocabinet-ruby
+* mongo
+* redis
+* sqlite3
+
+You will need to have KyotoCabinet, {Tokyo Cabinet}[http://fallabs.com/tokyocabinet/], {MongoDB}[http://www.mongodb.org/], and {Redis}[http://code.google.com/p/redis/] installed on your system and running.

data/Rakefile

@@ -0,0 +1,23 @@
+require 'rspec/core/rake_task'
+require 'rdoc/task'
+
+desc "Run all specs"
+RSpec::Core::RakeTask.new(:rspec) do |spec|
+  spec.pattern = 'spec/**/*_spec.rb'
+end
+
+RSpec::Core::RakeTask.new(:rcov) do |spec|
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :default => :rspec
+
+RDoc::Task.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "anemone #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.7.2

data/bin/anemone

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require 'anemone/cli'
+
+Anemone::CLI::run

data/lib/anemone.rb

@@ -0,0 +1,2 @@
+require 'rubygems'
+require 'anemone/core'

data/lib/anemone/cli.rb

@@ -0,0 +1,24 @@
+module Anemone
+  module CLI
+    COMMANDS = %w[count cron pagedepth serialize url-list]
+    
+    def self.run
+      command = ARGV.shift
+      
+      if COMMANDS.include? command
+        load "anemone/cli/#{command.tr('-', '_')}.rb"
+      else
+        puts <<-INFO
+Anemone is a web spider framework that can collect
+useful information about pages it visits.
+
+Usage:
+  anemone <command> [arguments]
+
+Commands:
+  #{COMMANDS.join(', ')}
+INFO
+      end
+    end
+  end
+end

data/lib/anemone/cli/count.rb

@@ -0,0 +1,22 @@
+require 'anemone'
+
+begin
+  # make sure that the first option is a URL we can crawl
+  url = URI(ARGV[0])
+rescue
+  puts <<-INFO
+Usage:
+  anemone count <url>
+
+Synopsis:
+  Crawls a site starting at the given URL and outputs the total number
+  of unique pages on the site.
+INFO
+  exit(0)
+end
+
+Anemone.crawl(url) do |anemone|
+  anemone.after_crawl do |pages|
+    puts pages.uniq!.size
+  end
+end

data/lib/anemone/cli/cron.rb

@@ -0,0 +1,90 @@
+require 'anemone'
+require 'optparse'
+require 'ostruct'
+
+options = OpenStruct.new
+options.relative = false
+options.output_file = 'urls.txt'
+
+begin
+  # make sure that the last argument is a URL we can crawl
+  root = URI(ARGV.last)
+rescue
+  puts <<-INFO
+Usage:
+  anemone cron [options] <url>
+
+Synopsis:
+  Combination of `count`, `pagedepth` and `url-list` commands.
+  Performs pagedepth, url list, and count functionality.
+  Outputs results to STDOUT and link list to file (urls.txt).
+  Meant to be run daily as a cron job.
+
+Options:
+  -r, --relative           Output relative URLs (rather than absolute)
+  -o, --output filename    Filename to save URL list to. Defautls to urls.txt.
+INFO
+  exit(0)
+end
+
+# parse command-line options
+opts = OptionParser.new
+opts.on('-r', '--relative')        { options.relative = true }
+opts.on('-o', '--output filename') {|o| options.output_file = o }
+opts.parse!(ARGV)
+
+Anemone.crawl(root, {:discard_page_bodies => true}) do |anemone|  
+  
+  anemone.after_crawl do |pages|
+    puts "Crawl results for #{root}\n"
+    
+    # print a list of 404's
+    not_found = []
+    pages.each_value do |page|
+      url = page.url.to_s
+      not_found << url if page.not_found?
+    end
+    unless not_found.empty?
+      puts "\n404's:"
+
+      missing_links = pages.urls_linking_to(not_found)
+      missing_links.each do |url, links|
+        if options.relative
+          puts URI(url).path.to_s
+        else
+          puts url
+        end
+        links.slice(0..10).each do |u|
+          u = u.path if options.relative
+          puts "  linked from #{u}"
+        end
+        
+        puts " ..." if links.size > 10
+      end
+
+      print "\n"
+    end  
+    
+    # remove redirect aliases, and calculate pagedepths
+    pages = pages.shortest_paths!(root).uniq
+    depths = pages.values.inject({}) do |depths, page|
+      depths[page.depth] ||= 0
+      depths[page.depth] += 1
+      depths
+    end
+    
+    # print the page count
+    puts "Total pages: #{pages.size}\n"
+    
+    # print a list of depths
+    depths.sort.each { |depth, count| puts "Depth: #{depth} Count: #{count}" }
+    
+    # output a list of urls to file
+    file = open(options.output_file, 'w')
+    pages.each_key do |url|
+      url = options.relative ? url.path.to_s : url.to_s
+      file.puts url
+    end
+  end
+  
+end

data/lib/anemone/cli/pagedepth.rb

@@ -0,0 +1,32 @@
+require 'anemone'
+
+begin
+  # make sure that the first option is a URL we can crawl
+  root = URI(ARGV[0])
+rescue
+  puts <<-INFO
+Usage:
+  anemone pagedepth <url>
+
+Synopsis:
+  Crawls a site starting at the given URL and outputs a count of
+  the number of pages at each depth of the crawl.
+INFO
+  exit(0)
+end
+
+Anemone.crawl(root) do |anemone|
+  anemone.skip_links_like %r{^/c/$}, %r{^/stores/$}
+
+  anemone.after_crawl do |pages|
+    pages = pages.shortest_paths!(root).uniq!
+
+    depths = pages.values.inject({}) do |depths, page|
+      depths[page.depth] ||= 0
+      depths[page.depth] += 1
+      depths
+    end
+
+    depths.sort.each { |depth, count| puts "Depth: #{depth} Count: #{count}" }
+  end
+end

data/lib/anemone/cli/serialize.rb

@@ -0,0 +1,35 @@
+require 'anemone'
+require 'optparse'
+require 'ostruct'
+
+begin
+  # make sure that the first option is a URL we can crawl
+  root = URI(ARGV[0])
+rescue
+  puts <<-INFO
+Usage:
+  anemone serialize [options] <url>
+
+Synopsis:
+  Crawls a site starting at the given URL and saves the resulting
+  PageStore object to a file using Marshal serialization.
+
+Options:
+  -o, --output filename      Filename to save PageStore to. Defaults to crawl.{Time.now}
+INFO
+  exit(0)
+end
+
+options = OpenStruct.new
+options.output_file = "crawl.#{Time.now.to_i}"
+
+# parse command-line options
+opts = OptionParser.new
+opts.on('-o', '--output filename') {|o| options.output_file = o }
+opts.parse!(ARGV)
+
+Anemone.crawl(root) do |anemone|
+  anemone.after_crawl do |pages|
+    open(options.output_file, 'w') {|f| Marshal.dump(pages, f)}
+  end
+end