crawler 0.2 → 0.2.1

data/.gitignore

@@ -0,0 +1,4 @@
+/doc/*
+/log/*
+/pkg/*
+*.gem

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2010 Tyler M Cunnion
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,33 @@
+= Crawler
+
+Crawler consists of two classes, Crawler::Webcrawler and Crawler::Observer. All actual crawling is contained in Webcrawler, so Observer can be replaced with any class that implements the update method (See the Observable module for more). 
+
+=== Webcrawler
+
+The Webcrawler class can be instantiated with a hash of options:
+* timeout -- Time limit for the crawl operation, after which a Timeout::Error exception is raised.
+* external -- Boolean; whether or not the crawler will go outside the original URI's host.
+* exclude -- A URI will be excluded if it includes any of the strings within this array.
+
+=== Observer
+
+The Observer class may be instantiated with an object for logging, as long as it responds to the puts method. It is applied by passing an instance of it to the add_observer method of a Crawler object. Any class may be substitute (or supplement) which implements the update method with two arguments, an instance of HTTPResponse and an instance of URI.
+
+== Installation
+
+gem install crawler
+
+It's just that easy!
+
+== Executable
+
+The gem comes with an executable version you may find useful:
+
+crawler url [-txl]
+	-t --timeout, Timeout limit in seconds
+	-x --exclude, Comma separated list of strings which will be passed to the exclude option of Crawler
+	-l --log, A filename for a log file, defaults to STDOUT
+
+== License
+
+This gem is licensed under the MIT/X11 license. A copy is included in the file LICENSE.

data/Rakefile

@@ -1,10 +1,51 @@
 require 'rake'
-require 'echoe'
+require 'rubygems'
+require 'jeweler'
 
-Echoe.new('crawler', '0.2') do |g|
-  g.description = "Simple webcrawler"
-  g.url = "http://github.com/tylercunnion/crawler"
+Jeweler::Tasks.new do |g|
+  g.name = "crawler"
+  g.summary = "Simple webcrawler"
+  g.description = "BFS webcrawler that implements Observable"
+  g.homepage = "http://github.com/tylercunnion/crawler"
   g.author = "Tyler Cunnion"
-  g.email = "tyler.cunnion+ruby@gmail.com"
-  g.ignore_pattern = ["tmp/*", "features/*", "log/*"]
+  g.email = "tyler.cunnion@gmail.com"
+  g.files.exclude 'features/**/*'
+  g.test_files.exclude 'features/**/*'
+  g.add_dependency('nokogiri')
+end
+
+Jeweler::GemcutterTasks.new
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/test_*.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+task :test => :check_dependencies
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "crawler #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
 end

data/VERSION

@@ -0,0 +1 @@
+0.2.1

data/crawler.gemspec

@@ -1,32 +1,74 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
 # -*- encoding: utf-8 -*-
 
 Gem::Specification.new do |s|
   s.name = %q{crawler}
-  s.version = "0.2"
+  s.version = "0.2.1"
 
-  s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Tyler Cunnion"]
   s.date = %q{2010-01-25}
   s.default_executable = %q{crawler}
-  s.description = %q{Simple webcrawler}
-  s.email = %q{tyler.cunnion+ruby@gmail.com}
+  s.description = %q{BFS webcrawler that implements Observable}
+  s.email = %q{tyler.cunnion@gmail.com}
   s.executables = ["crawler"]
-  s.extra_rdoc_files = ["bin/crawler", "lib/crawler.rb", "lib/crawler/observer.rb", "lib/crawler/webcrawler.rb"]
-  s.files = ["Rakefile", "bin/crawler", "features/step_definitions/crawler_steps.rb", "features/support/env.rb", "lib/crawler.rb", "lib/crawler/observer.rb", "lib/crawler/webcrawler.rb", "spec/crawler/crawler_spec.rb", "spec/crawler/observer_spec.rb", "spec/fixtures/excluded/shouldnt-hit.html", "spec/fixtures/exclusion.html", "spec/fixtures/external.html", "spec/fixtures/index.html", "spec/fixtures/messed-up.html", "spec/fixtures/non-html.html", "spec/fixtures/non-http.html", "spec/fixtures/page2.html", "spec/fixtures/page3.html", "spec/fixtures/page4.html", "spec/fixtures/page5.html", "spec/fixtures/pdf.pdf", "spec/fixtures/self-reference.html", "spec/spec_helper.rb", "Manifest", "crawler.gemspec"]
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.rdoc"
+  ]
+  s.files = [
+    ".gitignore",
+     "LICENSE",
+     "Manifest",
+     "README.rdoc",
+     "Rakefile",
+     "VERSION",
+     "bin/crawler",
+     "crawler.gemspec",
+     "lib/crawler.rb",
+     "lib/crawler/observer.rb",
+     "lib/crawler/webcrawler.rb",
+     "spec/crawler/crawler_spec.rb",
+     "spec/crawler/observer_spec.rb",
+     "spec/fixtures/excluded/shouldnt-hit.html",
+     "spec/fixtures/exclusion.html",
+     "spec/fixtures/external.html",
+     "spec/fixtures/index.html",
+     "spec/fixtures/messed-up.html",
+     "spec/fixtures/non-html.html",
+     "spec/fixtures/non-http.html",
+     "spec/fixtures/page2.html",
+     "spec/fixtures/page3.html",
+     "spec/fixtures/page4.html",
+     "spec/fixtures/page5.html",
+     "spec/fixtures/pdf.pdf",
+     "spec/fixtures/self-reference.html",
+     "spec/spec_helper.rb"
+  ]
   s.homepage = %q{http://github.com/tylercunnion/crawler}
-  s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Crawler"]
+  s.rdoc_options = ["--charset=UTF-8"]
   s.require_paths = ["lib"]
-  s.rubyforge_project = %q{crawler}
   s.rubygems_version = %q{1.3.5}
   s.summary = %q{Simple webcrawler}
+  s.test_files = [
+    "spec/crawler/crawler_spec.rb",
+     "spec/crawler/observer_spec.rb",
+     "spec/spec_helper.rb"
+  ]
 
   if s.respond_to? :specification_version then
     current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
     s.specification_version = 3
 
     if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<nokogiri>, [">= 0"])
     else
+      s.add_dependency(%q<nokogiri>, [">= 0"])
     end
   else
+    s.add_dependency(%q<nokogiri>, [">= 0"])
   end
 end
+

data/lib/crawler/webcrawler.rb

@@ -18,6 +18,8 @@ module Crawler
     
     # Accepts the following options:
     # * timeout -- Time limit for the crawl operation, after which a Timeout::Error exception is raised.
+    # * external -- Boolean; whether or not the crawler will go outside the original URI's host.
+    # * exclude -- A URI will be excluded if it includes any of the strings within this array.
     def initialize(options={})
       @crawled = Set.new
       @queue = []
@@ -30,6 +32,7 @@ module Crawler
     end
     
     # Given a URI object, the crawler will explore every linked page recursively using the Breadth First Search algorithm.
+    # Whenever it downloads a page, it notifies observers with an HTTPResponse subclass object and the downloaded URI object.
     def crawl(start_uri)
       start_uri = start_uri.normalize
       @queue << start_uri

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: crawler
 version: !ruby/object:Gem::Version 
-  version: "0.2"
+  version: 0.2.1
 platform: ruby
 authors: 
 - Tyler Cunnion
@@ -10,25 +10,36 @@ bindir: bin
 cert_chain: []
 
 date: 2010-01-25 00:00:00 -05:00
-default_executable: 
-dependencies: []
-
-description: Simple webcrawler
-email: tyler.cunnion+ruby@gmail.com
+default_executable: crawler
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: nokogiri
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: BFS webcrawler that implements Observable
+email: tyler.cunnion@gmail.com
 executables: 
 - crawler
 extensions: []
 
 extra_rdoc_files: 
-- bin/crawler
-- lib/crawler.rb
-- lib/crawler/observer.rb
-- lib/crawler/webcrawler.rb
+- LICENSE
+- README.rdoc
 files: 
+- .gitignore
+- LICENSE
+- Manifest
+- README.rdoc
 - Rakefile
+- VERSION
 - bin/crawler
-- features/step_definitions/crawler_steps.rb
-- features/support/env.rb
+- crawler.gemspec
 - lib/crawler.rb
 - lib/crawler/observer.rb
 - lib/crawler/webcrawler.rb
@@ -48,18 +59,13 @@ files:
 - spec/fixtures/pdf.pdf
 - spec/fixtures/self-reference.html
 - spec/spec_helper.rb
-- Manifest
-- crawler.gemspec
 has_rdoc: true
 homepage: http://github.com/tylercunnion/crawler
 licenses: []
 
 post_install_message: 
 rdoc_options: 
-- --line-numbers
-- --inline-source
-- --title
-- Crawler
+- --charset=UTF-8
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
@@ -72,14 +78,16 @@ required_rubygems_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      version: "1.2"
+      version: "0"
   version: 
 requirements: []
 
-rubyforge_project: crawler
+rubyforge_project: 
 rubygems_version: 1.3.5
 signing_key: 
 specification_version: 3
 summary: Simple webcrawler
-test_files: []
-
+test_files: 
+- spec/crawler/crawler_spec.rb
+- spec/crawler/observer_spec.rb
+- spec/spec_helper.rb

data/features/step_definitions/crawler_steps.rb

@@ -1,20 +0,0 @@
-Given /^the crawl has not begun$/ do
-end
-
-When /^I start a crawl with the URI "([^\"]*)"$/ do |arg1|
-  @obs = Crawler::Observer.new
-  @crawler = Crawler::Webcrawler.new
-  
-  @crawler.add_observer(@obs)
-  
-  @uri = URI.parse(arg1)
-  @crawler.crawl(@uri)
-end
-
-Then /^the page should be downloaded$/ do
-  @crawler.crawled.should include(@uri)
-end
-
-Then /^the observer should be updated$/ do
-  @obs.should_receive(:update)
-end

data/features/support/env.rb

@@ -1,2 +0,0 @@
-$LOAD_PATH << File.join(File.dirname(__FILE__), "..", "..", "lib")
-require 'crawler'