crawler 0.2 → 0.2.1

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,4 @@
1
+ /doc/*
2
+ /log/*
3
+ /pkg/*
4
+ *.gem
data/LICENSE ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2010 Tyler M Cunnion
2
+
3
+ Permission is hereby granted, free of charge, to any person
4
+ obtaining a copy of this software and associated documentation
5
+ files (the "Software"), to deal in the Software without
6
+ restriction, including without limitation the rights to use,
7
+ copy, modify, merge, publish, distribute, sublicense, and/or sell
8
+ copies of the Software, and to permit persons to whom the
9
+ Software is furnished to do so, subject to the following
10
+ conditions:
11
+
12
+ The above copyright notice and this permission notice shall be
13
+ included in all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
17
+ OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
19
+ HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
20
+ WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21
+ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22
+ OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,33 @@
1
+ = Crawler
2
+
3
+ Crawler consists of two classes, Crawler::Webcrawler and Crawler::Observer. All actual crawling is contained in Webcrawler, so Observer can be replaced with any class that implements the update method (See the Observable module for more).
4
+
5
+ === Webcrawler
6
+
7
+ The Webcrawler class can be instantiated with a hash of options:
8
+ * timeout -- Time limit for the crawl operation, after which a Timeout::Error exception is raised.
9
+ * external -- Boolean; whether or not the crawler will go outside the original URI's host.
10
+ * exclude -- A URI will be excluded if it includes any of the strings within this array.
11
+
12
+ === Observer
13
+
14
+ The Observer class may be instantiated with an object for logging, as long as it responds to the puts method. It is applied by passing an instance of it to the add_observer method of a Crawler object. Any class may be substitute (or supplement) which implements the update method with two arguments, an instance of HTTPResponse and an instance of URI.
15
+
16
+ == Installation
17
+
18
+ gem install crawler
19
+
20
+ It's just that easy!
21
+
22
+ == Executable
23
+
24
+ The gem comes with an executable version you may find useful:
25
+
26
+ crawler url [-txl]
27
+ -t --timeout, Timeout limit in seconds
28
+ -x --exclude, Comma separated list of strings which will be passed to the exclude option of Crawler
29
+ -l --log, A filename for a log file, defaults to STDOUT
30
+
31
+ == License
32
+
33
+ This gem is licensed under the MIT/X11 license. A copy is included in the file LICENSE.
data/Rakefile CHANGED
@@ -1,10 +1,51 @@
1
1
  require 'rake'
2
- require 'echoe'
2
+ require 'rubygems'
3
+ require 'jeweler'
3
4
 
4
- Echoe.new('crawler', '0.2') do |g|
5
- g.description = "Simple webcrawler"
6
- g.url = "http://github.com/tylercunnion/crawler"
5
+ Jeweler::Tasks.new do |g|
6
+ g.name = "crawler"
7
+ g.summary = "Simple webcrawler"
8
+ g.description = "BFS webcrawler that implements Observable"
9
+ g.homepage = "http://github.com/tylercunnion/crawler"
7
10
  g.author = "Tyler Cunnion"
8
- g.email = "tyler.cunnion+ruby@gmail.com"
9
- g.ignore_pattern = ["tmp/*", "features/*", "log/*"]
11
+ g.email = "tyler.cunnion@gmail.com"
12
+ g.files.exclude 'features/**/*'
13
+ g.test_files.exclude 'features/**/*'
14
+ g.add_dependency('nokogiri')
15
+ end
16
+
17
+ Jeweler::GemcutterTasks.new
18
+
19
+ require 'rake/testtask'
20
+ Rake::TestTask.new(:test) do |test|
21
+ test.libs << 'lib' << 'test'
22
+ test.pattern = 'test/**/test_*.rb'
23
+ test.verbose = true
24
+ end
25
+
26
+ begin
27
+ require 'rcov/rcovtask'
28
+ Rcov::RcovTask.new do |test|
29
+ test.libs << 'test'
30
+ test.pattern = 'test/**/test_*.rb'
31
+ test.verbose = true
32
+ end
33
+ rescue LoadError
34
+ task :rcov do
35
+ abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
36
+ end
37
+ end
38
+
39
+ task :test => :check_dependencies
40
+
41
+ task :default => :test
42
+
43
+ require 'rake/rdoctask'
44
+ Rake::RDocTask.new do |rdoc|
45
+ version = File.exist?('VERSION') ? File.read('VERSION') : ""
46
+
47
+ rdoc.rdoc_dir = 'rdoc'
48
+ rdoc.title = "crawler #{version}"
49
+ rdoc.rdoc_files.include('README*')
50
+ rdoc.rdoc_files.include('lib/**/*.rb')
10
51
  end
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 0.2.1
@@ -1,32 +1,74 @@
1
+ # Generated by jeweler
2
+ # DO NOT EDIT THIS FILE DIRECTLY
3
+ # Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
1
4
  # -*- encoding: utf-8 -*-
2
5
 
3
6
  Gem::Specification.new do |s|
4
7
  s.name = %q{crawler}
5
- s.version = "0.2"
8
+ s.version = "0.2.1"
6
9
 
7
- s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
10
+ s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
8
11
  s.authors = ["Tyler Cunnion"]
9
12
  s.date = %q{2010-01-25}
10
13
  s.default_executable = %q{crawler}
11
- s.description = %q{Simple webcrawler}
12
- s.email = %q{tyler.cunnion+ruby@gmail.com}
14
+ s.description = %q{BFS webcrawler that implements Observable}
15
+ s.email = %q{tyler.cunnion@gmail.com}
13
16
  s.executables = ["crawler"]
14
- s.extra_rdoc_files = ["bin/crawler", "lib/crawler.rb", "lib/crawler/observer.rb", "lib/crawler/webcrawler.rb"]
15
- s.files = ["Rakefile", "bin/crawler", "features/step_definitions/crawler_steps.rb", "features/support/env.rb", "lib/crawler.rb", "lib/crawler/observer.rb", "lib/crawler/webcrawler.rb", "spec/crawler/crawler_spec.rb", "spec/crawler/observer_spec.rb", "spec/fixtures/excluded/shouldnt-hit.html", "spec/fixtures/exclusion.html", "spec/fixtures/external.html", "spec/fixtures/index.html", "spec/fixtures/messed-up.html", "spec/fixtures/non-html.html", "spec/fixtures/non-http.html", "spec/fixtures/page2.html", "spec/fixtures/page3.html", "spec/fixtures/page4.html", "spec/fixtures/page5.html", "spec/fixtures/pdf.pdf", "spec/fixtures/self-reference.html", "spec/spec_helper.rb", "Manifest", "crawler.gemspec"]
17
+ s.extra_rdoc_files = [
18
+ "LICENSE",
19
+ "README.rdoc"
20
+ ]
21
+ s.files = [
22
+ ".gitignore",
23
+ "LICENSE",
24
+ "Manifest",
25
+ "README.rdoc",
26
+ "Rakefile",
27
+ "VERSION",
28
+ "bin/crawler",
29
+ "crawler.gemspec",
30
+ "lib/crawler.rb",
31
+ "lib/crawler/observer.rb",
32
+ "lib/crawler/webcrawler.rb",
33
+ "spec/crawler/crawler_spec.rb",
34
+ "spec/crawler/observer_spec.rb",
35
+ "spec/fixtures/excluded/shouldnt-hit.html",
36
+ "spec/fixtures/exclusion.html",
37
+ "spec/fixtures/external.html",
38
+ "spec/fixtures/index.html",
39
+ "spec/fixtures/messed-up.html",
40
+ "spec/fixtures/non-html.html",
41
+ "spec/fixtures/non-http.html",
42
+ "spec/fixtures/page2.html",
43
+ "spec/fixtures/page3.html",
44
+ "spec/fixtures/page4.html",
45
+ "spec/fixtures/page5.html",
46
+ "spec/fixtures/pdf.pdf",
47
+ "spec/fixtures/self-reference.html",
48
+ "spec/spec_helper.rb"
49
+ ]
16
50
  s.homepage = %q{http://github.com/tylercunnion/crawler}
17
- s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Crawler"]
51
+ s.rdoc_options = ["--charset=UTF-8"]
18
52
  s.require_paths = ["lib"]
19
- s.rubyforge_project = %q{crawler}
20
53
  s.rubygems_version = %q{1.3.5}
21
54
  s.summary = %q{Simple webcrawler}
55
+ s.test_files = [
56
+ "spec/crawler/crawler_spec.rb",
57
+ "spec/crawler/observer_spec.rb",
58
+ "spec/spec_helper.rb"
59
+ ]
22
60
 
23
61
  if s.respond_to? :specification_version then
24
62
  current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
25
63
  s.specification_version = 3
26
64
 
27
65
  if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
66
+ s.add_runtime_dependency(%q<nokogiri>, [">= 0"])
28
67
  else
68
+ s.add_dependency(%q<nokogiri>, [">= 0"])
29
69
  end
30
70
  else
71
+ s.add_dependency(%q<nokogiri>, [">= 0"])
31
72
  end
32
73
  end
74
+
@@ -18,6 +18,8 @@ module Crawler
18
18
 
19
19
  # Accepts the following options:
20
20
  # * timeout -- Time limit for the crawl operation, after which a Timeout::Error exception is raised.
21
+ # * external -- Boolean; whether or not the crawler will go outside the original URI's host.
22
+ # * exclude -- A URI will be excluded if it includes any of the strings within this array.
21
23
  def initialize(options={})
22
24
  @crawled = Set.new
23
25
  @queue = []
@@ -30,6 +32,7 @@ module Crawler
30
32
  end
31
33
 
32
34
  # Given a URI object, the crawler will explore every linked page recursively using the Breadth First Search algorithm.
35
+ # Whenever it downloads a page, it notifies observers with an HTTPResponse subclass object and the downloaded URI object.
33
36
  def crawl(start_uri)
34
37
  start_uri = start_uri.normalize
35
38
  @queue << start_uri
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: crawler
3
3
  version: !ruby/object:Gem::Version
4
- version: "0.2"
4
+ version: 0.2.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Tyler Cunnion
@@ -10,25 +10,36 @@ bindir: bin
10
10
  cert_chain: []
11
11
 
12
12
  date: 2010-01-25 00:00:00 -05:00
13
- default_executable:
14
- dependencies: []
15
-
16
- description: Simple webcrawler
17
- email: tyler.cunnion+ruby@gmail.com
13
+ default_executable: crawler
14
+ dependencies:
15
+ - !ruby/object:Gem::Dependency
16
+ name: nokogiri
17
+ type: :runtime
18
+ version_requirement:
19
+ version_requirements: !ruby/object:Gem::Requirement
20
+ requirements:
21
+ - - ">="
22
+ - !ruby/object:Gem::Version
23
+ version: "0"
24
+ version:
25
+ description: BFS webcrawler that implements Observable
26
+ email: tyler.cunnion@gmail.com
18
27
  executables:
19
28
  - crawler
20
29
  extensions: []
21
30
 
22
31
  extra_rdoc_files:
23
- - bin/crawler
24
- - lib/crawler.rb
25
- - lib/crawler/observer.rb
26
- - lib/crawler/webcrawler.rb
32
+ - LICENSE
33
+ - README.rdoc
27
34
  files:
35
+ - .gitignore
36
+ - LICENSE
37
+ - Manifest
38
+ - README.rdoc
28
39
  - Rakefile
40
+ - VERSION
29
41
  - bin/crawler
30
- - features/step_definitions/crawler_steps.rb
31
- - features/support/env.rb
42
+ - crawler.gemspec
32
43
  - lib/crawler.rb
33
44
  - lib/crawler/observer.rb
34
45
  - lib/crawler/webcrawler.rb
@@ -48,18 +59,13 @@ files:
48
59
  - spec/fixtures/pdf.pdf
49
60
  - spec/fixtures/self-reference.html
50
61
  - spec/spec_helper.rb
51
- - Manifest
52
- - crawler.gemspec
53
62
  has_rdoc: true
54
63
  homepage: http://github.com/tylercunnion/crawler
55
64
  licenses: []
56
65
 
57
66
  post_install_message:
58
67
  rdoc_options:
59
- - --line-numbers
60
- - --inline-source
61
- - --title
62
- - Crawler
68
+ - --charset=UTF-8
63
69
  require_paths:
64
70
  - lib
65
71
  required_ruby_version: !ruby/object:Gem::Requirement
@@ -72,14 +78,16 @@ required_rubygems_version: !ruby/object:Gem::Requirement
72
78
  requirements:
73
79
  - - ">="
74
80
  - !ruby/object:Gem::Version
75
- version: "1.2"
81
+ version: "0"
76
82
  version:
77
83
  requirements: []
78
84
 
79
- rubyforge_project: crawler
85
+ rubyforge_project:
80
86
  rubygems_version: 1.3.5
81
87
  signing_key:
82
88
  specification_version: 3
83
89
  summary: Simple webcrawler
84
- test_files: []
85
-
90
+ test_files:
91
+ - spec/crawler/crawler_spec.rb
92
+ - spec/crawler/observer_spec.rb
93
+ - spec/spec_helper.rb
@@ -1,20 +0,0 @@
1
- Given /^the crawl has not begun$/ do
2
- end
3
-
4
- When /^I start a crawl with the URI "([^\"]*)"$/ do |arg1|
5
- @obs = Crawler::Observer.new
6
- @crawler = Crawler::Webcrawler.new
7
-
8
- @crawler.add_observer(@obs)
9
-
10
- @uri = URI.parse(arg1)
11
- @crawler.crawl(@uri)
12
- end
13
-
14
- Then /^the page should be downloaded$/ do
15
- @crawler.crawled.should include(@uri)
16
- end
17
-
18
- Then /^the observer should be updated$/ do
19
- @obs.should_receive(:update)
20
- end
@@ -1,2 +0,0 @@
1
- $LOAD_PATH << File.join(File.dirname(__FILE__), "..", "..", "lib")
2
- require 'crawler'