discovery-mission 0.0.1

data/.gitignore

@@ -0,0 +1,2 @@
+pkg
+example.ru

data/MIT_LICENCE

@@ -0,0 +1,19 @@
+Copyright (c) 2009 - 2010 Mickael Riga
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,37 @@
+= Discovery Mission
+
+A SPACE ODYSSEY
+
+This is just a simple, basic (dummy?) and lightweight crawler.
+It is used to reference all URLs of your website in order to record them on a database, and use them for a sitemap manager.
+
+Just install it:
+
+	# sudo gem install discovery-mission
+
+And then you're ready to take off:
+
+	require 'discovery_mission'
+
+	# Use a block to do something with each path
+	# response is a 'net/http' response object, therefore responds to :body, :code ... 
+
+	DiscoveryMission.for("http://www.my-domain.com") do |url, response|
+		puts "Dave Bowman landed on #{url}"
+		case response
+		when Net::HTTPSuccess
+			puts "Fuck you HAL 9000"
+		when Net::HTTPRedirection
+			puts "Shit! That's Jupiter!"
+		else
+			puts "Bad trip indeed"
+		end
+	end
+
+	# Or just without a block because an Array with all paths is returned
+
+	my_sitemap_entries = DiscoveryMission.for("http://www.my-domain.com")
+
+-
+
+Copyright (c) 2009 - 2010 Mickael Riga. See MIT_LICENCE file for details.

data/Rakefile

@@ -0,0 +1,17 @@
+spec = Gem::Specification.new do |s| 
+  s.name = 'discovery_mission'
+  s.version = "0.0.1"
+  s.platform = Gem::Platform::RUBY
+  s.summary = "This is just a simple, basic (dummy?) and lightweight crawler."
+  s.files = ['discovery_mission.rb']
+  s.test_files = ['spec.rb']
+  s.require_path = '.'
+  s.autorequire = 'discovery_mission'
+  s.author = "Mickael Riga"
+  s.email = "mig@mypeplum.com"
+  s.homepage = "http://github.com/mig-hub/discovery_mission"
+end
+ 
+Rake::GemPackageTask.new(spec) do |pkg| 
+  pkg.need_tar = true 
+end

data/discovery_mission.gemspec

@@ -0,0 +1,13 @@
+Gem::Specification.new do |s| 
+  s.name = 'discovery-mission'
+  s.version = "0.0.1"
+  s.platform = Gem::Platform::RUBY
+  s.summary = "A simple website crawler"
+  s.description = "Discovery Mission is an easy-to-use website crawler. Use it for generating sitemaps."
+  s.files = `git ls-files`.split("\n").sort
+  s.test_files = ['spec.rb']
+  s.require_path = '.'
+  s.author = "Mickael Riga"
+  s.email = "mig@mypeplum.com"
+  s.homepage = "http://github.com/mig-hub/discovery_mission"
+end

data/discovery_mission.rb

@@ -0,0 +1,56 @@
+require 'net/http'
+require 'uri'
+
+class DiscoveryMission
+  attr_accessor :roadmap
+  
+  def self.for(domain, &block)
+    new(domain).launch(&block)
+  end
+  
+  def initialize(domain)
+    @domain = URI(domain)
+    @domain.path = "/" if @domain.path == ""
+    reset
+    puts "\nDiscovery Mission Planned for #{@domain}\nEverything is under control Dave\n\n"
+  end
+  
+  def launch
+    while destination = @roadmap.find {|k,v| v==false}
+      response = land_on(destination.first)
+      yield(destination.first, response) if block_given?
+      explore(destination.first, response)
+      @roadmap[destination.first] = true
+    end
+    uri_list = @roadmap.keys
+    reset
+    uri_list
+  end
+  
+  def reset
+    @roadmap = {@domain.path => false}
+  end
+
+  def land_on(destination)
+    begin
+      response = Net::HTTP.get_response(@domain.host, destination)
+    rescue Exception
+      puts "Error: #{$!}"
+    end
+    return response
+  end
+
+  def explore(destination, response)
+    html = response.body
+    html.scan(/<a href\s*=\s*["']([^"']+)["']/i) do |w|
+      uri_found = URI("#{w}") rescue nil
+      unless (uri_found.nil? or (uri_found.absolute? and uri_found.host!=@domain.host) or (uri_found.path=='' or uri_found.path=='#' or uri_found.path[/^javascript/]))
+        destination += '/' unless destination[/\/$/]
+        uri_found.path = destination + uri_found.path unless uri_found.path[/^\//]
+        unless @roadmap.key?(uri_found.path)
+          @roadmap.store(uri_found.path, false)
+        end
+      end
+    end
+  end
+end

data/spec.rb

@@ -0,0 +1,65 @@
+require 'rubygems'
+require 'bacon'
+Bacon.summary_on_exit
+
+MyResponse = Struct::new(:body)
+
+def template_this(content)
+  "<html><head><title></title</head><body>#{content}</body></html>"
+end
+
+require File.expand_path('./discovery_mission', File.dirname(__FILE__))
+class DiscoveryMission
+  attr_reader :roadmap
+  def land_on(destination)
+    destination=='/' ? MyResponse.new("<html><head><title></title</head><body><a href='/good/destination'>click</a></body></html>") : MyResponse.new('')
+  end
+end
+
+describe DiscoveryMission do
+  describe 'explore' do
+    before do
+      @dm = DiscoveryMission.new('http://www.domain.com')
+    end
+    it 'Should add correct destinations to roadmap' do
+      html = template_this("<a href='/good/destination'>click</a>")
+      @dm.explore("/", MyResponse.new(html))
+      @dm.roadmap.keys.should.include?('/good/destination')
+    end
+    it 'Should not raise when bad URI' do
+      html = template_this("<a href='http://'>click</a>")
+      @dm.explore("/", MyResponse.new(html))
+      @dm.roadmap.keys.should==["/"]
+    end
+    it 'Should skip pointless uri' do
+      html = template_this(<<-EOT)
+        <a href='http://www.another_domain.com'>click</a>
+        <a href='http://www.another_domain.com/with/path'>click</a>
+        <a href='#'>click</a>
+        <a href='javascript:void(0);'>click</a>
+      EOT
+      @dm.explore("/", MyResponse.new(html))
+      @dm.roadmap.keys.should==["/"]
+    end
+    it 'Sould add current path if uri is relative to it' do
+      html = template_this("<a href='chapter_one'>click</a>")
+      @dm.explore("/novel", MyResponse.new(html))
+      @dm.roadmap.keys.should.include?("/novel/chapter_one")
+    end
+    it 'Should not duplicate entries' do
+      html = template_this(<<-EOT)
+        <a href='/'>click</a>
+        <a href='http://www.domain.com'>click</a>
+      EOT
+      @dm.explore("/", MyResponse.new(html))
+      @dm.roadmap.keys.should==["/"]
+    end
+  end
+  describe 'launch' do
+    it 'Should be reseted for next launch' do
+      dm = DiscoveryMission.new('http://www.domain.com')
+      dm.launch.size.should==2
+      dm.roadmap.size.should==1
+    end
+  end
+end

metadata

@@ -0,0 +1,73 @@
+--- !ruby/object:Gem::Specification 
+name: discovery-mission
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Mickael Riga
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-06-02 00:00:00 +01:00
+default_executable: 
+dependencies: []
+
+description: Discovery Mission is an easy-to-use website crawler. Use it for generating sitemaps.
+email: mig@mypeplum.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- MIT_LICENCE
+- README.rdoc
+- Rakefile
+- discovery_mission.gemspec
+- discovery_mission.rb
+- spec.rb
+has_rdoc: true
+homepage: http://github.com/mig-hub/discovery_mission
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- .
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: A simple website crawler
+test_files: 
+- spec.rb