rawler 0.1.1 → 0.1.3

data/{README.rdoc → README.md}

@@ -8,12 +8,13 @@ Rawler will only parse pages with content type 'text/html', but it will check fo
 
   rawler http://example.com [options]
 
-	where [options] are:
-	  --username, -u <s>:   HTTP Basic Username
-	  --password, -p <s>:   HTTP Basic Password
-                --wait:   Number of seconds to wait betweet requests (default 3)
-	       --version, -v:   Print version and exit
-	          --help, -h:   Show this message
+      where [options] are:
+        --username, -u <s>:   HTT Basic Username
+        --password, -p <s>:   HTT Basic Password
+            --wait, -w <f>:   Seconds to wait between requests, may be fractional e.g. '1.5' (default: 3.0)
+                 --log, -l:   Log results to file rawler_log.txt
+             --version, -v:   Print version and exit
+                --help, -h:   Show this message
 
 ### INSTALL:
 

data/VERSION

@@ -1 +1 @@
-0.1.1
+0.1.3

data/bin/rawler

@@ -16,7 +16,8 @@ EOS
   
   opt :username, "HTT Basic Username", :type => :string
   opt :password, "HTT Basic Password", :type => :string
-  opt :wait, "Number of seconds to wait betweet requests", :default => 3
+  opt :wait, "Seconds to wait between requests, may be fractional e.g. '1.5'", :type => :float, :default => 3.0
+  opt :log, "Log results to file rawler_log.txt", :type => :boolean, :default => false
 end
 
 domain = ARGV.shift

data/lib/rawler.rb

@@ -35,7 +35,7 @@ require 'rawler/core_extensions'
 # The Rawler module itself is very simple, and it's only used for storing configuration data like the url that we want to fetch, basic username and password.
 
 module Rawler
-  VERSION = '0.1.1'
+  VERSION = "#{File.read(File.expand_path(File.dirname(__FILE__)) + '/../VERSION')}"
   
   # `output` is where we want to direct output. It's set to `$stdout` by default.
 
@@ -52,7 +52,11 @@ module Rawler
   # Username and Password for basic auth, if needed.
 
   mattr_accessor :username, :password
-  
+
+  # Log switch
+
+  mattr_accessor :log
+
   # Here we autoload when needed the specific namespaces.
 
   # [Rawler::Base](rawler/base.html) is responsible for validating all the pages in a domain. It's where all the magic happens.

data/lib/rawler/base.rb

@@ -22,13 +22,16 @@ module Rawler
       Rawler.output   = Logger.new(output)
       Rawler.username = options[:username]
       Rawler.password = options[:password]
-      Rawler.wait     = options[:wait].to_i
+      Rawler.wait     = options[:wait]
+      Rawler.log      = options[:log]
+      @logfile = File.new("rawler_log.txt", "w") if Rawler.log
     end
     
     # The method used to start the real validation process
 
     def validate
       validate_links_in_page(Rawler.url)
+      @logfile.close if Rawler.log
     end
     
     private
@@ -113,9 +116,7 @@ module Rawler
 
       code = code.to_i
       case code / 100
-      when 1
-        Rawler.output.info(message)
-      when 2 then
+      when 1,2
         Rawler.output.info(message)
       when 3 then
         Rawler.output.warn(message)
@@ -124,6 +125,7 @@ module Rawler
       else
         Rawler.output.error("Unknown code #{message}")
       end
+      @logfile.puts(message) if Rawler.log
     end
     
   end

data/lib/rawler/request.rb

@@ -16,11 +16,17 @@ module Rawler
       
       private
       
-      def perform_request(method, url)     
+      def perform_request(method, url)
         uri = URI.parse(url)
-        http = Net::HTTP.new(uri.host, uri.port)
-        http.use_ssl = (uri.scheme == 'https')
 
+        # Use http_proxy if set
+        proxy = URI.parse(ENV['http_proxy']) if ENV['http_proxy'] rescue nil
+        if proxy
+          http = Net::HTTP::Proxy(proxy.host, proxy.port).new(uri.host, uri.port)
+        else
+          http = Net::HTTP.new(uri.host, uri.port)
+        end
+        http.use_ssl = (uri.scheme == 'https')
         http.verify_mode = OpenSSL::SSL::VERIFY_NONE
         
         path = (uri.path.size == 0)  ? "/" : uri.path

data/rawler.gemspec

@@ -0,0 +1,75 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = "rawler"
+  s.version = "0.1.3"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Oscar Del Ben"]
+  s.date = "2011-11-11"
+  s.description = "Rawler is a tool that crawls the links of your website"
+  s.email = "info@oscardelben.com"
+  s.executables = ["rawler"]
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.md"
+  ]
+  s.files = [
+    "Gemfile",
+    "Gemfile.lock",
+    "LICENSE.txt",
+    "README.md",
+    "Rakefile",
+    "VERSION",
+    "bin/rawler",
+    "lib/rawler.rb",
+    "lib/rawler/base.rb",
+    "lib/rawler/core_extensions.rb",
+    "lib/rawler/core_extensions/module.rb",
+    "lib/rawler/crawler.rb",
+    "lib/rawler/request.rb",
+    "rawler.gemspec",
+    "spec/lib/base_spec.rb",
+    "spec/lib/rawler/base_spec.rb",
+    "spec/lib/rawler/crawler_spec.rb",
+    "spec/lib/rawler_spec.rb",
+    "spec/spec.opts",
+    "spec/spec_helper.rb",
+    "test/helper.rb",
+    "test/test_rawler.rb",
+    "vendor/lib-trollop.rb"
+  ]
+  s.homepage = "http://github.com/oscardelben/rawler"
+  s.licenses = ["MIT"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = "1.8.11"
+  s.summary = "Rawler is a tool that crawls the links of your website"
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<nokogiri>, [">= 0"])
+      s.add_development_dependency(%q<shoulda>, [">= 0"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
+      s.add_development_dependency(%q<rcov>, [">= 0"])
+    else
+      s.add_dependency(%q<nokogiri>, [">= 0"])
+      s.add_dependency(%q<shoulda>, [">= 0"])
+      s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
+      s.add_dependency(%q<rcov>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<nokogiri>, [">= 0"])
+    s.add_dependency(%q<shoulda>, [">= 0"])
+    s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+    s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
+    s.add_dependency(%q<rcov>, [">= 0"])
+  end
+end
+

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: rawler
 version: !ruby/object:Gem::Version 
-  hash: 25
+  hash: 29
   prerelease: 
   segments: 
   - 0
   - 1
-  - 1
-  version: 0.1.1
+  - 3
+  version: 0.1.3
 platform: ruby
 authors: 
 - Oscar Del Ben
@@ -15,12 +15,9 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-09-16 00:00:00 +02:00
-default_executable: rawler
+date: 2011-11-11 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  prerelease: false
-  type: :runtime
   requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -30,11 +27,11 @@ dependencies:
         segments: 
         - 0
         version: "0"
-  name: nokogiri
   version_requirements: *id001
-- !ruby/object:Gem::Dependency 
+  name: nokogiri
   prerelease: false
-  type: :development
+  type: :runtime
+- !ruby/object:Gem::Dependency 
   requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -44,11 +41,11 @@ dependencies:
         segments: 
         - 0
         version: "0"
-  name: shoulda
   version_requirements: *id002
-- !ruby/object:Gem::Dependency 
+  name: shoulda
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency 
   requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -60,11 +57,11 @@ dependencies:
         - 0
         - 0
         version: 1.0.0
-  name: bundler
   version_requirements: *id003
-- !ruby/object:Gem::Dependency 
+  name: bundler
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency 
   requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -76,11 +73,11 @@ dependencies:
         - 6
         - 4
         version: 1.6.4
-  name: jeweler
   version_requirements: *id004
-- !ruby/object:Gem::Dependency 
+  name: jeweler
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency 
   requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -90,8 +87,10 @@ dependencies:
         segments: 
         - 0
         version: "0"
-  name: rcov
   version_requirements: *id005
+  name: rcov
+  prerelease: false
+  type: :development
 description: Rawler is a tool that crawls the links of your website
 email: info@oscardelben.com
 executables: 
@@ -100,12 +99,12 @@ extensions: []
 
 extra_rdoc_files: 
 - LICENSE.txt
-- README.rdoc
+- README.md
 files: 
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
-- README.rdoc
+- README.md
 - Rakefile
 - VERSION
 - bin/rawler
@@ -115,6 +114,7 @@ files:
 - lib/rawler/core_extensions/module.rb
 - lib/rawler/crawler.rb
 - lib/rawler/request.rb
+- rawler.gemspec
 - spec/lib/base_spec.rb
 - spec/lib/rawler/base_spec.rb
 - spec/lib/rawler/crawler_spec.rb
@@ -124,7 +124,6 @@ files:
 - test/helper.rb
 - test/test_rawler.rb
 - vendor/lib-trollop.rb
-has_rdoc: true
 homepage: http://github.com/oscardelben/rawler
 licenses: 
 - MIT
@@ -154,7 +153,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.6.2
+rubygems_version: 1.8.11
 signing_key: 
 specification_version: 3
 summary: Rawler is a tool that crawls the links of your website