krawler 0.0.2 → 0.1.0

data/bin/krawl

@@ -1,4 +1,25 @@
 #!/usr/bin/env ruby
 require 'krawler'
+require 'optparse'
 
-Krawler::Base.new(ARGV[0] || 'http://localhost:3000/').base
+options = {}
+optparse = OptionParser.new do |opts|
+  opts.banner = "Usage: krawl [url] [options]"
+
+  opts.separator ""
+  opts.separator "Specific options:"
+
+  opts.on("-ex [regex]", "Exclude matching paths") do |ex|
+    options[:ex] = ex
+  end
+end
+optparse.parse!
+
+if ARGV.empty? || !(ARGV[0] =~ /^http/)
+  puts optparse
+  exit(-1)
+end
+
+Krawler::Base.new(ARGV[0] || 'http://localhost:3000/', {
+  :exclude => options[:ex]
+}).base

data/krawler.gemspec

@@ -4,9 +4,9 @@ require File.expand_path('../lib/krawler/version', __FILE__)
 Gem::Specification.new do |gem|
   gem.authors       = ["Mike Evans"]
   gem.email         = ["mike@urlgonomics.com"]
-  gem.description   = %q{Simple little rake task to crawl a site.}
+  gem.description   = %q{Simple little website crawler.}
   gem.summary       = %q{}
-  gem.homepage      = ""
+  gem.homepage      = 'https://github.com/mje113/krawl'
 
   gem.add_dependency 'mechanize', '~> 2.5.0'
   gem.rubyforge_project = 'krawler'

data/lib/krawler/version.rb

@@ -1,3 +1,3 @@
 module Krawler
-  VERSION = "0.0.2"
+  VERSION = "0.1.0"
 end

data/lib/krawler.rb

@@ -1,17 +1,19 @@
 require 'krawler/version'
 require 'mechanize'
+require 'timeout'
 
 module Krawler
 
   class Base
 
-    def initialize(url)
+    def initialize(url, options)
       @base = url
       @agent = Mechanize.new
       @links_to_crawl = [@base]
       @crawled_links  = []
       @bad_links      = []
       @suspect_links  = []
+      @exclude        = options[:exclude]
     end
   
     def base
@@ -41,14 +43,10 @@ module Krawler
         @bad_links << link
         return
       rescue Timeout::Error => e
-        puts "SLOW PAGE, timeout at #{Time.now - start} seconds"
         @suspect_links << link
         return
-      end
-  
-      elapsed = Time.now - start
-      if elapsed > 7.0
-        puts "SLOW PAGE, #{Time.now - start} seconds"
+      ensure
+        puts "    [#{Time.now - start}s] #{@links_to_crawl.size} links..."
       end
   
       return if !page.respond_to?(:links)
@@ -56,6 +54,7 @@ module Krawler
         new_link = new_link.href
         if (new_link =~ /^#{Regexp.escape(@base)}/) || (new_link =~ /^\//)
           next if @crawled_links.include?(new_link)
+          next if @exclude && new_link =~ /#{@exclude}/
     
           @links_to_crawl << new_link
         end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: krawler
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-10 00:00:00.000000000 Z
+date: 2012-05-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
-  requirement: &70335080854540 !ruby/object:Gem::Requirement
+  requirement: &70120618617640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,8 +21,8 @@ dependencies:
         version: 2.5.0
   type: :runtime
   prerelease: false
-  version_requirements: *70335080854540
-description: Simple little rake task to crawl a site.
+  version_requirements: *70120618617640
+description: Simple little website crawler.
 email:
 - mike@urlgonomics.com
 executables:
@@ -40,7 +40,7 @@ files:
 - lib/krawler.rb
 - lib/krawler/version.rb
 - tasks/krawler.rake
-homepage: ''
+homepage: https://github.com/mje113/krawl
 licenses: []
 post_install_message: 
 rdoc_options: []