krawler 1.0.11 → 1.0.12

data/bin/krawl

@@ -21,8 +21,12 @@ optparse = OptionParser.new do |opts|
     options[:s] = true
   end
 
+  opts.on('-d', '--ignore-domain', 'Ignore domain restrictions', 'Default: false') do |d|
+    options[:s] = true
+  end
+
   opts.on('-c', '--concurrent count', 'Crawl with count number of concurrent connections', 'Default: 4') do |c|
-    options[:c] = c.to_i
+    options[:d] = true
   end
 
   opts.on('-r', '--randomize', 'Randomize crawl path', 'Default: true') do |r|
@@ -66,5 +70,6 @@ Krawler::Base.new(ARGV[0] || 'http://localhost:3000/', {
   :no_cache  => options[:nc],
   :username  => options[:u],
   :password  => options[:p],
-  :login_url => options[:l]
+  :login_url => options[:l],
+  :domain    => options[:d]
 }).base

data/lib/krawler/version.rb

@@ -1,3 +1,3 @@
 module Krawler
-  VERSION = '1.0.11'
+  VERSION = '1.0.12'
 end

data/lib/krawler.rb

@@ -22,6 +22,7 @@ module Krawler
       @exclude          = options[:exclude]
       @include          = options[:include]
       @restrict         = options[:restrict]
+      @domain           = options[:domain]
       @randomize        = options[:randomize]
       @threads          = options[:threads]   || 1
       @username         = options[:username]
@@ -127,7 +128,7 @@ module Krawler
             next
           end
   
-          if (new_link =~ /^#{Regexp.escape(@host)}/) || (new_link =~ /^\//) # don't crawl external domains
+          if @domain || (new_link =~ /^#{Regexp.escape(@host)}/) || (new_link =~ /^\//) # don't crawl external domains
   
             next if @crawled_links.include?(new_link) || @links_to_crawl.include?(new_link) # don't crawl what we've alread crawled
             next if @exclude  && new_link =~ /#{@exclude}/   # don't crawl excluded matched paths

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: krawler
 version: !ruby/object:Gem::Version
-  version: 1.0.11
+  version: 1.0.12
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-11-28 00:00:00.000000000 Z
+date: 2012-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -71,3 +71,4 @@ signing_key:
 specification_version: 3
 summary: ''
 test_files: []
+has_rdoc: