codesake_links 0.50 → 0.71

data/bin/links

@@ -64,52 +64,55 @@ target = ARGV[0]
 logger.helo "#{APPNAME} v#{Codesake::Links::VERSION} (C) 2013 - paolo@armoredcode.com is starting up"
 
 
-list<<target if list.empty?
+# list<<target if list.empty?
 
-logger.die("links: missing target") if list[0].nil?
+logger.die("missing target") if target.nil?
+logger.die("no -b or -r option specified") unless bulk or robots
 
 if robots
-  list = Codesake::Links::Api.robots(target)
-  logger.err "#{target}: no robots.txt found\n" if list.empty?
+  res = Codesake::Links::Api.robots(target)
+  list = res[:disallow_list]
+  logger.err "#{target}: no robots.txt found (#{res[:error]})\n" if res[:status] == :KO
+  logger.ok "no disallowed entries to test on #{target}" if list.empty?
+  logger.ok "found #{list.size} disallowed url(s) on #{target}" unless list.empty?
+  list.each do |l|
+    logger.ok "#{l} - #{Codesake::Links::Api.code(target+l, nil)}"
+  end
+  logger.helo "leaving"
+  Kernel.exit(0)
 end
 
+
 list.each do |l|
-  if robots or bulk
-    if ! l.start_with? '/'
-      l = '/'+l.chomp
-    end
-    if ! target.start_with? 'http://' and ! target.start_with? 'https://'
-      #defaulting to HTTP when no protocol has been supplied
-      target = "http://"+target
-    end
+  unless l.start_with? "#"
 
-    logger.log "#{target}#{l}:"
-    start = Time.now
-    code = Codesake::Links::Api.code(target+l, proxy)
-    stop = Time.now
-  else
-    logger.log "#{l}:"
+    l = l.chomp if l.end_with? "\n"
+    l = '/'+l unless l.start_with? '/'
+
+    url = target + l
     start = Time.now
-    code = Codesake::Links::Api.code(l, proxy)
+    code = Codesake::Links::Api.code(url, nil)
     stop = Time.now
-  end
 
-  str=Codesake::Links::Api.human(code)
+    str=Codesake::Links::Api.human(code)
 
-  Codesake::Links::Utils.print_str(logger, str, start, stop)     unless show_code
-  Codesake::Links::Utils.print_code(logger, str, code, start, stop) if show_code
- 
+    if code == "200"
+      Codesake::Links::Utils.print_str(url, logger, str, start, stop)     unless show_code 
+      Codesake::Links::Utils.print_code(url, logger, code, start, stop)   if show_code 
+    end
 
-  if code == 301 or code == 302
-    start = Time.now
-    new_link = Codesake::Links::Api.follow(l, proxy)
-    stop = Time.now
-    logger.log "following from #{l} to #{new_link}\n"
-    str=Codesake::Links::Api.human(code)
 
-    Codesake::Links::Utils.print_str(logger, str, start, stop)     unless show_code
-    Codesake::Links::Utils.print_code(logger, str, code, start, stop) if show_code
+    if code == 301 or code == 302
+      start = Time.now
+      new_link = Codesake::Links::Api.follow(l, proxy)
+      stop = Time.now
+      logger.log "following from #{l} to #{new_link}\n"
+      str=Codesake::Links::Api.human(code)
+
+      Codesake::Links::Utils.print_str(logger, str, start, stop)    unless show_code
+      Codesake::Links::Utils.print_code(logger, code, start, stop)  if show_code
 
+    end
   end
 
 end

data/codesake_links.gemspec

@@ -22,9 +22,7 @@ Gem::Specification.new do |s|
   # specify any dependencies here; for example:
   s.add_development_dependency "rake"
   s.add_development_dependency "rspec"
-  s.add_development_dependency "rainbow"
-  # s.add_runtime_dependency "rest-client"
-  s.add_runtime_dependency "rainbow"
+  s.add_development_dependency 'webmock'
 
   s.add_dependency "nokogiri"
   s.add_dependency "mechanize"

data/lib/codesake/links/api.rb

@@ -32,35 +32,30 @@ module Codesake
       end
 
       # TESTING: SPIDERS, ROBOTS, AND CRAWLERS (OWASP-IG-001)
-      def self.robots(site, only_disallow=true)
+      def self.robots(site)
 
-        if (! site.start_with? 'http://') and (! site.start_with? 'https://')
-          site = 'http://'+site
-        end
+        site = 'http://'+site unless site.start_with? 'http://' or site.start_with? 'https://'
+        
+
+        allow_list = []
+        disallow_list = []
 
-        list = []
         begin
           res=Net::HTTP.get_response(URI(site+'/robots.txt'))
-          if (res.code != "200")
-            return []
-          end
+          return {:status=>:KO, :allow_list=>[], :disallow_list=>[], :error=>"robots.txt response code was #{res.code}"} if (res.code != "200")
+
 
           res.body.split("\n").each do |line|
-            if only_disallow
-              if (line.downcase.start_with?('disallow'))
-                list << line.split(":")[1].strip.chomp
-              end
-            else
-              if (line.downcase.start_with?('allow') or line.downcase.start_with?('disallow'))
-                list << line.split(":")[1].strip.chomp
-              end
-            end
+
+            disallow_list << line.split(":")[1].strip.chomp if (line.downcase.start_with?('disallow'))
+            allow_list << line.split(":")[1].strip.chomp if (line.downcase.start_with?('allow'))
+
           end
-        rescue
-          return []
+        rescue Exception => e
+          return {:status=>:KO, :allow_list=>[], :disallow_list=>[], :error=>e.message}
         end
 
-        list
+        {:status=>:OK, :allow_list=>allow_list, :disallow_list=>disallow_list, :error=>""}
       end
 
       def self.follow(url, proxy)
@@ -97,14 +92,18 @@ module Codesake
         begin
           uri = URI(url)
           if uri.scheme == 'http'
-            Net::HTTP::Proxy(proxy[:host], proxy[:port]).start(uri.host) {|http|
-              if (method == :get)
-                res = http.get(uri.request_uri)
-              else
-                res = http.head(uri.request_uri)
-              end
-              return res
-            }
+            unless proxy.nil?
+              Net::HTTP::Proxy(proxy[:host], proxy[:port]).start(uri.host) {|http|
+                if (method == :get)
+                  res = http.get(uri.request_uri)
+                else
+                  res = http.head(uri.request_uri)
+                end
+                return res
+              }
+            else
+              res = Net::HTTP.get_response(URI(url))
+            end
             # res = Net::HTTP.get_response(URI(url))
           else
             request=Net::HTTP.new(uri.host, uri.port)

data/lib/codesake/links/utils.rb

@@ -3,18 +3,16 @@ module Codesake
   module Links
     class Utils
 
-      def self.print_str(logger, str, start, stop)
-        logger.ok "#{str} (#{((stop-start) * 1000).round} msec)\n" if str == "Open"
-        logger.err " #{str} (#{((stop-start) * 1000).round} msec)\n" if (str == "Closed" or str == "Non existent")
-        logger.warn " #{str} (#{((stop-start) * 1000).round} msec)\n" if (str != "Closed" and str != "Non existent" and str != "Open")
+      def self.print_str(url, logger, str, start, stop)
+        logger.ok "#{url}: #{str} (#{((stop-start) * 1000).round} msec)" if str == "Open"
+        logger.log "#{url}: #{str} (#{((stop-start) * 1000).round} msec)" unless str == "Open"
 
         return
       end
 
-      def self.print_code(logger, str, code, start, stop)
-        logger.ok "#{code} (#{((stop-start) * 1000).round} msec)\n" if str == "Open"
-        logger.err " #{code} (#{((stop-start) * 1000).round} msec)\n" if (str == "Closed" or str == "Non existent")
-        logger.warn " #{code} (#{((stop-start) * 1000).round} msec)\n" if (str != "Closed" and str != "Non existent" and str != "Open")
+      def self.print_code(url, logger, code, start, stop)
+        logger.ok "#{url}: #{code} (#{((stop-start) * 1000).round} msec)" if code == "200"
+        logger.warn "#{url}: #{code} (#{((stop-start) * 1000).round} msec)" unless code == "200"
 
         return
       end

data/lib/codesake/links/version.rb

@@ -1,5 +1,5 @@
 module Codesake
   module Links
-    VERSION = "0.50"
+    VERSION = "0.71"
   end
 end

data/spec/codesake_links_api_spec.rb

@@ -0,0 +1,60 @@
+require 'spec_helper'
+include WebMock::API
+
+describe "The API for Codesake Links" do
+  it "returns an array with a single / if the robots.txt contains only Allow: /" do
+    stub_request(:get, "http://www.test.com/robots.txt").
+      with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
+      to_return(:status=>200, :body=>"User-agent: *\nAllow: /", :headers=>{})
+
+    ret = Codesake::Links::Api.robots("http://www.test.com")
+    ret[:status].should           == :OK
+    ret[:allow_list].size.should  == 1
+    ret[:allow_list].should       == [ '/' ]
+  end
+
+  it "returns an array with a single / if the robots.txt contains only Allow: / for an HTTPS site" do
+    stub_request(:get, "http://www.test.com:443/robots.txt").
+      with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
+      to_return(:status=>200, :body=>"User-agent: *\nAllow: /", :headers=>{})
+
+    ret = Codesake::Links::Api.robots("https://www.test.com")
+    ret[:status].should           == :OK
+    ret[:allow_list].size.should  == 1
+    ret[:allow_list].should       == [ '/' ]
+  end
+
+
+  it "can handle an input without the protocol if target talks HTTP" do
+    stub_request(:get, "http://www.test.com/robots.txt").
+      with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
+      to_return(:status=>200, :body=>"User-agent: *\nAllow: /", :headers=>{})
+
+    ret = Codesake::Links::Api.robots("www.test.com")
+    ret[:status].should           == :OK
+    ret[:allow_list].size.should  == 1
+    ret[:allow_list].should       == [ '/' ]
+  end
+
+
+  it "can't handle an input without the protocol if target talks *only* HTTPS" do
+    stub_request(:get, "http://www.test.com:443/robots.txt").
+      with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
+      to_return(:status=>200, :body=>"User-agent: *\nAllow: /", :headers=>{})
+
+    ret = Codesake::Links::Api.robots("www.test.com")
+    ret[:status].should           == :KO
+  end
+
+  it "returns a list of disallowed URLs" do
+    stub_request(:get, "http://www.test.com/robots.txt").
+      with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
+      to_return(:status=>200, :body=>"User-agent: *\nAllow: /\nDisallow: /private\nDisallow: /cgi-bin\nDisallow: /a-secret-dir", :headers=>{})
+
+    ret = Codesake::Links::Api.robots("www.test.com")
+    ret[:disallow_list].size.should     == 3
+    ret[:disallow_list].should          == [ '/private', '/cgi-bin', '/a-secret-dir' ]
+
+  end
+
+end

data/spec/spec_helper.rb

@@ -1 +1,2 @@
-require 'links'
+require 'codesake_links'
+require 'webmock/rspec'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: codesake_links
 version: !ruby/object:Gem::Version
-  version: '0.50'
+  version: '0.71'
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-11 00:00:00.000000000 Z
+date: 2013-04-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -44,7 +44,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rainbow
+  name: webmock
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
@@ -59,22 +59,6 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: rainbow
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -145,6 +129,7 @@ files:
 - lib/codesake/links/utils.rb
 - lib/codesake/links/version.rb
 - lib/codesake_links.rb
+- spec/codesake_links_api_spec.rb
 - spec/spec_helper.rb
 homepage: http://codesake.com
 licenses:
@@ -161,7 +146,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -1757904362167703742
+      hash: 2079895705737470252
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -170,7 +155,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -1757904362167703742
+      hash: 2079895705737470252
 requirements: []
 rubyforge_project: links
 rubygems_version: 1.8.24
@@ -178,4 +163,5 @@ signing_key:
 specification_version: 3
 summary: Fetch, discover and crawl what's available in a website.
 test_files:
+- spec/codesake_links_api_spec.rb
 - spec/spec_helper.rb