codesake_links 0.50

data/.gitignore

@@ -0,0 +1,6 @@
+.rvmrc
+*.swp
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in w3ping.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2010-2012 Paolo Perego
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,51 @@
+# links
+
+## Introduction
+
+[links](https://github.com/thesp0nge/links) is a tool for discovering a website
+available pages without making too much noise.
+
+The idea came to me during a penetration test since I had a bulk list of URLs
+to check for availability and I wanted to automate this process.
+
+## Installing links
+
+Installing links is easy. You can always obtain the latest stable code by using the following command: 
+
+``` 
+gem install links
+```
+
+If you want to install a _pre_ release, such as a _release candidate_ you can do it this way:
+```
+gem install links --pre
+```
+
+## Using links
+
+After you installed links gem, you have the links command you can use this way:
+
+```
+links http://www.some.org/somepage.html
+```
+
+## Contributing to links
+ 
+* Check out the latest master to make sure the feature hasn't been implemented
+  or the bug hasn't been fixed yet
+* Check out the issue tracker to make sure someone already hasn't requested it
+  and/or contributed it
+* Fork the project
+* Start a feature/bugfix branch
+* Commit and push until you are happy with your contribution
+* Make sure to add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to
+  have your own version, or is otherwise necessary, that is fine, but please
+  isolate to its own commit so I can cherry-pick around it.
+
+## Copyright
+
+Copyright (c) 2010-2013 Paolo Perego, <paolo@armoredcode.com>. See LICENSE for
+further details.
+

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new
+
+task :default => :spec
+task :test => :spec
+

data/bin/links

@@ -0,0 +1,115 @@
+#!/usr/bin/env ruby
+require "rainbow"
+require 'getoptlong'
+
+require "codesake_commons"
+require "codesake_links"
+
+APPNAME = File.basename($0)
+
+logger = Codesake::Commons::Logging.instance
+opts = GetoptLong.new(
+  [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
+  [ '--version', '-v', GetoptLong::NO_ARGUMENT ],
+  [ '--bulk', '-b', GetoptLong::REQUIRED_ARGUMENT ],
+  [ '--proxy', '-P',  GetoptLong::REQUIRED_ARGUMENT ],
+  [ '--code', '-c', GetoptLong::NO_ARGUMENT ],
+  [ '--robots', '-r', GetoptLong::NO_ARGUMENT ]
+)
+trap("INT") { logger.die("[INTERRUPTED]") }
+
+
+list=[]
+robots=false
+bulk=false
+show_code=false
+proxy={:host=>nil, :port=>-1}
+
+opts.each do |opt, arg|
+  case opt
+  when '--help'
+    puts "usage: links [-bvh] [filename]"
+    puts "   -b filename: loads the url list from a plain text file"
+    puts "   -r : parse robots.txt and make requests to disallowed urls"
+    puts "   -c : shows the return code instead of human readable answer"
+    puts "   -P host:port : connect using a proxy server. Useful in combination with Paros, Owasp Zap and other"
+    puts "   -v : shows version information"
+    puts "   -h : shows this help"
+    exit 0
+  when '--version'
+    puts "#{Codesake::Links::VERSION}"
+    exit 0
+  when '--code'
+    show_code = true
+  when '--proxy'
+    proxy[:host]=arg.split(':')[0]
+    proxy[:port]=arg.split(':')[1].to_i
+  when '--robots'
+    robots=true
+  when '--bulk'
+    bulk=true
+    if ! File.exists?(arg)
+      puts "links: file not found (#{arg})".color(:red)
+      exit 1
+    end
+    list = File.open(arg).readlines
+    if list.count <= 0
+      puts "links: invalid url list".color(:red)
+      exit 1
+    end
+  end
+end
+
+target = ARGV[0]
+logger.helo "#{APPNAME} v#{Codesake::Links::VERSION} (C) 2013 - paolo@armoredcode.com is starting up"
+
+
+list<<target if list.empty?
+
+logger.die("links: missing target") if list[0].nil?
+
+if robots
+  list = Codesake::Links::Api.robots(target)
+  logger.err "#{target}: no robots.txt found\n" if list.empty?
+end
+
+list.each do |l|
+  if robots or bulk
+    if ! l.start_with? '/'
+      l = '/'+l.chomp
+    end
+    if ! target.start_with? 'http://' and ! target.start_with? 'https://'
+      #defaulting to HTTP when no protocol has been supplied
+      target = "http://"+target
+    end
+
+    logger.log "#{target}#{l}:"
+    start = Time.now
+    code = Codesake::Links::Api.code(target+l, proxy)
+    stop = Time.now
+  else
+    logger.log "#{l}:"
+    start = Time.now
+    code = Codesake::Links::Api.code(l, proxy)
+    stop = Time.now
+  end
+
+  str=Codesake::Links::Api.human(code)
+
+  Codesake::Links::Utils.print_str(logger, str, start, stop)     unless show_code
+  Codesake::Links::Utils.print_code(logger, str, code, start, stop) if show_code
+ 
+
+  if code == 301 or code == 302
+    start = Time.now
+    new_link = Codesake::Links::Api.follow(l, proxy)
+    stop = Time.now
+    logger.log "following from #{l} to #{new_link}\n"
+    str=Codesake::Links::Api.human(code)
+
+    Codesake::Links::Utils.print_str(logger, str, start, stop)     unless show_code
+    Codesake::Links::Utils.print_code(logger, str, code, start, stop) if show_code
+
+  end
+
+end

data/codesake_links.gemspec

@@ -0,0 +1,33 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "codesake/links/version"
+
+Gem::Specification.new do |s|
+  s.name        = "codesake_links"
+  s.version     = Codesake::Links::VERSION
+  s.authors     = ["Paolo Perego"]
+  s.email       = ["paolo@armoredcode.com"]
+  s.homepage    = "http://codesake.com"
+  s.summary     = %q{Fetch, discover and crawl what's available in a website.}
+  s.description = %q{During the first stage of a security test, it's useful to enumerate website urls without making too much noise. Links can help in this using robots.txt or link in a web page telling you the website contents.}
+  s.license     = "BSD"
+
+  s.rubyforge_project = "links"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  # specify any dependencies here; for example:
+  s.add_development_dependency "rake"
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "rainbow"
+  # s.add_runtime_dependency "rest-client"
+  s.add_runtime_dependency "rainbow"
+
+  s.add_dependency "nokogiri"
+  s.add_dependency "mechanize"
+
+  s.add_dependency "codesake_commons"
+end

data/lib/codesake/links/api.rb

@@ -0,0 +1,130 @@
+require "net/http"
+require "nokogiri"
+
+module Codesake
+
+  module Links
+    module Api
+
+      #    include Links::Google
+
+      def self.get(url, proxy)
+        return Links::Api.request({:url=>url, :proxy=>proxy, :method=>:get})
+      end
+
+      def self.head(url, proxy)
+        return Links::Api.request({:url=>url, :proxy=>proxy, :method=>:head})
+      end
+
+      def self.code(url, proxy)
+        res = Links::Api.get(url, proxy)
+        (res.nil?)? -1 : res.code
+      end
+
+      def self.links(url, proxy)
+        res = Links::Api.get(url, proxy)
+        if res.nil?
+          return []
+        end
+        doc = Nokogiri::HTML.parse(res.body)
+        l = doc.css('a').map { |link| link['href'] }
+        l
+      end
+
+      # TESTING: SPIDERS, ROBOTS, AND CRAWLERS (OWASP-IG-001)
+      def self.robots(site, only_disallow=true)
+
+        if (! site.start_with? 'http://') and (! site.start_with? 'https://')
+          site = 'http://'+site
+        end
+
+        list = []
+        begin
+          res=Net::HTTP.get_response(URI(site+'/robots.txt'))
+          if (res.code != "200")
+            return []
+          end
+
+          res.body.split("\n").each do |line|
+            if only_disallow
+              if (line.downcase.start_with?('disallow'))
+                list << line.split(":")[1].strip.chomp
+              end
+            else
+              if (line.downcase.start_with?('allow') or line.downcase.start_with?('disallow'))
+                list << line.split(":")[1].strip.chomp
+              end
+            end
+          end
+        rescue
+          return []
+        end
+
+        list
+      end
+
+      def self.follow(url, proxy)
+        l = Links::Api.links(url)
+        l[0]
+      end
+
+      def self.human(code)
+        case code.to_i
+        when 200
+          return "Open"
+        when 301
+          return "Moved"
+        when 404
+          return "Non existent"
+        when 401
+          return "Closed"
+        when 403
+          return "Forbidden"
+        when -1
+          return "No answer"
+        else
+          return "Broken"
+        end
+      end
+
+      private
+
+      def self.request(options)
+        url    = options[:url]
+        proxy  = options[:proxy]
+        method = options[:method]
+
+        begin
+          uri = URI(url)
+          if uri.scheme == 'http'
+            Net::HTTP::Proxy(proxy[:host], proxy[:port]).start(uri.host) {|http|
+              if (method == :get)
+                res = http.get(uri.request_uri)
+              else
+                res = http.head(uri.request_uri)
+              end
+              return res
+            }
+            # res = Net::HTTP.get_response(URI(url))
+          else
+            request=Net::HTTP.new(uri.host, uri.port)
+            request.use_ssl=true
+            request.verify_mode = OpenSSL::SSL::VERIFY_NONE
+            if (method == :get)
+              res = request.get(uri.request_uri)
+            else
+              res = request.head(uri.request_uri)
+            end
+
+          end
+          return res
+        rescue
+          return nil
+        end
+
+      end
+
+
+    end
+  end
+end

data/lib/codesake/links/google.rb

@@ -0,0 +1,47 @@
+require 'mechanize'
+module Codesake
+
+  module Links
+
+    # Public: it implements check described into SEARCH ENGINE DISCOVERY/RECONNAISSANCE (OWASP-IG-002)
+    #  
+    #  The idea underneath is to use a search engine like google as tool to
+    #  discovery entrypoints, web applications or whatever about the domain we
+    #  want to test for.
+    #
+    #  Please bear in mind that you **must be authorized** from the system owner
+    #  before doing any sort of security test.
+    # 
+    #  Be ethical.
+    #
+    # Usage
+    #  google = Links::Api::Google.search('somedomain.org')
+    #  google.results.each |res| do
+    #     puts "Discovered #{res}"
+    #  end
+    module Google
+
+
+      attr_reader :results
+
+      def self.search(domain) do
+
+        a = Mechanize.new { |agent|
+          agent.user_agent_alias = 'Mac Safari'
+        }
+
+        a.get('http://google.com/') do |page|
+          search_result = page.form_with(:name => 'f') do |search|
+            search.q = 'Hello world'
+          end.submit
+
+          search_result.links.each do |link|
+            puts link.text
+          end
+        end
+        return [] 
+      end
+      end
+    end
+  end
+end

data/lib/codesake/links/utils.rb

@@ -0,0 +1,26 @@
+module Codesake
+
+  module Links
+    class Utils
+
+      def self.print_str(logger, str, start, stop)
+        logger.ok "#{str} (#{((stop-start) * 1000).round} msec)\n" if str == "Open"
+        logger.err " #{str} (#{((stop-start) * 1000).round} msec)\n" if (str == "Closed" or str == "Non existent")
+        logger.warn " #{str} (#{((stop-start) * 1000).round} msec)\n" if (str != "Closed" and str != "Non existent" and str != "Open")
+
+        return
+      end
+
+      def self.print_code(logger, str, code, start, stop)
+        logger.ok "#{code} (#{((stop-start) * 1000).round} msec)\n" if str == "Open"
+        logger.err " #{code} (#{((stop-start) * 1000).round} msec)\n" if (str == "Closed" or str == "Non existent")
+        logger.warn " #{code} (#{((stop-start) * 1000).round} msec)\n" if (str != "Closed" and str != "Non existent" and str != "Open")
+
+        return
+      end
+
+
+    end
+  end
+end
+

data/lib/codesake/links/version.rb

@@ -0,0 +1,5 @@
+module Codesake
+  module Links
+    VERSION = "0.50"
+  end
+end

data/lib/codesake_links.rb

@@ -0,0 +1,4 @@
+require "codesake/links/version"
+require "codesake/links/utils"
+require "codesake/links/api"
+

data/spec/spec_helper.rb

@@ -0,0 +1 @@
+require 'links'

metadata

@@ -0,0 +1,181 @@
+--- !ruby/object:Gem::Specification
+name: codesake_links
+version: !ruby/object:Gem::Version
+  version: '0.50'
+  prerelease: 
+platform: ruby
+authors:
+- Paolo Perego
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-04-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rainbow
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rainbow
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: codesake_commons
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: During the first stage of a security test, it's useful to enumerate website
+  urls without making too much noise. Links can help in this using robots.txt or link
+  in a web page telling you the website contents.
+email:
+- paolo@armoredcode.com
+executables:
+- links
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/links
+- codesake_links.gemspec
+- lib/codesake/links/api.rb
+- lib/codesake/links/google.rb
+- lib/codesake/links/utils.rb
+- lib/codesake/links/version.rb
+- lib/codesake_links.rb
+- spec/spec_helper.rb
+homepage: http://codesake.com
+licenses:
+- BSD
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -1757904362167703742
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -1757904362167703742
+requirements: []
+rubyforge_project: links
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Fetch, discover and crawl what's available in a website.
+test_files:
+- spec/spec_helper.rb