links 0.30.0 → 0.75

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c727c4e55bc5c6ebd300a23d9dcf183d7814d774
+  data.tar.gz: 9d5c4b2fac1b72bd0cfb4ea92064642a143490f2
+SHA512:
+  metadata.gz: 0b74f49cb283654eda4214df8ecbe7cd2bda931bc640bec4880df4ce4a458e6b90c557c010e6a544ef22c1cfd65428dde339b3ec2e28cf76d728b70844e5bd13
+  data.tar.gz: 2d3f8355d3c75e3b3b50caa04bec8a6c9160616b71e4f330510e0c807d8482c52dcda73f0cc3ecfc647d36afaac7b8ffa7aae6a81632803cbfadf6646d73500e

data/.gitignore

@@ -1,3 +1,4 @@
+*.db
 .rvmrc
 *.swp
 *.gem

data/README.md

@@ -46,6 +46,6 @@ links http://www.some.org/somepage.html
 
 ## Copyright
 
-Copyright (c) 2010-2012 Paolo Perego, <thesp0nge@gmail.com>. See LICENSE for
+Copyright (c) 2010-2013 Paolo Perego, <paolo@armoredcode.com>. See LICENSE for
 further details.
 

data/bin/links

@@ -1,19 +1,71 @@
 #!/usr/bin/env ruby
-require "links"
 require "rainbow"
 require 'getoptlong'
 
+require 'anemone'
+require "codesake-commons"
+require "links"
+
+require 'data_mapper'
+require 'dm-sqlite-adapter'
+
+class Scan
+  include DataMapper::Resource
+
+  property :id,           Serial
+  property :base,         String, :length=>256, :required => true
+  property :tool,         String
+  property :version,      String
+  property :created_at,   DateTime, :default=>DateTime.now
+  property :updated_at,   DateTime, :default=>DateTime.now
+
+end
+
+class Link
+  include DataMapper::Resource
+
+  property :id,           Serial
+  property :path,         String, :length=>256, :required => true
+  property :q,            String, :length=>256
+  property :tested,       Boolean, :default=>false
+  property :created_at,   DateTime, :default=>DateTime.now
+  property :updated_at,   DateTime, :default=>DateTime.now
+
+  def self.all_dynamic
+    Link.all(:q.not => nil)
+  end
+end
+
+POST_WITHOUT_SLASH  = %r[\d{4}\/\d{2}\/[^\/]+$]
+POST_WITH_SLASH     = %r[\d{4}\/\d{2}\/[\w-]+\/$]
+ANY_POST            = Regexp.union POST_WITHOUT_SLASH, POST_WITH_SLASH
+ANY_PAGE            = %r[page\/\d+]
+ANY_PATTERN         = Regexp.union ANY_PAGE, ANY_POST
+
+APPNAME = File.basename($0)
+
+logger = Codesake::Commons::Logging.instance
+logger.toggle_syslog
+
 opts = GetoptLong.new(
   [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
   [ '--version', '-v', GetoptLong::NO_ARGUMENT ],
+  [ '--dynamic', '-d', GetoptLong::NO_ARGUMENT ],
   [ '--bulk', '-b', GetoptLong::REQUIRED_ARGUMENT ],
+  [ '--proxy', '-P',  GetoptLong::REQUIRED_ARGUMENT ],
+  [ '--crawl', '-c', GetoptLong::NO_ARGUMENT ],
   [ '--robots', '-r', GetoptLong::NO_ARGUMENT ]
 )
+trap("INT") { logger.die("[INTERRUPTED]") }
 
-trap("INT") { puts '['+'INTERRUPTED'.color(:red)+']'; exit -1 }
-list=[]
-robots=false
-bulk=false
+
+list      = []
+robots    = false
+bulk      = false
+show_code = false
+crawl     = false
+dynamic   = false
+proxy     = {:host=>nil, :port=>-1}
 
 opts.each do |opt, arg|
   case opt
@@ -21,12 +73,21 @@ opts.each do |opt, arg|
     puts "usage: links [-bvh] [filename]"
     puts "   -b filename: loads the url list from a plain text file"
     puts "   -r : parse robots.txt and make requests to disallowed urls"
+    puts "   -c : shows the return code instead of human readable answer"
+    puts "   -P host:port : connect using a proxy server. Useful in combination with Paros, Owasp Zap and other"
     puts "   -v : shows version information"
     puts "   -h : shows this help"
     exit 0
+  when '--dynamic'
+    dynamic = true
   when '--version'
-    puts "links #{Links::Version.version}"
+    puts "#{Codesake::Links::VERSION}"
     exit 0
+  when '--crawl'
+    crawl= true
+  when '--proxy'
+    proxy[:host]=arg.split(':')[0]
+    proxy[:port]=arg.split(':')[1].to_i
   when '--robots'
     robots=true
   when '--bulk'
@@ -43,65 +104,117 @@ opts.each do |opt, arg|
   end
 end
 
-target = ARGV[0]
+target = ARGV.shift
+logger.helo APPNAME, Codesake::Links::VERSION
 
-if list.count == 0
-  list<<target
-end
+db_name = URI.parse(target).host.gsub('.','_')
+DataMapper.setup(:default, "sqlite3://#{File.join(Dir.pwd, db_name)}.db")
+DataMapper.finalize
+DataMapper.auto_upgrade!
 
-if list[0].nil?
-  puts "links: missing target".color(:red)
-  exit 1
-end
+# list<<target if list.empty?
+
+logger.die("missing target") if target.nil?
+# logger.die("no -b or -r option specified") unless bulk or robots
 
 if robots
-  list = Links::Api.robots(target)
+  res = Codesake::Links::Api.robots(target)
+  list = res[:disallow_list]
+  logger.err "#{target}: no robots.txt found (#{res[:error]})\n" if res[:status] == :KO
+  logger.ok "no disallowed entries to test on #{target}" if list.empty?
+  logger.ok "found #{list.size} disallowed url(s) on #{target}" unless list.empty?
+  list.each do |l|
+    logger.ok "#{l} - #{Codesake::Links::Api.code(target+l, nil)}"
+  end
+  logger.bye
+  Kernel.exit(0)
 end
 
-list.each do |l|
-  if robots or bulk
-    if ! l.start_with? '/'
-      l = '/'+l.chomp
-    end
-    if ! target.start_with? 'http://' and ! target.start_with? 'https://'
-      #defaulting to HTTP when no protocol has been supplied
-      target = "http://"+target
+if bulk
+
+  list.each do |l|
+    unless l.start_with? "#"
+
+      l = l.chomp if l.end_with? "\n"
+      l = '/'+l unless l.start_with? '/'
+
+      url = target + l
+      start = Time.now
+      code = Codesake::Links::Api.code(url, nil)
+      stop = Time.now
+
+      str=Codesake::Links::Api.human(code)
+
+      if code == "200"
+        Codesake::Links::Utils.print_str(url, logger, str, start, stop)     unless show_code 
+        Codesake::Links::Utils.print_code(url, logger, code, start, stop)   if show_code 
+      end
+
+
+      if code == 301 or code == 302
+        start = Time.now
+        new_link = Codesake::Links::Api.follow(l, proxy)
+        stop = Time.now
+        logger.log "following from #{l} to #{new_link}\n"
+        str=Codesake::Links::Api.human(code)
+
+        Codesake::Links::Utils.print_str(logger, str, start, stop)    unless show_code
+        Codesake::Links::Utils.print_code(logger, code, start, stop)  if show_code
+
+      end
     end
 
-    #puts "T: #{target}"
-    #puts "L: #{l}"
-    print "#{target}#{l}:".color(:white)
-    code = Links::Api.human(target+l)
-  else
-    print "#{l}:".color(:white)
-    code = Links::Api.human(l)
-  end
-  case code
-  when "Open"
-    print " #{code}\n".color(:green)
-  when "Non existent"
-    print " #{code}\n".color(:red)
-  when "Closed"
-    print " #{code}\n".color(:red)
-  else
-    print " #{code}\n".color(:yellow)
   end
+end
 
-  if code == 301
-    new_link = Links::Api.follow(l)
-    printf "following from #{l} to #{new_link}\n".color(:grey)
-    code = Links::Api.human(l)
-    case code
-    when "Open"
-      print " #{code}\n".color(:green)
-    when "Non existent"
-      print " #{code}\n".color(:red)
-    when "Closed"
-      print " #{code}\n".color(:red)
-    else
-      print " #{code}\n".color(:yellow)
-    end
+if dynamic
+  list = Link.all_dynamic
+  logger.log "#{list.size} dynamic urls found during last crawl"
+  list.each do |l|
+    logger.ok "#{l.path}/#{l.q}"
+  end
+end
 
+if crawl
+  s=Scan.first(:base=>target)
+  unless s.nil?
+    s=Scan.new
+    s.base=target
+    s.tool=APPNAME
+    s.version = VERSION
+    s.save
   end
 
+  logger.log "start crawling #{target}"
+
+  Anemone.crawl(target, :redirect_limit=>2, :depth_limit => 5) do |anemone|
+    anemone.on_every_page do |page|
+      l = Link.first(:path=>page.url.path)
+      if l.nil?
+        l = Link.new
+        l.path = page.url.path
+        l.q = page.url.query
+        saved = l.save
+        logger.ok "adding #{page.url.path}" if saved
+        logger.err "error saving #{page.url.path}: #{l.errors.inspect}" unless saved
+      else
+        logger.warn "skipping #{page.url.path}"
+      end
+    end
+  end
 end
+
+# Anemone.crawl(target) do |anemone|
+#   anemone.focus_crawl do |page|
+#     page.links.keep_if { |link| link.to_s.match(ANY_PATTERN) } # crawl only links that are pages or blog posts
+#   end
+#   anemone.on_pages_like(POST_WITH_SLASH) do |page|
+#     title = page.doc.at_xpath("//div[@role='main']/header/h1").text rescue nil
+#     tag = page.doc.at_xpath("//header/div[@class='post-data']/p/a").text rescue nil
+
+#     if title and tag
+#       post = {title: title, tag: tag}
+#       logger.log "Inserting #{post.inspect}"
+#     end
+#   end
+# end

data/lib/codesake/links/api.rb

@@ -0,0 +1,129 @@
+require "net/http"
+require "nokogiri"
+
+module Codesake
+
+  module Links
+    module Api
+
+      #    include Links::Google
+
+      def self.get(url, proxy)
+        return Links::Api.request({:url=>url, :proxy=>proxy, :method=>:get})
+      end
+
+      def self.head(url, proxy)
+        return Links::Api.request({:url=>url, :proxy=>proxy, :method=>:head})
+      end
+
+      def self.code(url, proxy)
+        res = Links::Api.get(url, proxy)
+        (res.nil?)? -1 : res.code
+      end
+
+      def self.links(url, proxy)
+        res = Links::Api.get(url, proxy)
+        if res.nil?
+          return []
+        end
+        doc = Nokogiri::HTML.parse(res.body)
+        l = doc.css('a').map { |link| link['href'] }
+        l
+      end
+
+      # TESTING: SPIDERS, ROBOTS, AND CRAWLERS (OWASP-IG-001)
+      def self.robots(site)
+
+        site = 'http://'+site unless site.start_with? 'http://' or site.start_with? 'https://'
+        
+
+        allow_list = []
+        disallow_list = []
+
+        begin
+          res=Net::HTTP.get_response(URI(site+'/robots.txt'))
+          return {:status=>:KO, :allow_list=>[], :disallow_list=>[], :error=>"robots.txt response code was #{res.code}"} if (res.code != "200")
+
+
+          res.body.split("\n").each do |line|
+
+            disallow_list << line.split(":")[1].strip.chomp if (line.downcase.start_with?('disallow'))
+            allow_list << line.split(":")[1].strip.chomp if (line.downcase.start_with?('allow'))
+
+          end
+        rescue Exception => e
+          return {:status=>:KO, :allow_list=>[], :disallow_list=>[], :error=>e.message}
+        end
+
+        {:status=>:OK, :allow_list=>allow_list, :disallow_list=>disallow_list, :error=>""}
+      end
+
+      def self.follow(url, proxy)
+        l = Links::Api.links(url)
+        l[0]
+      end
+
+      def self.human(code)
+        case code.to_i
+        when 200
+          return "Open"
+        when 301
+          return "Moved"
+        when 404
+          return "Non existent"
+        when 401
+          return "Closed"
+        when 403
+          return "Forbidden"
+        when -1
+          return "No answer"
+        else
+          return "Broken"
+        end
+      end
+
+      private
+
+      def self.request(options)
+        url    = options[:url]
+        proxy  = options[:proxy]
+        method = options[:method]
+
+        begin
+          uri = URI(url)
+          if uri.scheme == 'http'
+            unless proxy.nil?
+              Net::HTTP::Proxy(proxy[:host], proxy[:port]).start(uri.host) {|http|
+                if (method == :get)
+                  res = http.get(uri.request_uri)
+                else
+                  res = http.head(uri.request_uri)
+                end
+                return res
+              }
+            else
+              res = Net::HTTP.get_response(URI(url))
+            end
+            # res = Net::HTTP.get_response(URI(url))
+          else
+            request=Net::HTTP.new(uri.host, uri.port)
+            request.use_ssl=true
+            request.verify_mode = OpenSSL::SSL::VERIFY_NONE
+            if (method == :get)
+              res = request.get(uri.request_uri)
+            else
+              res = request.head(uri.request_uri)
+            end
+
+          end
+          return res
+        rescue
+          return nil
+        end
+
+      end
+
+
+    end
+  end
+end

data/lib/codesake/links/google.rb

@@ -0,0 +1,47 @@
+require 'mechanize'
+module Codesake
+
+  module Links
+
+    # Public: it implements check described into SEARCH ENGINE DISCOVERY/RECONNAISSANCE (OWASP-IG-002)
+    #  
+    #  The idea underneath is to use a search engine like google as tool to
+    #  discovery entrypoints, web applications or whatever about the domain we
+    #  want to test for.
+    #
+    #  Please bear in mind that you **must be authorized** from the system owner
+    #  before doing any sort of security test.
+    # 
+    #  Be ethical.
+    #
+    # Usage
+    #  google = Links::Api::Google.search('somedomain.org')
+    #  google.results.each |res| do
+    #     puts "Discovered #{res}"
+    #  end
+    module Google
+
+
+      attr_reader :results
+
+      def self.search(domain) do
+
+        a = Mechanize.new { |agent|
+          agent.user_agent_alias = 'Mac Safari'
+        }
+
+        a.get('http://google.com/') do |page|
+          search_result = page.form_with(:name => 'f') do |search|
+            search.q = 'Hello world'
+          end.submit
+
+          search_result.links.each do |link|
+            puts link.text
+          end
+        end
+        return [] 
+      end
+      end
+    end
+  end
+end

data/lib/codesake/links/utils.rb

@@ -0,0 +1,24 @@
+module Codesake
+
+  module Links
+    class Utils
+
+      def self.print_str(url, logger, str, start, stop)
+        logger.ok "#{url}: #{str} (#{((stop-start) * 1000).round} msec)" if str == "Open"
+        logger.log "#{url}: #{str} (#{((stop-start) * 1000).round} msec)" unless str == "Open"
+
+        return
+      end
+
+      def self.print_code(url, logger, code, start, stop)
+        logger.ok "#{url}: #{code} (#{((stop-start) * 1000).round} msec)" if code == "200"
+        logger.warn "#{url}: #{code} (#{((stop-start) * 1000).round} msec)" unless code == "200"
+
+        return
+      end
+
+
+    end
+  end
+end
+

data/lib/codesake/links/version.rb

@@ -0,0 +1,5 @@
+module Codesake
+  module Links
+    VERSION = "0.75"
+  end
+end

data/lib/links.rb

@@ -1,2 +1,4 @@
-require "links/version"
-require "links/api"
+require "codesake/links/version"
+require "codesake/links/utils"
+require "codesake/links/api"
+

data/links.gemspec

@@ -1,13 +1,13 @@
 # -*- encoding: utf-8 -*-
 $:.push File.expand_path("../lib", __FILE__)
-require "links/version"
+require "codesake/links/version"
 
 Gem::Specification.new do |s|
   s.name        = "links"
-  s.version     = Links::Version.version
+  s.version     = Codesake::Links::VERSION
   s.authors     = ["Paolo Perego"]
-  s.email       = ["thesp0nge@gmail.com"]
-  s.homepage    = ""
+  s.email       = ["paolo@armoredcode.com"]
+  s.homepage    = "http://codesake.com"
   s.summary     = %q{Fetch, discover and crawl what's available in a website.}
   s.description = %q{During the first stage of a security test, it's useful to enumerate website urls without making too much noise. Links can help in this using robots.txt or link in a web page telling you the website contents.}
   s.license     = "BSD"
@@ -22,9 +22,13 @@ Gem::Specification.new do |s|
   # specify any dependencies here; for example:
   s.add_development_dependency "rake"
   s.add_development_dependency "rspec"
-  s.add_development_dependency "nokogiri"
-  s.add_development_dependency "rainbow"
-  # s.add_runtime_dependency "rest-client"
-  s.add_runtime_dependency "nokogiri"
-  s.add_runtime_dependency "rainbow"
+  s.add_development_dependency 'webmock'
+
+  s.add_dependency 'anemone'
+  s.add_dependency 'data_mapper'
+  s.add_dependency 'dm-sqlite-adapter'
+  s.add_dependency "nokogiri"
+  s.add_dependency "mechanize"
+
+  s.add_dependency "codesake-commons"
 end

data/spec/codesake_links_api_spec.rb

@@ -0,0 +1,60 @@
+require 'spec_helper'
+include WebMock::API
+
+describe "The API for Codesake Links" do
+  it "returns an array with a single / if the robots.txt contains only Allow: /" do
+    stub_request(:get, "http://www.test.com/robots.txt").
+      with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
+      to_return(:status=>200, :body=>"User-agent: *\nAllow: /", :headers=>{})
+
+    ret = Codesake::Links::Api.robots("http://www.test.com")
+    ret[:status].should           == :OK
+    ret[:allow_list].size.should  == 1
+    ret[:allow_list].should       == [ '/' ]
+  end
+
+  it "returns an array with a single / if the robots.txt contains only Allow: / for an HTTPS site" do
+    stub_request(:get, "http://www.test.com:443/robots.txt").
+      with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
+      to_return(:status=>200, :body=>"User-agent: *\nAllow: /", :headers=>{})
+
+    ret = Codesake::Links::Api.robots("https://www.test.com")
+    ret[:status].should           == :OK
+    ret[:allow_list].size.should  == 1
+    ret[:allow_list].should       == [ '/' ]
+  end
+
+
+  it "can handle an input without the protocol if target talks HTTP" do
+    stub_request(:get, "http://www.test.com/robots.txt").
+      with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
+      to_return(:status=>200, :body=>"User-agent: *\nAllow: /", :headers=>{})
+
+    ret = Codesake::Links::Api.robots("www.test.com")
+    ret[:status].should           == :OK
+    ret[:allow_list].size.should  == 1
+    ret[:allow_list].should       == [ '/' ]
+  end
+
+
+  it "can't handle an input without the protocol if target talks *only* HTTPS" do
+    stub_request(:get, "http://www.test.com:443/robots.txt").
+      with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
+      to_return(:status=>200, :body=>"User-agent: *\nAllow: /", :headers=>{})
+
+    ret = Codesake::Links::Api.robots("www.test.com")
+    ret[:status].should           == :KO
+  end
+
+  it "returns a list of disallowed URLs" do
+    stub_request(:get, "http://www.test.com/robots.txt").
+      with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
+      to_return(:status=>200, :body=>"User-agent: *\nAllow: /\nDisallow: /private\nDisallow: /cgi-bin\nDisallow: /a-secret-dir", :headers=>{})
+
+    ret = Codesake::Links::Api.robots("www.test.com")
+    ret[:disallow_list].size.should     == 3
+    ret[:disallow_list].should          == [ '/private', '/cgi-bin', '/a-secret-dir' ]
+
+  end
+
+end

data/spec/spec_helper.rb

@@ -1 +1,2 @@
-require 'w3ping'
+require 'links'
+require 'webmock/rspec'

metadata

@@ -1,87 +1,146 @@
 --- !ruby/object:Gem::Specification
 name: links
 version: !ruby/object:Gem::Version
-  version: 0.30.0
-  prerelease: 
+  version: '0.75'
 platform: ruby
 authors:
 - Paolo Perego
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-27 00:00:00.000000000Z
+date: 2014-09-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70331248053140 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70331248053140
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70331248052400 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70331248052400
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: nokogiri
-  requirement: &70331248051700 !ruby/object:Gem::Requirement
-    none: false
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70331248051700
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: rainbow
-  requirement: &70331248050860 !ruby/object:Gem::Requirement
-    none: false
+  name: anemone
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: data_mapper
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: dm-sqlite-adapter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
   prerelease: false
-  version_requirements: *70331248050860
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70331248050260 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70331248050260
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: rainbow
-  requirement: &70331248049560 !ruby/object:Gem::Requirement
-    none: false
+  name: codesake-commons
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70331248049560
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: During the first stage of a security test, it's useful to enumerate website
   urls without making too much noise. Links can help in this using robots.txt or link
   in a web page telling you the website contents.
 email:
-- thesp0nge@gmail.com
+- paolo@armoredcode.com
 executables:
 - links
 extensions: []
@@ -93,37 +152,38 @@ files:
 - README.md
 - Rakefile
 - bin/links
+- lib/codesake/links/api.rb
+- lib/codesake/links/google.rb
+- lib/codesake/links/utils.rb
+- lib/codesake/links/version.rb
 - lib/links.rb
-- lib/links/api.rb
-- lib/links/version.rb
 - links.gemspec
+- spec/codesake_links_api_spec.rb
 - spec/spec_helper.rb
-- spec/w3ping_spec.rb
-homepage: ''
+homepage: http://codesake.com
 licenses:
 - BSD
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: links
-rubygems_version: 1.8.10
+rubygems_version: 2.2.1
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Fetch, discover and crawl what's available in a website.
 test_files:
+- spec/codesake_links_api_spec.rb
 - spec/spec_helper.rb
-- spec/w3ping_spec.rb

data/lib/links/api.rb

@@ -1,96 +0,0 @@
-require "net/http"
-require "nokogiri"
-
-module Links
-  class Api
-
-    def self.code(url)
-      res = Links::Api.get(url)
-      (res.nil?)? -1 : res.code
-    end
-
-    def self.links(url)
-      res = Links::Api.get(url)
-      if res.nil?
-        return []
-      end
-      doc = Nokogiri::HTML.parse(res.body)
-      l = doc.css('a').map { |link| link['href'] }
-      l
-    end
-
-    def self.robots(site, only_disallow=true)
-
-      if (! site.start_with? 'http://') and (! site.start_with? 'https://')
-        site = 'http://'+site
-      end
-
-      list = []
-      begin
-        res=Net::HTTP.get_response(URI(site+'/robots.txt'))
-        if (res.code != "200")
-          return []
-        end
-
-        res.body.split("\n").each do |line|
-          if only_disallow
-            if (line.start_with?('Disallow'))
-              list << line.split(":")[1].strip.chomp
-            end
-          else
-            if (line.start_with?('Allow') or line.start_with?('Disallow'))
-              list << line.split(":")[1].strip.chomp
-            end
-          end
-        end
-      rescue
-        return []
-      end
-      
-      list
-    end
-
-    def self.follow(url)
-      l = Links::Api.links(url)
-      l[0]
-    end
-
-    def self.human(url)
-      case self.code(url).to_i
-      when 200
-        return "Open"
-      when 301
-        return "Moved"
-      when 404
-        return "Non existent"
-      when 401
-        return "Closed"
-      when 403
-        return "Forbidden"
-      when -1
-        return "No answer"
-      else
-        return "Broken"
-      end
-    end
-
-    private
-    def self.get(url)
-      begin
-        uri = URI(url)
-        if uri.scheme == 'http'
-          res = Net::HTTP.get_response(URI(url))
-        else
-          request=Net::HTTP.new(uri.host, uri.port)
-          request.use_ssl=true
-          request.verify_mode = OpenSSL::SSL::VERIFY_NONE
-          res = request.get(uri.request_uri)
-        end
-        return res
-      rescue
-        return nil
-      end
-    end
-
-  end
-end

data/lib/links/version.rb

@@ -1,16 +0,0 @@
-module Links
-  module Version
-    MAJOR = 0 
-    MINOR = 30
-    PATCH = 0
-    BUILD = ''
-
-    def self.version
-      if BUILD.empty?
-        return [MAJOR, MINOR, PATCH].compact.join('.')
-      else
-        return [MAJOR, MINOR, PATCH, BUILD].compact.join('.')
-      end
-    end
-  end
-end

data/spec/w3ping_spec.rb

@@ -1,8 +0,0 @@
-require 'spec_helper'
-
-describe W3ping do
-  it "should pass" do
-    pending
-  end
-
-end