links 0.20.0

data/.gitignore

@@ -0,0 +1,6 @@
+.rvmrc
+*.swp
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in w3ping.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2010-2012 Paolo Perego
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,51 @@
+# links
+
+## Introduction
+
+[links](https://github.com/thesp0nge/links) is a tool for discovering a website
+available pages without making too much noise.
+
+The idea came to me during a penetration test since I had a bulk list of URLs
+to check for availability and I wanted to automate this process.
+
+## Installing links
+
+Installing links is easy. You can always obtain the latest stable code by using the following command: 
+
+``` 
+gem install links
+```
+
+If you want to install a _pre_ release, such as a _release candidate_ you can do it this way:
+```
+gem install links --pre
+```
+
+## Using links
+
+After you installed links gem, you have the links command you can use this way:
+
+```
+links http://www.some.org/somepage.html
+```
+
+## Contributing to links
+ 
+* Check out the latest master to make sure the feature hasn't been implemented
+  or the bug hasn't been fixed yet
+* Check out the issue tracker to make sure someone already hasn't requested it
+  and/or contributed it
+* Fork the project
+* Start a feature/bugfix branch
+* Commit and push until you are happy with your contribution
+* Make sure to add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to
+  have your own version, or is otherwise necessary, that is fine, but please
+  isolate to its own commit so I can cherry-pick around it.
+
+## Copyright
+
+Copyright (c) 2010-2012 Paolo Perego, <thesp0nge@gmail.com>. See LICENSE for
+further details.
+

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new
+
+task :default => :spec
+task :test => :spec
+

data/bin/links

@@ -0,0 +1,94 @@
+#!/usr/bin/env ruby
+require "links"
+require "rainbow"
+require 'getoptlong'
+
+opts = GetoptLong.new(
+  [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
+  [ '--version', '-v', GetoptLong::NO_ARGUMENT ],
+  [ '--bulk', '-b', GetoptLong::REQUIRED_ARGUMENT ],
+  [ '--robots', '-r', GetoptLong::NO_ARGUMENT ]
+)
+
+trap("INT") { puts '['+'INTERRUPTED'.color(:red)+']'; exit -1 }
+list=[]
+robots=false
+
+opts.each do |opt, arg|
+  case opt
+  when '--help'
+    puts "usage: links [-bvh] [filename]"
+    puts "   -b filename: loads the url list from a plain text file"
+    puts "   -r : parse robots.txt and make requests to disallowed urls"
+    puts "   -v : shows version information"
+    puts "   -h : shows this help"
+    exit 0
+  when '--version'
+    puts "links #{Links::Version.version}"
+    exit 0
+  when '--robots'
+    robots=true
+  when '--bulk'
+    if ! File.exists?(arg)
+      puts "links: file not found (#{arg})".color(:red)
+      exit 1
+    end
+    list = File.open(arg).readlines
+    if list.count <= 0
+      puts "links: invalid url list".color(:red)
+      exit 1
+    end
+  end
+end
+
+target = ARGV[0]
+
+if list.count == 0
+  list<<target
+end
+
+if list[0].nil?
+  puts "links: missing target".color(:red)
+  exit 1
+end
+
+if robots
+  list = Links::Api.robots(target)
+end
+
+list.each do |l|
+  print "#{l}:".color(:white)
+  if robots
+    code = Links::Api.human('http://'+target+l)
+  else
+    code = Links::Api.human(l)
+  end
+  case code
+  when "Open"
+    print " #{code}\n".color(:green)
+  when "Non existent"
+    print " #{code}\n".color(:red)
+  when "Closed"
+    print " #{code}\n".color(:red)
+  else
+    print " #{code}\n".color(:yellow)
+  end
+
+  if code == 301
+    new_link = Links::Api.follow(l)
+    printf "following from #{l} to #{new_link}\n".color(:grey)
+    code = Links::Api.human(l)
+    case code
+    when "Open"
+      print " #{code}\n".color(:green)
+    when "Non existent"
+      print " #{code}\n".color(:red)
+    when "Closed"
+      print " #{code}\n".color(:red)
+    else
+      print " #{code}\n".color(:yellow)
+    end
+
+  end
+
+end

data/lib/links/api.rb

@@ -0,0 +1,85 @@
+require "net/http"
+require "nokogiri"
+
+module Links
+  class Api
+
+    def self.code(url)
+      res = Links::Api.get(url)
+      res.code ||= -1
+    end
+
+    def self.links(url)
+      res = Links::Api.get(url)
+      if res.nil?
+        return []
+      end
+      doc = Nokogiri::HTML.parse(res.body)
+      l = doc.css('a').map { |link| link['href'] }
+      l
+    end
+
+    def self.robots(site, only_disallow=true)
+
+      if (! site.start_with? 'http://') and (! site.start_with? 'https://')
+        site = 'http://'+site
+      end
+
+      list = []
+      begin
+        res=Net::HTTP.get_response(URI(site+'/robots.txt'))
+        if (res.code != "200")
+          return []
+        end
+
+        res.body.split("\n").each do |line|
+          if only_disallow
+            if (line.start_with?('Disallow'))
+              list << line.split(":")[1].strip.chomp
+            end
+          else
+            if (line.start_with?('Allow') or line.start_with?('Disallow'))
+              list << line.split(":")[1].strip.chomp
+            end
+          end
+        end
+      rescue
+        return []
+      end
+      
+      list
+    end
+
+    def self.follow(url)
+      l = Links::Api.links(url)
+      l[0]
+    end
+
+    def self.human(url)
+      case self.code(url).to_i
+      when 200
+        return "Open"
+      when 301
+        return "Moved"
+      when 404
+        return "Non existent"
+      when 401
+        return "Closed"
+      when 403
+        return "Forbidden"
+      else
+        return "Broken"
+      end
+    end
+
+    private
+    def self.get(url)
+      begin
+        Net::HTTP.get_response(URI(url))
+      rescue
+        return nil
+      end
+    end
+
+  end
+end

data/lib/links/version.rb

@@ -0,0 +1,16 @@
+module Links
+  module Version
+    MAJOR = 0 
+    MINOR = 20
+    PATCH = 0
+    BUILD = ''
+
+    def self.version
+      if BUILD.empty?
+        return [MAJOR, MINOR, PATCH].compact.join('.')
+      else
+        return [MAJOR, MINOR, PATCH, BUILD].compact.join('.')
+      end
+    end
+  end
+end

data/lib/links.rb

@@ -0,0 +1,2 @@
+require "links/version"
+require "links/api"

data/links.gemspec

@@ -0,0 +1,30 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "links/version"
+
+Gem::Specification.new do |s|
+  s.name        = "links"
+  s.version     = Links::Version.version
+  s.authors     = ["Paolo Perego"]
+  s.email       = ["thesp0nge@gmail.com"]
+  s.homepage    = ""
+  s.summary     = %q{Fetch, discover and crawl what's available in a website.}
+  s.description = %q{During the first stage of a security test, it's useful to enumerate website urls without making too much noise. Links can help in this using robots.txt or link in a web page telling you the website contents.}
+  s.license     = "BSD"
+
+  s.rubyforge_project = "links"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  # specify any dependencies here; for example:
+  s.add_development_dependency "rake"
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "nokogiri"
+  s.add_development_dependency "rainbow"
+  # s.add_runtime_dependency "rest-client"
+  s.add_runtime_dependency "nokogiri"
+  s.add_runtime_dependency "rainbow"
+end

data/spec/spec_helper.rb

@@ -0,0 +1 @@
+require 'w3ping'

data/spec/w3ping_spec.rb

@@ -0,0 +1,8 @@
+require 'spec_helper'
+
+describe W3ping do
+  it "should pass" do
+    pending
+  end
+
+end

metadata

@@ -0,0 +1,129 @@
+--- !ruby/object:Gem::Specification
+name: links
+version: !ruby/object:Gem::Version
+  version: 0.20.0
+  prerelease: 
+platform: ruby
+authors:
+- Paolo Perego
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-02-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &70178993924920 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70178993924920
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &70178998811440 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70178998811440
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: &70178998810960 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70178998810960
+- !ruby/object:Gem::Dependency
+  name: rainbow
+  requirement: &70178998810280 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70178998810280
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: &70178998809620 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70178998809620
+- !ruby/object:Gem::Dependency
+  name: rainbow
+  requirement: &70178998808840 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70178998808840
+description: During the first stage of a security test, it's useful to enumerate website
+  urls without making too much noise. Links can help in this using robots.txt or link
+  in a web page telling you the website contents.
+email:
+- thesp0nge@gmail.com
+executables:
+- links
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/links
+- lib/links.rb
+- lib/links/api.rb
+- lib/links/version.rb
+- links.gemspec
+- spec/spec_helper.rb
+- spec/w3ping_spec.rb
+homepage: ''
+licenses:
+- BSD
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: links
+rubygems_version: 1.8.17
+signing_key: 
+specification_version: 3
+summary: Fetch, discover and crawl what's available in a website.
+test_files:
+- spec/spec_helper.rb
+- spec/w3ping_spec.rb