arachni 0.2.2.1

data/lib/anemone/exceptions.rb

@@ -0,0 +1,5 @@
+module Anemone
+  class Error < ::StandardError
+    attr_accessor :wrapped_exception
+  end
+end

data/lib/anemone/http.rb

@@ -0,0 +1,144 @@
+=begin
+                  Arachni
+  Copyright (c) 2010-2011 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+
+  This is free software; you can copy and distribute and modify
+  this program under the term of the GPL v2.0 License
+  (See LICENSE file for details)
+
+=end
+
+require Arachni::Options.instance.dir['lib'] + 'anemone/page'
+require Arachni::Options.instance.dir['lib'] + 'anemone/cookie_store'
+
+
+#
+# Overides Anemone's HTTP class methods:
+#  o refresh_connection( ): added proxy support
+#  o get_response( ): upped the retry counter to 7 and generalized exception handling
+#
+# @author: Tasos "Zapotek" Laskos
+#                                      <tasos.laskos@gmail.com>
+#                                      <zapotek@segfault.gr>
+# @version: 0.1.1
+#
+module Anemone
+
+class HTTP
+
+    include Arachni::UI::Output
+
+    # Maximum number of redirects to follow on each get_response
+    REDIRECT_LIMIT = 5
+
+    # CookieStore for this HTTP client
+    attr_reader :cookie_store
+
+    def initialize(opts = {})
+      @connections = {}
+      @opts = opts
+      @cookie_store = CookieStore.new(@opts[:cookies])
+    end
+
+    #
+    # Fetch a single Page from the response of an HTTP request to *url*.
+    # Just gets the final destination page.
+    #
+    def fetch_page(url, referer = nil, depth = nil)
+      fetch_pages(url, referer, depth).last
+    end
+
+    #
+    # Create new Pages from the response of an HTTP request to *url*,
+    # including redirects
+    #
+    def fetch_pages(url, referer = nil, depth = nil)
+      begin
+        url = URI(url) unless url.is_a?(URI)
+        pages = []
+        get(url, referer) do |response, code, location, redirect_to, response_time|
+          pages << Page.new(location, :body => response.body.dup,
+                                      :code => code,
+                                      :headers => response.headers_hash,
+                                      :referer => referer,
+                                      :depth => depth,
+                                      :redirect_to => redirect_to,
+                                      :response_time => response_time)
+        end
+
+        return pages
+      rescue => e
+        if verbose?
+          puts e.inspect
+          puts e.backtrace
+        end
+        return [Page.new(url, :error => e)]
+      end
+    end
+
+    #
+    # The maximum number of redirects to follow
+    #
+    def redirect_limit
+      @opts[:redirect_limit] || REDIRECT_LIMIT
+    end
+
+    #
+    # The user-agent string which will be sent with each request,
+    # or nil if no such option is set
+    #
+    def user_agent
+      @opts[:user_agent]
+    end
+
+    #
+    # Does this HTTP client accept cookies from the server?
+    #
+    def accept_cookies?
+      @opts[:accept_cookies]
+    end
+
+    private
+
+    #
+    # Retrieve HTTP responses for *url*, including redirects.
+    # Yields the response object, response code, and URI location
+    # for each response.
+    #
+    def get(url, referer = nil)
+        response = get_response(url, referer)
+        yield response, response.code, url, '', response.time
+    end
+
+    #
+    # Get an HTTPResponse for *url*, sending the appropriate User-Agent string
+    #
+    def get_response(url, referer = nil)
+        opts = {}
+        opts['Referer'] = referer.to_s if referer
+
+        response = Arachni::HTTP.instance.get( url.to_s,
+            :headers         => opts,
+            :follow_location => true,
+            :async           => false,
+            :remove_id       => true
+        ).response
+
+        return response
+    end
+
+
+    def verbose?
+      @opts[:verbose]
+    end
+
+    #
+    # Allowed to connect to the requested url?
+    #
+    def allowed?(to_url, from_url)
+      to_url.host.nil? || (to_url.host == from_url.host)
+    end
+
+
+end
+end

data/lib/anemone/page.rb

@@ -0,0 +1,337 @@
+=begin
+                  Arachni
+  Copyright (c) 2010-2011 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+
+  This is free software; you can copy and distribute and modify
+  this program under the term of the GPL v2.0 License
+  (See LICENSE file for details)
+
+=end
+
+require 'nokogiri'
+require 'ostruct'
+require 'webrick/cookie'
+
+#
+# Overides Anemone's Page class methods:<br/>
+# o in_domain?( uri ): adding support for subdomain crawling<br/>
+# o links(): adding support for frame and iframe src URLs<br/>
+#
+# @author: Tasos "Zapotek" Laskos
+#                                      <tasos.laskos@gmail.com>
+#                                      <zapotek@segfault.gr>
+# @version: 0.1
+#
+module Anemone
+
+module Extractors
+#
+# Base Spider parser class for modules.
+#
+# The aim of such modules is to extract paths from a webpage for the Spider to follow.
+#
+#
+# @author: Tasos "Zapotek" Laskos
+#                                      <tasos.laskos@gmail.com>
+#                                      <zapotek@segfault.gr>
+# @version: 0.1
+# @abstract
+#
+class Paths
+
+    #
+    # This method must be implemented by all modules and must return an array
+    # of paths as plain strings
+    #
+    # @param    [Nokogiri]  Nokogiri document
+    #
+    # @return   [Array<String>]  paths
+    #
+    def parse( doc )
+
+    end
+end
+end
+
+class Page
+
+    include Arachni::UI::Output
+
+    # The URL of the page
+    attr_reader :url
+    # The raw HTTP response body of the page
+    attr_reader :body
+    # Headers of the HTTP response
+    attr_reader :headers
+    # URL of the page this one redirected to, if any
+    attr_reader :redirect_to
+    # Exception object, if one was raised during HTTP#fetch_page
+    attr_reader :error
+
+    # OpenStruct for user-stored data
+    attr_accessor :data
+    # Integer response code of the page
+    attr_accessor :code
+    # Boolean indicating whether or not this page has been visited in PageStore#shortest_paths!
+    attr_accessor :visited
+    # Depth of this page from the root of the crawl. This is not necessarily the
+    # shortest path; use PageStore#shortest_paths! to find that value.
+    attr_accessor :depth
+    # URL of the page that brought us to this page
+    attr_accessor :referer
+    # Response time of the request for this page in milliseconds
+    attr_accessor :response_time
+
+    #
+    # Create a new page
+    #
+    def initialize(url, params = {})
+      @url = url
+      @data = OpenStruct.new
+
+      @code = params[:code]
+      @headers = params[:headers] || {}
+      @headers['content-type'] ||= ['']
+      @aliases = Array(params[:aka]).compact
+      @referer = params[:referer]
+      @depth = params[:depth] || 0
+      @redirect_to = to_absolute(params[:redirect_to])
+      @response_time = params[:response_time]
+      @body = params[:body]
+      @error = params[:error]
+
+      @fetched = !params[:code].nil?
+    end
+
+    #
+    # Runs all Spider (path extraction) modules and returns an array of paths
+    #
+    # @return   [Array]   paths
+    #
+    def run_modules
+        opts = Arachni::Options.instance
+        require opts.dir['lib'] + 'component_manager'
+
+        lib = opts.dir['root'] + 'path_extractors/'
+
+
+        begin
+            @@manager ||= ::Arachni::ComponentManager.new( lib, Extractors )
+
+            return @@manager.available.map {
+                |name|
+                @@manager[name].new.run( doc )
+            }.flatten.uniq
+
+        rescue ::Exception => e
+            print_error( e.to_s )
+            print_debug_backtrace( e )
+        end
+    end
+
+    def dir( url )
+        URI( File.dirname( URI( url.to_s ).path ) + '/' )
+    end
+
+    #
+    # Array of distinct links to follow
+    #
+    # @return   [Array<URI>]
+    #
+    def links
+      return @links unless @links.nil?
+      @links = []
+      return @links if !doc
+
+      run_modules( ).each {
+          |path|
+          next if path.nil? or path.empty?
+          abs = to_absolute( URI( path ) ) rescue next
+
+          if in_domain?( abs )
+              @links << abs
+              # force dir listing
+              # ap to_absolute( get_path( abs.to_s ).to_s ).to_s
+              # @links << to_absolute( dir( abs.to_s ).to_s ) rescue next
+          end
+      }
+
+      @links.uniq!
+      return @links
+    end
+
+    #
+    # Nokogiri document for the HTML body
+    #
+    def doc
+      type = Arachni::HTTP.content_type( @headers )
+      return if type.is_a?( String) && !type.substring?( 'text' )
+
+      return @doc if @doc
+      @doc = Nokogiri::HTML( @body ) if @body rescue nil
+    end
+
+    #
+    # Delete the Nokogiri document and response body to conserve memory
+    #
+    def discard_doc!
+      links # force parsing of page links before we trash the document
+      @doc = @body = nil
+    end
+
+    #
+    # Was the page successfully fetched?
+    # +true+ if the page was fetched with no error, +false+ otherwise.
+    #
+    def fetched?
+      @fetched
+    end
+
+    #
+    # Array of cookies received with this page as WEBrick::Cookie objects.
+    #
+    def cookies
+      WEBrick::Cookie.parse_set_cookies(@headers['Set-Cookie']) rescue []
+    end
+
+    #
+    # The content-type returned by the HTTP request for this page
+    #
+    def content_type
+      headers['content-type'].first
+    end
+
+    #
+    # Returns +true+ if the page is a HTML document, returns +false+
+    # otherwise.
+    #
+    def html?
+      !!(content_type =~ %r{^(text/html|application/xhtml+xml)\b})
+    end
+
+    #
+    # Returns +true+ if the page is a HTTP redirect, returns +false+
+    # otherwise.
+    #
+    def redirect?
+      (300..307).include?(@code)
+    end
+
+    #
+    # Returns +true+ if the page was not found (returned 404 code),
+    # returns +false+ otherwise.
+    #
+    def not_found?
+      404 == @code
+    end
+
+    #
+    # Converts relative URL *link* into an absolute URL based on the
+    # location of the page
+    #
+    def to_absolute(link)
+      return nil if link.nil?
+
+      # remove anchor
+      link = URI.encode(link.to_s.gsub(/#[a-zA-Z0-9_-]*$/,''))
+
+      if url = base
+        base_url = URI(url)
+      else
+        base_url = @url.dup
+      end
+
+      relative = URI(link)
+      absolute = base_url.merge(relative)
+
+      absolute.path = '/' if absolute.path.empty?
+
+      return absolute
+    end
+
+    def base
+      begin
+        tmp = doc.search( '//base[@href]' )
+        return tmp[0]['href'].dup
+      rescue
+        return
+      end
+    end
+
+    #
+    # Returns +true+ if *uri* is in the same domain as the page, returns
+    # +false+ otherwise.
+    #
+    # The added code enables optional subdomain crawling.
+    #
+    def in_domain?( uri )
+        if( Arachni::Options.instance.follow_subdomains )
+            return extract_domain( uri ) ==  extract_domain( @url )
+        end
+
+        uri.host == @url.host
+    end
+
+    #
+    # Extracts the domain from a URI object
+    #
+    # @param [URI] url
+    #
+    # @return [String]
+    #
+    def extract_domain( url )
+
+        if !url.host then return false end
+
+        splits = url.host.split( /\./ )
+
+        if splits.length == 1 then return true end
+
+        splits[-2] + "." + splits[-1]
+    end
+
+
+    def marshal_dump
+      [@url, @headers, @data, @body, @links, @code, @visited, @depth, @referer, @redirect_to, @response_time, @fetched]
+    end
+
+    def marshal_load(ary)
+      @url, @headers, @data, @body, @links, @code, @visited, @depth, @referer, @redirect_to, @response_time, @fetched = ary
+    end
+
+    def to_hash
+      {'url' => @url.to_s,
+       'headers' => Marshal.dump(@headers),
+       'data' => Marshal.dump(@data),
+       'body' => @body,
+       'links' => links.map(&:to_s),
+       'code' => @code,
+       'visited' => @visited,
+       'depth' => @depth,
+       'referer' => @referer.to_s,
+       'redirect_to' => @redirect_to.to_s,
+       'response_time' => @response_time,
+       'fetched' => @fetched}
+    end
+
+    def self.from_hash(hash)
+      page = self.new(URI(hash['url']))
+      {'@headers' => Marshal.load(hash['headers']),
+       '@data' => Marshal.load(hash['data']),
+       '@body' => hash['body'],
+       '@links' => hash['links'].map { |link| URI(link) },
+       '@code' => hash['code'].to_i,
+       '@visited' => hash['visited'],
+       '@depth' => hash['depth'].to_i,
+       '@referer' => hash['referer'],
+       '@redirect_to' => URI(hash['redirect_to']),
+       '@response_time' => hash['response_time'].to_i,
+       '@fetched' => hash['fetched']
+      }.each do |var, value|
+        page.instance_variable_set(var, value)
+      end
+      page
+    end
+
+end
+end