arachni 0.2.4 → 0.3

data/lib/anemone/exceptions.rb

@@ -1,5 +0,0 @@
-module Anemone
-  class Error < ::StandardError
-    attr_accessor :wrapped_exception
-  end
-end

data/lib/anemone/http.rb

@@ -1,144 +0,0 @@
-=begin
-                  Arachni
-  Copyright (c) 2010-2011 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
-
-  This is free software; you can copy and distribute and modify
-  this program under the term of the GPL v2.0 License
-  (See LICENSE file for details)
-
-=end
-
-require Arachni::Options.instance.dir['lib'] + 'anemone/page'
-require Arachni::Options.instance.dir['lib'] + 'anemone/cookie_store'
-
-
-#
-# Overides Anemone's HTTP class methods:
-#  o refresh_connection( ): added proxy support
-#  o get_response( ): upped the retry counter to 7 and generalized exception handling
-#
-# @author: Tasos "Zapotek" Laskos
-#                                      <tasos.laskos@gmail.com>
-#                                      <zapotek@segfault.gr>
-# @version: 0.1.1
-#
-module Anemone
-
-class HTTP
-
-    include Arachni::UI::Output
-
-    # Maximum number of redirects to follow on each get_response
-    REDIRECT_LIMIT = 5
-
-    # CookieStore for this HTTP client
-    attr_reader :cookie_store
-
-    def initialize(opts = {})
-      @connections = {}
-      @opts = opts
-      @cookie_store = CookieStore.new(@opts[:cookies])
-    end
-
-    #
-    # Fetch a single Page from the response of an HTTP request to *url*.
-    # Just gets the final destination page.
-    #
-    def fetch_page(url, referer = nil, depth = nil)
-      fetch_pages(url, referer, depth).last
-    end
-
-    #
-    # Create new Pages from the response of an HTTP request to *url*,
-    # including redirects
-    #
-    def fetch_pages(url, referer = nil, depth = nil)
-      begin
-        url = URI(url) unless url.is_a?(URI)
-        pages = []
-        get(url, referer) do |response, code, location, redirect_to, response_time|
-          pages << Page.new(location, :body => response.body.dup,
-                                      :code => code,
-                                      :headers => response.headers_hash,
-                                      :referer => referer,
-                                      :depth => depth,
-                                      :redirect_to => redirect_to,
-                                      :response_time => response_time)
-        end
-
-        return pages
-      rescue => e
-        if verbose?
-          puts e.inspect
-          puts e.backtrace
-        end
-        return [Page.new(url, :error => e)]
-      end
-    end
-
-    #
-    # The maximum number of redirects to follow
-    #
-    def redirect_limit
-      @opts[:redirect_limit] || REDIRECT_LIMIT
-    end
-
-    #
-    # The user-agent string which will be sent with each request,
-    # or nil if no such option is set
-    #
-    def user_agent
-      @opts[:user_agent]
-    end
-
-    #
-    # Does this HTTP client accept cookies from the server?
-    #
-    def accept_cookies?
-      @opts[:accept_cookies]
-    end
-
-    private
-
-    #
-    # Retrieve HTTP responses for *url*, including redirects.
-    # Yields the response object, response code, and URI location
-    # for each response.
-    #
-    def get(url, referer = nil)
-        response = get_response(url, referer)
-        yield response, response.code, url, '', response.time
-    end
-
-    #
-    # Get an HTTPResponse for *url*, sending the appropriate User-Agent string
-    #
-    def get_response(url, referer = nil)
-        opts = {}
-        opts['Referer'] = referer.to_s if referer
-
-        response = Arachni::HTTP.instance.get( url.to_s,
-            :headers         => opts,
-            :follow_location => true,
-            :async           => false,
-            :remove_id       => true
-        ).response
-
-        return response
-    end
-
-
-    def verbose?
-      @opts[:verbose]
-    end
-
-    #
-    # Allowed to connect to the requested url?
-    #
-    def allowed?(to_url, from_url)
-      to_url.host.nil? || (to_url.host == from_url.host)
-    end
-
-
-end
-end

data/lib/anemone/page.rb

@@ -1,338 +0,0 @@
-=begin
-                  Arachni
-  Copyright (c) 2010-2011 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
-
-  This is free software; you can copy and distribute and modify
-  this program under the term of the GPL v2.0 License
-  (See LICENSE file for details)
-
-=end
-
-require 'nokogiri'
-require Arachni::Options.instance.dir['lib'] + 'nokogiri/xml/node'
-require 'ostruct'
-require 'webrick/cookie'
-
-#
-# Overides Anemone's Page class methods:<br/>
-# o in_domain?( uri ): adding support for subdomain crawling<br/>
-# o links(): adding support for frame and iframe src URLs<br/>
-#
-# @author: Tasos "Zapotek" Laskos
-#                                      <tasos.laskos@gmail.com>
-#                                      <zapotek@segfault.gr>
-# @version: 0.1
-#
-module Anemone
-
-module Extractors
-#
-# Base Spider parser class for modules.
-#
-# The aim of such modules is to extract paths from a webpage for the Spider to follow.
-#
-#
-# @author: Tasos "Zapotek" Laskos
-#                                      <tasos.laskos@gmail.com>
-#                                      <zapotek@segfault.gr>
-# @version: 0.1
-# @abstract
-#
-class Paths
-
-    #
-    # This method must be implemented by all modules and must return an array
-    # of paths as plain strings
-    #
-    # @param    [Nokogiri]  Nokogiri document
-    #
-    # @return   [Array<String>]  paths
-    #
-    def parse( doc )
-
-    end
-end
-end
-
-class Page
-
-    include Arachni::UI::Output
-
-    # The URL of the page
-    attr_reader :url
-    # The raw HTTP response body of the page
-    attr_reader :body
-    # Headers of the HTTP response
-    attr_reader :headers
-    # URL of the page this one redirected to, if any
-    attr_reader :redirect_to
-    # Exception object, if one was raised during HTTP#fetch_page
-    attr_reader :error
-
-    # OpenStruct for user-stored data
-    attr_accessor :data
-    # Integer response code of the page
-    attr_accessor :code
-    # Boolean indicating whether or not this page has been visited in PageStore#shortest_paths!
-    attr_accessor :visited
-    # Depth of this page from the root of the crawl. This is not necessarily the
-    # shortest path; use PageStore#shortest_paths! to find that value.
-    attr_accessor :depth
-    # URL of the page that brought us to this page
-    attr_accessor :referer
-    # Response time of the request for this page in milliseconds
-    attr_accessor :response_time
-
-    #
-    # Create a new page
-    #
-    def initialize(url, params = {})
-      @url = url
-      @data = OpenStruct.new
-
-      @code = params[:code]
-      @headers = params[:headers] || {}
-      @headers['content-type'] ||= ['']
-      @aliases = Array(params[:aka]).compact
-      @referer = params[:referer]
-      @depth = params[:depth] || 0
-      @redirect_to = to_absolute(params[:redirect_to])
-      @response_time = params[:response_time]
-      @body = params[:body]
-      @error = params[:error]
-
-      @fetched = !params[:code].nil?
-    end
-
-    #
-    # Runs all Spider (path extraction) modules and returns an array of paths
-    #
-    # @return   [Array]   paths
-    #
-    def run_modules
-        opts = Arachni::Options.instance
-        require opts.dir['lib'] + 'component_manager'
-
-        lib = opts.dir['root'] + 'path_extractors/'
-
-
-        begin
-            @@manager ||= ::Arachni::ComponentManager.new( lib, Extractors )
-
-            return @@manager.available.map {
-                |name|
-                @@manager[name].new.run( doc )
-            }.flatten.uniq
-
-        rescue ::Exception => e
-            print_error( e.to_s )
-            print_debug_backtrace( e )
-        end
-    end
-
-    def dir( url )
-        URI( File.dirname( URI( url.to_s ).path ) + '/' )
-    end
-
-    #
-    # Array of distinct links to follow
-    #
-    # @return   [Array<URI>]
-    #
-    def links
-      return @links unless @links.nil?
-      @links = []
-      return @links if !doc
-
-      run_modules( ).each {
-          |path|
-          next if path.nil? or path.empty?
-          abs = to_absolute( URI( path ) ) rescue next
-
-          if in_domain?( abs )
-              @links << abs
-              # force dir listing
-              # ap to_absolute( get_path( abs.to_s ).to_s ).to_s
-              # @links << to_absolute( dir( abs.to_s ).to_s ) rescue next
-          end
-      }
-
-      @links.uniq!
-      return @links
-    end
-
-    #
-    # Nokogiri document for the HTML body
-    #
-    def doc
-      type = Arachni::HTTP.content_type( @headers )
-      return if type.is_a?( String) && !type.substring?( 'text' )
-
-      return @doc if @doc
-      @doc = Nokogiri::HTML( @body ) if @body rescue nil
-    end
-
-    #
-    # Delete the Nokogiri document and response body to conserve memory
-    #
-    def discard_doc!
-      links # force parsing of page links before we trash the document
-      @doc = @body = nil
-    end
-
-    #
-    # Was the page successfully fetched?
-    # +true+ if the page was fetched with no error, +false+ otherwise.
-    #
-    def fetched?
-      @fetched
-    end
-
-    #
-    # Array of cookies received with this page as WEBrick::Cookie objects.
-    #
-    def cookies
-      WEBrick::Cookie.parse_set_cookies(@headers['Set-Cookie']) rescue []
-    end
-
-    #
-    # The content-type returned by the HTTP request for this page
-    #
-    def content_type
-      headers['content-type'].first
-    end
-
-    #
-    # Returns +true+ if the page is a HTML document, returns +false+
-    # otherwise.
-    #
-    def html?
-      !!(content_type =~ %r{^(text/html|application/xhtml+xml)\b})
-    end
-
-    #
-    # Returns +true+ if the page is a HTTP redirect, returns +false+
-    # otherwise.
-    #
-    def redirect?
-      (300..307).include?(@code)
-    end
-
-    #
-    # Returns +true+ if the page was not found (returned 404 code),
-    # returns +false+ otherwise.
-    #
-    def not_found?
-      404 == @code
-    end
-
-    #
-    # Converts relative URL *link* into an absolute URL based on the
-    # location of the page
-    #
-    def to_absolute(link)
-      return nil if link.nil?
-
-      # remove anchor
-      link = URI.encode(link.to_s.gsub(/#[a-zA-Z0-9_-]*$/,''))
-
-      if url = base
-        base_url = URI(url)
-      else
-        base_url = @url.dup
-      end
-
-      relative = URI(link)
-      absolute = base_url.merge(relative)
-
-      absolute.path = '/' if absolute.path.empty?
-
-      return absolute
-    end
-
-    def base
-      begin
-        tmp = doc.search( '//base[@href]' )
-        return tmp[0]['href'].dup
-      rescue
-        return
-      end
-    end
-
-    #
-    # Returns +true+ if *uri* is in the same domain as the page, returns
-    # +false+ otherwise.
-    #
-    # The added code enables optional subdomain crawling.
-    #
-    def in_domain?( uri )
-        if( Arachni::Options.instance.follow_subdomains )
-            return extract_domain( uri ) ==  extract_domain( @url )
-        end
-
-        uri.host == @url.host
-    end
-
-    #
-    # Extracts the domain from a URI object
-    #
-    # @param [URI] url
-    #
-    # @return [String]
-    #
-    def extract_domain( url )
-
-        if !url.host then return false end
-
-        splits = url.host.split( /\./ )
-
-        if splits.length == 1 then return true end
-
-        splits[-2] + "." + splits[-1]
-    end
-
-
-    def marshal_dump
-      [@url, @headers, @data, @body, @links, @code, @visited, @depth, @referer, @redirect_to, @response_time, @fetched]
-    end
-
-    def marshal_load(ary)
-      @url, @headers, @data, @body, @links, @code, @visited, @depth, @referer, @redirect_to, @response_time, @fetched = ary
-    end
-
-    def to_hash
-      {'url' => @url.to_s,
-       'headers' => Marshal.dump(@headers),
-       'data' => Marshal.dump(@data),
-       'body' => @body,
-       'links' => links.map(&:to_s),
-       'code' => @code,
-       'visited' => @visited,
-       'depth' => @depth,
-       'referer' => @referer.to_s,
-       'redirect_to' => @redirect_to.to_s,
-       'response_time' => @response_time,
-       'fetched' => @fetched}
-    end
-
-    def self.from_hash(hash)
-      page = self.new(URI(hash['url']))
-      {'@headers' => Marshal.load(hash['headers']),
-       '@data' => Marshal.load(hash['data']),
-       '@body' => hash['body'],
-       '@links' => hash['links'].map { |link| URI(link) },
-       '@code' => hash['code'].to_i,
-       '@visited' => hash['visited'],
-       '@depth' => hash['depth'].to_i,
-       '@referer' => hash['referer'],
-       '@redirect_to' => URI(hash['redirect_to']),
-       '@response_time' => hash['response_time'].to_i,
-       '@fetched' => hash['fetched']
-      }.each do |var, value|
-        page.instance_variable_set(var, value)
-      end
-      page
-    end
-
-end
-end