arachni 0.4.2 → 0.4.3

data/lib/arachni/platform.rb

@@ -0,0 +1,19 @@
+=begin
+    Copyright 2010-2013 Tasos Laskos <tasos.laskos@gmail.com>
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+=end
+
+lib = Arachni::Options.dir['lib']
+require lib + 'component/manager'
+require lib + 'platform/manager'

data/lib/arachni/platform/fingerprinter.rb

@@ -0,0 +1,100 @@
+=begin
+    Copyright 2010-2013 Tasos Laskos <tasos.laskos@gmail.com>
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+=end
+
+module Arachni
+
+module Platform
+
+# Namespace under which all platform fingerprinter components reside.
+module Fingerprinters
+end
+
+#
+# Provides utility methods for fingerprinter components as well as
+# the {Page} object to be fingerprinted
+#
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+# @abstract
+class Fingerprinter
+    include Utilities
+
+    # @return   [Page]  Page to fingerprint.
+    attr_reader :page
+
+    def initialize( page )
+        @page = page
+    end
+
+    # Executes the payload of the fingerprinter.
+    # @abstract
+    def run
+    end
+
+    # @param    [String]    string
+    # @return   [Boolean]
+    #   `true` if either {#server} or {#powered_by} include `string`,
+    #   `false` otherwise.
+    def server_or_powered_by_include?( string )
+        server.include?( string.downcase ) || powered_by.include?( string.downcase )
+    end
+
+    # @return   [Arachni::URI]  Parsed URL of the {#page}.
+    def uri
+        uri_parse( page.url )
+    end
+
+    # @return   [Hash]  URI parameters with keys and values downcased.
+    def parameters
+        @parameters ||= page.query_vars.downcase
+    end
+
+    # @return   [Hash]  Cookies as headers with keys and values downcased.
+    def cookies
+        @cookies ||= page.cookies.
+            inject({}) { |h, c| h.merge! c.simple }.downcase
+    end
+
+    # @return   [Hash]  Response headers with keys and values downcased.
+    def headers
+        @headers ||= page.response_headers.downcase
+    end
+
+    # @return   [String. nil] Value of the `X-Powered-By` header.
+    def powered_by
+        headers['x-powered-by'].to_s
+    end
+
+    # @return   [String. nil] Value of the `Server` header.
+    def server
+        headers['server'].to_s
+    end
+
+    # @return   [String] Downcased file extension of the page.
+    def extension
+        @extension ||= uri_parse( page.url ).resource_extension.to_s.downcase
+    end
+
+    # @return   [Platform]
+    #   Platform for the given page, should be updated by the
+    #   fingerprinter accordingly.
+    def platforms
+        page.platforms
+    end
+
+end
+
+end
+end

data/lib/arachni/platform/list.rb

@@ -0,0 +1,293 @@
+=begin
+    Copyright 2010-2013 Tasos Laskos <tasos.laskos@gmail.com>
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+=end
+
+module Arachni
+
+module Platform
+
+# Represents a collection of applicable platforms.
+#
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+class List
+    include Enumerable
+
+    # @param    [Array<String, Symbol>]    valid_platforms
+    #   Valid platforms for this list.
+    def initialize( valid_platforms )
+        @valid_platforms = normalize!( valid_platforms )
+        @platforms       = Set.new
+    end
+
+    # @return   [Array<Symbol>] Supported platforms.
+    def valid
+        @valid ||= Set.new( hierarchical? ? flat_list : @valid_platforms )
+    end
+
+    # Selects appropriate data depending on the applicable platforms
+    # from `data_per_platform`.
+    #
+    # @param    [Hash{<Symbol, String> => Object}]   data_per_platform
+    #   Hash with platform names as keys and arbitrary data as values.
+    #
+    # @return   [Hash]  `data_per_platform` with non-applicable entries removed.
+    # @raise    [Error::Invalid]  On {#invalid?} platforms.
+    def pick( data_per_platform )
+        orig_data_per_platform = data_per_platform.dup
+        data_per_platform      = data_per_platform.dup
+
+        data_per_platform.select! { |k, v| include? k }
+
+        # Bail out if the valid platforms are just a flat array, without hierarchy.
+        return data_per_platform if !hierarchical?
+
+        # Keep track of parents which will be removed due to the existence of
+        # their children.
+        specified_parents = []
+
+        # Remove parents if we have children.
+        data_per_platform.keys.each do |platform|
+            specified_parents |= parents = find_parents( platform )
+            data_per_platform.reject! { |k, _| parents.include? k }
+        end
+
+        # Include all of the parents' children if parents are specified but no
+        # children for them.
+
+        children = {}
+        children_for = valid & @platforms.to_a
+        children_for.each do |platform|
+            next if specified_parents.include? platform
+            c = find_children( platform )
+            children.merge! orig_data_per_platform.select { |k, _| c.include? k }
+        end
+
+        data_per_platform.merge! children
+
+        # Include the nearest parent data there is a child platform but there
+        # are no data for it.
+
+        ignore = data_per_platform.keys | specified_parents
+        orig_data_per_platform.each do |platform, data|
+            next if ignore.include?( platform ) ||
+                !include_any?( find_children( platform ) )
+            data_per_platform[platform] = data
+        end
+
+        data_per_platform
+    end
+
+    # @param    [Array<Symbol, String> Symbol, String]  platforms
+    #   Platform(s) to check.
+    # @return   [Boolean]
+    #   `true` if platforms are valid (i.e. in {#valid}), `false` otherwise.
+    # @see #invalid?
+    def valid?( platforms )
+        normalize( platforms )
+        true
+    rescue
+        false
+    end
+
+    # @param    [Array<Symbol, String> Symbol, String]  platforms
+    #   Platform(s) to check.
+    # @return   [Boolean]
+    #   `true` if platforms are invalid (i.e. not in {#valid}), `false` otherwise.
+    # @see #valid?
+    def invalid?( platforms )
+        !valid?( platforms )
+    end
+
+    # @return   [Boolean]
+    #   `true` if there are no applicable platforms, `false` otherwise.
+    def empty?
+        @platforms.empty?
+    end
+
+    # @return   [Boolean]
+    #   `true` if there are applicable platforms, `false` otherwise.
+    def any?
+        !empty?
+    end
+
+    # @param    [Symbol, String]    platform    Platform to add to the list.
+    # @return   [Platform] `self`
+    # @raise    [Error::Invalid]  On {#invalid?} platforms.
+    def <<( platform )
+        @platforms << normalize( platform )
+        self
+    end
+
+    # @param    [Platform, Enumerable] enum
+    #   Enumerable object containing platforms.
+    # @return   [Platform] Updated copy of `self`.
+    # @raise    [Error::Invalid]  On {#invalid?} platforms.
+    def merge( enum )
+        dup.merge!( enum )
+    end
+
+    # @param    [Enumerable] enum
+    #   Enumerable object containing platforms.
+    # @return   [Platform] Updated `self`.
+    # @raise    [Error::Invalid]  On {#invalid?} platforms.
+    def merge!( enum )
+        @platforms.merge normalize( enum )
+        self
+    end
+    alias update merge!
+
+    # @param    [Platform, Enumerable] enum
+    #   {Platform} or enumerable object containing platforms.
+    # @return   [Platform]
+    #   New {Platform} built by merging `self` and the elements of the
+    #   given enumerable object.
+    # @raise    [Error::Invalid]  On {#invalid?} platforms.
+    def |( enum )
+        dup.merge( enum )
+    end
+    alias + |
+
+    # @param    [Block] block   Block to be passed each platform.
+    # @return   [Enumerator, Platform]
+    #   `Enumerator` if no `block` is given, `self` otherwise.
+    def each( &block )
+        return enum_for( __method__ ) if !block_given?
+        @platforms.each( &block )
+        self
+    end
+
+    # @param    [Symbol, String]    platform    Platform to check.
+    # @return   [Boolean]
+    #   `true` if `platform` applies to the given resource, `false` otherwise.
+    # @raise    [Error::Invalid]  On {#invalid?} `platforms`.
+    def include?( platform )
+        @platforms.include? normalize( platform )
+    end
+
+    # @param    [Array<Symbol, String>]    platforms    Platform to check.
+    # @return   [Boolean]
+    #   `true` if any platform in `platforms` applies to the given resource,
+    #   `false` otherwise.
+    # @raise    [Error::Invalid]  On {#invalid?} `platforms`.
+    def include_any?( platforms )
+        (@platforms & normalize( platforms )).any?
+    end
+
+    # Clears platforms.
+    def clear
+        @platforms.clear
+    end
+
+    # @return   [Platform] Copy of `self`.
+    def dup
+        self.class.new( @valid_platforms ).tap { |p| p.platforms = @platforms }
+    end
+
+    # @return   [Boolean]
+    #   `true` if the list has a hierarchy, `false` otherwise.
+    def hierarchical?
+        @valid_platforms.is_a? Hash
+    end
+
+    def self.normalize( platforms )
+        case platforms
+            when self.class, Symbol
+                platforms
+            when String
+                platforms.to_sym
+            when Hash
+                platforms.symbolize_keys
+            when Enumerable, Array
+                platforms.to_a.flatten.map( &:to_sym ).uniq.sort
+        end
+    end
+
+    protected
+
+    def platforms=( enum )
+        @platforms = Set.new( enum )
+    end
+
+    private
+
+    # @return   [Array<Symbol>] Flat list of supported {OS operating systems}.
+    def flat_list( hash = @valid_platforms )
+        flat = []
+        hash.each do |k, v|
+            flat << k
+            flat |= flat_list( v ) if v.any?
+        end
+        flat.reject { |i| !i.is_a? Symbol }
+    end
+
+    def find_children( platform, hash = @valid_platforms )
+        return [] if hash.empty?
+
+        children = []
+        hash.each do |k, v|
+            if k == platform
+                children |= flat_list( v )
+            elsif v.is_a? Hash
+                children |= find_children( platform, v )
+            end
+
+        end
+        children
+    end
+
+    def find_parents( platform, hash = @valid_platforms )
+        return [] if hash.empty?
+
+        parents = []
+        hash.each do |k, v|
+            if v.include?( platform )
+                parents << k
+            elsif v.is_a? Hash
+                parents |= find_parents( platform, v )
+            end
+        end
+        parents
+    end
+
+    def normalize( platforms )
+        return platforms if platforms.is_a? self.class
+
+        if platforms.is_a?( Symbol ) || platforms.is_a?( String )
+            platform = normalize!( platforms )
+            if !valid.include?( platform )
+                fail Error::Invalid, "Invalid platform: #{platform}"
+            end
+
+            return platform
+        end
+
+        platforms = normalize!( platforms )
+        invalid   = (valid + platforms) - valid
+
+        if invalid.any?
+            fail Error::Invalid, "Invalid platforms: #{invalid.to_a.join( ', ' )}"
+        end
+
+        platforms
+    end
+
+    def normalize!( platforms )
+        self.class.normalize( platforms )
+    end
+
+end
+
+end
+end

data/lib/arachni/platform/manager.rb

@@ -0,0 +1,435 @@
+=begin
+    Copyright 2010-2013 Tasos Laskos <tasos.laskos@gmail.com>
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+=end
+
+require_relative 'list'
+require_relative 'fingerprinter'
+
+module Arachni
+
+module Platform
+
+#
+# {Platform} error namespace.
+#
+# All {Platform} errors inherit from and live under it.
+#
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+#
+class Error < Arachni::Error
+
+    # Raised on {Platform#invalid?} platform names.
+    #
+    # @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+    class Invalid < Error
+    end
+end
+
+#
+# Represents a collection of platform {List lists}.
+#
+# It also holds a DB of all fingerprints per URI as a class variable and
+# provides helper method for accessing and manipulating it.
+#
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+#
+class Manager
+    include Enumerable
+    include Utilities
+    extend  Utilities
+    include UI::Output
+    extend  UI::Output
+
+    TYPES = {
+        os:         'Operating systems',
+        db:         'Databases',
+        servers:    'Web servers',
+        languages:  'Programming languages',
+        frameworks: 'Frameworks'
+    }
+
+    OS = {
+        # Generic *nix, flavor couldn't be identified.
+        unix:    {
+            linux:   {},
+
+            # Generic BSD, flavor couldn't be identified.
+            bsd:     {},
+            solaris: {}
+        },
+        windows: {}
+    }
+
+    DB = [
+        :mysql,
+        :pgsql,
+        :mssql,
+        :oracle,
+        :sqlite,
+        :emc,
+        :db2,
+        :coldfusion,
+        :interbase,
+        :informix
+    ]
+
+    SERVERS = [
+        :apache,
+        :nginx,
+        :tomcat,
+        :iis,
+        :jetty
+    ]
+
+    LANGUAGES = [
+        :php,
+        :jsp,
+        :python,
+        :ruby,
+        :asp,
+        :aspx,
+        :perl
+    ]
+
+    # WebApp frameworks.
+    FRAMEWORKS = [
+        :rack
+    ]
+
+    PLATFORM_NAMES = {
+        # Operating systems
+        unix:       'Generic Unix family',
+        linux:      'Linux',
+        bsd:        'Generic BSD family',
+        solaris:    'Solaris',
+        windows:    'MS Windows',
+
+        # Databases
+        mysql:      'MySQL',
+        pgsql:      'Postgresql',
+        mssql:      'MSSQL',
+        oracle:     'Oracle',
+        sqlite:     'SQLite',
+        emc:        'EMC',
+        db2:        'DB2',
+        coldfusion: 'ColdFusion',
+        interbase:  'InterBase',
+        informix:   'Informix',
+
+        # Web servers
+        apache:     'Apache',
+        nginx:      'Nginx',
+        tomcat:     'TomCat',
+        iis:        'IIS',
+        jetty:      'Jetty',
+
+        # Programming languages
+        php:    'PHP',
+        jsp:    'JSP',
+        python: 'Python',
+        ruby:   'Ruby',
+        asp:    'ASP',
+        aspx:   'ASP.NET',
+        perl:   'Perl',
+
+        # Web frameworks
+        rack:   'Rack'
+    }
+
+    # Sets global platforms fingerprints
+    # @private
+    def self.set( platforms )
+        @platforms = platforms
+    end
+
+    # Clears global platforms DB.
+    def self.clear
+        @platforms.clear
+    end
+
+    # Empties the global platform fingerprints.
+    def self.reset
+        set Hash.new
+        @manager.clear if @manager
+        @manager = nil
+        self
+    end
+    reset
+
+    def self.fingerprinters
+        @manager ||=
+            Component::Manager.new( Options.dir['fingerprinters'],
+                                    Platform::Fingerprinters )
+    end
+    fingerprinters.load_all
+
+    # Runs all fingerprinters against the given `page`.
+    #
+    # @param    [Page]  page    Page to fingerprint.
+    # @return   [Manager]   Updated `self`.
+    def self.fingerprint( page )
+        fingerprinters.available.each do |name|
+            exception_jail( false ) do
+                fingerprinters[name].new( page ).run
+            end
+        end
+        page
+    end
+
+    #
+    # Sets platform manager for the given `uri`.
+    #
+    # @param    [String, URI]   uri
+    # @param    [Enumerable] platforms
+    #
+    # @return   [Manager]
+    # @raise    [Error::Invalid]  On {#invalid?} platforms.
+    def self.[]=( uri, platforms )
+        @platforms[make_key( uri )] =
+            platforms.is_a?( self ) ? platforms : new( platforms )
+    end
+
+    #
+    # Updates the `platforms` for the given `uri`.
+    #
+    # @param    [String, URI]   uri
+    # @param    [Manager] platforms
+    #
+    # @return   [Manager] Updated manager.
+    # @raise    [Error::Invalid]  On {#invalid?} platforms.
+    def self.update( uri, platforms )
+        self[uri].update platforms
+    end
+
+    # @param    [String, URI]   uri
+    # @return   [Manager] Platform for the given `uri`
+    def self.[]( uri )
+        @platforms[make_key( uri )] ||= new
+    end
+
+    # @return   [Boolean]
+    #   `true` if there are no platforms fingerprints, `false` otherwise.
+    def self.empty?
+        @platforms.empty?
+    end
+
+    # @return   [Boolean]
+    #   `true` if there are platforms fingerprints, `false` otherwise.
+    def self.any?
+        !empty?
+    end
+
+    # @return   [Hash<Integer, Platform>]
+    #   Platform per {URI#persistent_hash hashed URL}.
+    def self.all
+        @platforms
+    end
+
+    # @return   [Hash{Integer=>Array<Symbol>}]
+    #   Light representation of the fingerprint DB with URL hashes as keys
+    #   and arrays of symbols for platforms as values.
+    def self.light
+        all.inject({}) { |h, (k, v)| h[k] = v.to_a; h }
+    end
+
+    # @param    [Hash{Integer=>Array<Symbol>}]   light_platforms
+    #   Return value of {.light}.
+    # @return   [Manager]
+    def self.update_light( light_platforms )
+        light_platforms.each do |url, platforms|
+            @platforms[url] ||= new( platforms )
+        end
+        self
+    end
+
+    # @param    [Array<String, Symbol>] platforms
+    #   Platforms with which to initialize the lists.
+    def initialize( platforms = [] )
+        @platforms = {}
+        TYPES.keys.each do |type|
+            @platforms[type] =
+                List.new( self.class.const_get( type.to_s.upcase.to_sym ) )
+        end
+
+        update [platforms | Options.platforms].flatten.compact
+    end
+
+    # @!method os
+    #   @return [List] Platform list for operating systems.
+    #   @see OS
+
+    # @!method db
+    #   @return [List] Platform list for databases.
+    #   @see DB
+
+    # @!method servers
+    #   @return [List] Platform list for web servers.
+    #   @see SERVERS
+
+    # @!method languages
+    #   @return [List] Platform list for languages.
+    #   @see LANGUAGES
+
+    # @!method frameworks
+    #   @return [List] Platform list for frameworks.
+    #   @see FRAMEWORKS
+
+    [:os, :db, :servers, :languages, :frameworks].each do |type|
+        define_method type do
+            @platforms[type]
+        end
+    end
+
+    # Converts a platform shortname to a full name.
+    #
+    # @param    [String, Symbol]   platform
+    #   Platform shortname.
+    #
+    # @return   [String]    Full name.
+    # @raise    [Error::Invalid]  On {#invalid?} platforms.
+    def fullname( platform )
+        PLATFORM_NAMES[normalize( platform )]
+    end
+
+    # Selects appropriate data, depending on the applicable platforms,
+    # from `data_per_platform`.
+    #
+    # @param    [Hash{<Symbol, String> => Object}]   data_per_platform
+    #   Hash with platform names as keys and arbitrary data as values.
+    #
+    # @return   [Hash]
+    #   `data_per_platform` with non-applicable entries (for non-empty platform
+    #   lists) removed. Data for platforms whose list is empty will not be removed.
+    # @raise    [Error::Invalid]  On {#invalid?} platforms.
+    def pick( data_per_platform )
+        data_per_list = {}
+        data_per_platform.each do |platform, value|
+            list = find_list( platform )
+            data_per_list[list]           ||= {}
+            data_per_list[list][platform]   = value
+        end
+
+        picked = {}
+        data_per_list.each do |list, data|
+            # If a platform list is empty pass the given data without picking...
+            if list.empty?
+                picked.merge! data
+                next
+            end
+
+            # ...otherwise enforce its platform restrictions.
+            picked.merge! list.pick( data )
+        end
+
+        picked
+    end
+
+    # @return   [Set<Symbol>]   List of valid platforms.
+    def valid
+        @valid ||= Set.new( @platforms.map { |_, p| p.valid.to_a }.flatten )
+    end
+
+    # @param    [Symbol, String]  platform Platform to check.
+    # @return   [Boolean]
+    #   `true` if platform is valid (i.e. in {#valid}), `false` otherwise.
+    # @see #invalid?
+    def valid?( platform )
+        valid.include? platform
+    end
+
+    # @param    [Symbol, String]  platform Platform to check.
+    # @return   [Boolean]
+    #   `true` if platform is invalid (i.e. not in {#valid}), `false` otherwise.
+    # @see #invalid?
+    def invalid?( platform )
+        !valid?( platform )
+    end
+
+    # @param    [Block] block   Block to be passed each platform.
+    # @return   [Enumerator, Manager]
+    #   `Enumerator` if no `block` is given, `self` otherwise.
+    def each( &block )
+        return enum_for( __method__ ) if !block_given?
+        @platforms.map { |_, p| p.to_a }.flatten.each( &block )
+        self
+    end
+
+    # @param    [Symbol, String]    platform    Platform to check.
+    # @return   [Boolean]
+    #   `true` if one of the lists contains the `platform`, `false` otherwise.
+    # @raise    [Error::Invalid]  On {#invalid?} `platforms`.
+    def include?( platform )
+        find_list( platform ).include?( platform )
+    end
+
+    # @return   [Boolean]
+    #   `true` if there are no applicable platforms, `false` otherwise.
+    def empty?
+        !@platforms.map { |_, p| p.empty? }.include?( false )
+    end
+
+    # @return   [Boolean]
+    #   `true` if there are applicable platforms, `false` otherwise.
+    def any?
+        !empty?
+    end
+
+    # @param    [Enumerable] enum Enumerable object containing platforms.
+    # @return   [Manager] Updated `self`.
+    # @raise    [Error::Invalid]  On {#invalid?} platforms.
+    def update( enum )
+        enum.each { |p| self << p }
+        self
+    end
+
+    # @param    [Symbol, String]    platform
+    #   Platform to add to the appropriate list.
+    # @return   [Manager] `self`
+    # @raise    [Error::Invalid]  On {#invalid?} platforms.
+    def <<( platform )
+        find_list( platform ) << platform
+        self
+    end
+
+    # @param    [String, Symbol]    platform
+    #   Platform whose type to find
+    # @return   [Symbol]    Platform type.
+    def find_type( platform )
+        platform = normalize( platform )
+        @platforms.each { |type, list| return type if list.valid? platform }
+        nil
+    end
+
+    # @param    [String, Symbol]    platform Platform whose list to find.
+    # @return   [List]    Platform list.
+    def find_list( platform )
+        @platforms[find_type( normalize( platform ) )]
+    end
+
+    private
+
+    def normalize( platform )
+        platform = List.normalize( platform )
+        fail Error::Invalid, "Invalid platform: #{platform}" if invalid?( platform )
+        platform
+    end
+
+    def self.make_key( uri )
+        Arachni::URI( uri ).without_query.persistent_hash
+    end
+
+end
+end
+end