arachni 1.4 → 1.6.0

data/lib/arachni/uri/scope.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -43,8 +43,11 @@ class Scope < Arachni::Scope
     # @see OptionGroups::Scope#exclude_path_patterns
     # @see #exclude_file_extension?
     def exclude?
-        exclude_file_extension? ||
-            !!options.exclude_path_patterns.find { |pattern| @url.to_s =~ pattern }
+        return true  if exclude_file_extension?
+        return false if options.exclude_path_patterns.empty?
+
+        s = @url.to_s
+        !!options.exclude_path_patterns.find { |pattern| s =~ pattern }
     end
 
     # @return   [Bool]
@@ -53,7 +56,10 @@ class Scope < Arachni::Scope
     #
     # @see OptionGroups::Scope#@exclude_file_extensions
     def exclude_file_extension?
-        options.exclude_file_extensions.include? @url.resource_extension.to_s.downcase
+        options.exclude_file_extensions.any? &&
+            options.exclude_file_extensions.include?(
+                @url.resource_extension.to_s.downcase
+            )
     end
 
     # @return   [Bool]
@@ -75,10 +81,9 @@ class Scope < Arachni::Scope
     def in_domain?
         return true if !Options.url
 
-        reference = Arachni::URI( Options.url )
-
         options.include_subdomains ?
-            reference.domain == @url.domain : reference.host == @url.host
+            Options.parsed_url.domain == @url.domain :
+            Options.parsed_url.host == @url.host
     end
 
     # @return   [Bool]
@@ -89,14 +94,11 @@ class Scope < Arachni::Scope
     def follow_protocol?
         return true if !Options.url
 
-        check_scheme = @url.scheme.to_s
-
-        return false if check_scheme != 'http' && check_scheme != 'https'
+        check_scheme = @url.scheme
 
-        parsed_ref = Arachni::URI( Options.url )
-        return false if !parsed_ref
+        return false if !check_scheme
 
-        ref_scheme = parsed_ref.scheme
+        ref_scheme = Options.parsed_url.scheme
 
         return true if ref_scheme != 'https'
         return true if ref_scheme == check_scheme

data/lib/arachni/uri.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -47,12 +47,15 @@ class URI
     end
 
     CACHE_SIZES = {
-        parse:       5_000,
-        fast_parse:  5_000,
-        encode:      10_000,
-        decode:      10_000,
-        normalize:   10_000,
-        to_absolute: 10_000
+        parse:       2_500,
+
+        normalize:   2_500,
+        to_absolute: 2_500,
+
+        encode:      1_000,
+        decode:      1_000,
+
+        scope:       1_000
     }
 
     CACHE = {
@@ -64,6 +67,10 @@ class URI
 
     QUERY_CHARACTER_CLASS = Addressable::URI::CharacterClasses::QUERY.sub( '\\&', '' )
 
+    VALID_SCHEMES     = Set.new(%w(http https))
+    PARTS             = %w(scheme userinfo host port path query)
+    TO_ABSOLUTE_PARTS = %w(scheme userinfo host port)
+
     class <<self
 
         # @return [URI::Parser] cached URI parser
@@ -82,7 +89,12 @@ class URI
         #   Encoded string.
         def encode( string, good_characters = nil )
             CACHE[__method__].fetch [string, good_characters] do
-                Addressable::URI.encode_component( *[string, good_characters].compact )
+                s = Addressable::URI.encode_component(
+                    *[string, good_characters].compact
+                )
+                s.recode!
+                s.gsub!( '+', '%2B' )
+                s
             end
         end
 
@@ -92,7 +104,11 @@ class URI
         #
         # @return   [String]
         def decode( string )
-            CACHE[__method__].fetch( string ) { Addressable::URI.unencode( string ) }
+            CACHE[__method__].fetch( string ) do
+                s = Addressable::URI.unencode( string.gsub( '+', '%20' ) )
+                s.recode! if s
+                s
+            end
         end
 
         # @note This method's results are cached for performance reasons.
@@ -118,10 +134,6 @@ class URI
             end
         end
 
-        # @note This method's results are cached for performance reasons.
-        #   If you plan on doing something destructive with its return value
-        #   duplicate it first because there may be references to it elsewhere.
-        #
         # Performs a parse that is less resource intensive than Ruby's URI lib's
         # method while normalizing the URL (will also discard the fragment and
         # path parameters).
@@ -139,10 +151,11 @@ class URI
         #     * `:query`
         def fast_parse( url )
             return if !url || url.empty?
-            return if url.downcase.start_with?( 'javascript:' ) ||
-                url.start_with?( '#' )
+            return if url.start_with?( '#' )
 
-            cache = CACHE[__method__]
+            durl = url.downcase
+            return if durl.start_with?( 'javascript:' ) ||
+                durl.start_with?( 'data:' )
 
             # One to rip apart.
             url = url.dup
@@ -162,20 +175,12 @@ class URI
                 query:    nil
             }
 
-            valid_schemes = %w(http https)
-
             begin
-                if (v = cache[url]) && v == :err
-                    return
-                elsif v
-                    return v
-                end
-
                 # Parsing the URL in its schemeless form is trickier, so we
                 # fake it, pass a valid scheme to get through the parsing and
                 # then remove it at the other end.
                 if (schemeless = url.start_with?( '//' ))
-                    url = "http:#{url}"
+                    url.insert 0, 'http:'
                 end
 
                 # url.recode!
@@ -185,14 +190,16 @@ class URI
                 has_path = true
 
                 splits = url.split( ':' )
-                if !splits.empty? && valid_schemes.include?( splits.first.downcase )
+                if !splits.empty? && VALID_SCHEMES.include?( splits.first.downcase )
 
                     splits = url.split( '://', 2 )
                     components[:scheme] = splits.shift
                     components[:scheme].downcase! if components[:scheme]
 
                     if (url = splits.shift)
-                        userinfo_host, url = url.to_s.split( '?' ).first.to_s.split( '/', 2 )
+                        userinfo_host, url =
+                            url.to_s.split( '?' ).first.to_s.split( '/', 2 )
+
                         url    = url.to_s
                         splits = userinfo_host.to_s.split( '@', 2 )
 
@@ -265,20 +272,12 @@ class URI
 
                 components[:path] ||= components[:scheme] ? '/' : nil
 
-                components.values.each(&:freeze)
-
-                cache[c_url] = components.freeze
+                components
             rescue => e
-                ap e
-                ap e.backtrace
-                ap c_url
-                ap url
-
                 print_debug "Failed to parse '#{c_url}'."
                 print_debug "Error: #{e}"
                 print_debug_backtrace( e )
 
-                cache[c_url] = :err
                 nil
             end
         end
@@ -335,8 +334,8 @@ class URI
         #   If you plan on doing something destructive with its return value
         #   duplicate it first because there may be references to it elsewhere.
         #
-        # Uses {.fast_parse} to parse and normalize the URL and then converts
-        # it to a common {String} format.
+        # Uses {.parse} to parse and normalize the URL and then converts it to
+        # a common {String} format.
         #
         # @param    [String]    url
         #
@@ -407,17 +406,13 @@ class URI
 
     # @note Will discard the fragment component, if there is one.
     #
-    # {.normalize Normalizes} and parses the provided URL.
-    #
     # @param    [String]    url
-    #   {String} URL to parse, `URI` to convert, or a `Hash` holding URL components
-    #   (for `URI::Generic.build`). Also accepts {Arachni::URI} for convenience.
     def initialize( url )
         @data = self.class.fast_parse( url )
 
         fail Error, 'Failed to parse URL.' if !@data
 
-        %w(scheme userinfo host port path query).each do |part|
+        PARTS.each do |part|
             instance_variable_set( "@#{part}", @data[part.to_sym] )
         end
 
@@ -426,7 +421,9 @@ class URI
 
     # @return   [Scope]
     def scope
-        @scope ||= Scope.new( self )
+        # We could have several identical URLs in play at any given time and
+        # they will all have the same scope.
+        CACHE[:scope].fetch( self ){ Scope.new( self ) }
     end
 
     def ==( other )
@@ -454,7 +451,7 @@ class URI
             reference = self.class.new( reference.to_s )
         end
 
-        %w(scheme userinfo host port).each do |part|
+        TO_ABSOLUTE_PARTS.each do |part|
             next if send( part )
 
             ref_part = reference.send( "#{part}" )
@@ -538,13 +535,13 @@ class URI
     # @return   [String]
     #   The URL up to its resource component (query, fragment, etc).
     def without_query
-        to_s.split( '?', 2 ).first.to_s
+        @without_query ||= to_s.split( '?', 2 ).first.to_s
     end
 
     # @return   [String]
     #   Name of the resource.
     def resource_name
-        path.split( '/' ).last
+        @resource_name ||= path.split( '/' ).last
     end
 
     # @return   [String, nil]
@@ -553,45 +550,53 @@ class URI
         name = resource_name.to_s
         return if !name.include?( '.' )
 
-        name.split( '.' ).last
+        @resource_extension ||= name.split( '.' ).last
     end
 
     # @return   [String]
     #   The URL up to its path component (no resource name, query, fragment, etc).
     def up_to_path
         return if !path
-        uri_path = path.dup
 
-        uri_path = File.dirname( uri_path ) if !File.extname( path ).empty?
+        @up_to_path ||= begin
+            uri_path = path.dup
+            uri_path = File.dirname( uri_path ) if !File.extname( path ).empty?
 
-        uri_path << '/' if uri_path[-1] != '/'
+            uri_path << '/' if uri_path[-1] != '/'
 
-        up_to_port << uri_path
+            up_to_port + uri_path
+        end
     end
 
     # @return   [String]
     #   Scheme, host & port only.
     def up_to_port
-        uri_str = "#{scheme}://#{host}"
+        @up_to_port ||= begin
+            uri_str = "#{scheme}://#{host}"
+
+            if port && (
+                (scheme == 'http' && port != 80) ||
+                    (scheme == 'https' && port != 443)
+            )
+                uri_str << ':' + port.to_s
+            end
 
-        if port && ((scheme == 'http' && port != 80) || (scheme == 'https' && port != 443))
-            uri_str << ':' + port.to_s
+            uri_str
         end
-
-        uri_str
     end
 
     # @return [String]
     #   `domain_name.tld`
     def domain
         return if !host
-        return host if ip_address?
+        return @domain if @domain
+        return @domain = host if ip_address?
 
         s = host.split( '.' )
-        return s.first if s.size == 1
-        return host if s.size == 2
+        return @domain = s.first if s.size == 1
+        return @domain = host    if s.size == 2
 
-        s[1..-1].join( '.' )
+        @domain = s[1..-1].join( '.' )
     end
 
     # @param    [Hash<Regexp => String>]    rules
@@ -622,6 +627,10 @@ class URI
     end
 
     def query=( q )
+        @to_s             = nil
+        @without_query    = nil
+        @query_parameters = nil
+
         q = q.to_s
         q = nil if q.empty?
 
@@ -634,14 +643,20 @@ class URI
         q = self.query
         return {} if q.to_s.empty?
 
-        q.split( '&' ).inject( {} ) do |h, pair|
-            name, value = pair.split( '=', 2 )
-            h[::URI.decode( name.to_s )] = ::URI.decode( value.to_s )
-            h
+        @query_parameters ||= begin
+            q.split( '&' ).inject( {} ) do |h, pair|
+                name, value = pair.split( '=', 2 )
+                h[::URI.decode_www_form_component( name.to_s )] =
+                  ::URI.decode_www_form_component( value.to_s )
+                h
+            end
         end
     end
 
     def userinfo=( ui )
+        @without_query = nil
+        @to_s          = nil
+
         @userinfo = ui
     ensure
         reset_userpass
@@ -664,6 +679,9 @@ class URI
     end
 
     def port=( p )
+        @without_query = nil
+        @to_s          = nil
+
         if p
             @port = p.to_i
         else
@@ -676,6 +694,11 @@ class URI
     end
 
     def host=( h )
+        @to_s          = nil
+        @up_to_port    = nil
+        @without_query = nil
+        @domain        = nil
+
         @host = h
     end
 
@@ -684,6 +707,12 @@ class URI
     end
 
     def path=( p )
+        @up_to_path         = nil
+        @resource_name      = nil
+        @resource_extension = nil
+        @without_query      = nil
+        @to_s               = nil
+
         @path = p
     end
 
@@ -692,48 +721,59 @@ class URI
     end
 
     def scheme=( s )
+        @up_to_port    = nil
+        @without_query = nil
+        @to_s          = nil
+
         @scheme = s
     end
 
     # @return   [String]
     def to_s
-        s = ''
+        @to_s ||= begin
+            s = ''
 
-        if @scheme
-            s << @scheme
-            s << '://'
-        end
+            if @scheme
+                s << @scheme
+                s << '://'
+            end
 
-        if @userinfo
-            s << @userinfo
-            s << '@'
-        end
+            if @userinfo
+                s << @userinfo
+                s << '@'
+            end
 
-        if @host
-            s << @host
+            if @host
+                s << @host
 
-            if @port
-                if (@scheme == 'http' && @port != 80) ||
-                    (@scheme == 'https' && @port != 443)
+                if @port
+                    if (@scheme == 'http' && @port != 80) ||
+                        (@scheme == 'https' && @port != 443)
 
-                    s << ':'
-                    s << @port.to_s
+                        s << ':'
+                        s << @port.to_s
+                    end
                 end
             end
-        end
 
-        s << @path.to_s
+            s << @path.to_s
 
-        if @query
-            s << '?'
-            s << @query
-        end
+            if @query
+                s << '?'
+                s << @query
+            end
 
-        s
+            s
+        end
     end
 
     def dup
-        self.class.new( to_s )
+        i = self.class.allocate
+        instance_variables.each do |iv|
+            next if !(v = instance_variable_get( iv ))
+            i.instance_variable_set iv, (v.dup rescue v)
+        end
+        i
     end
 
     def _dump( _ )
@@ -762,19 +802,5 @@ class URI
         end
     end
 
-    # Delegates unimplemented methods to Ruby's `URI::Generic` class for
-    # compatibility.
-    # def method_missing( sym, *args, &block )
-    #     if @data.respond_to?( sym )
-    #         @parsed_url.send( sym, *args, &block )
-    #     else
-    #         super
-    #     end
-    # end
-    #
-    # def respond_to?( *args )
-    #     super || @parsed_url.respond_to?( *args )
-    # end
-
 end
 end

data/lib/arachni/utilities.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -39,9 +39,9 @@ module Utilities
         Form.from_response( *args )
     end
 
-    # @see Arachni::Element::Form.from_document
-    def forms_from_document( *args )
-        Form.from_document( *args )
+    # @see Arachni::Element::Form.from_parser
+    def forms_from_parser( *args )
+        Form.from_parser(*args )
     end
 
     # @see Arachni::Element::Form.encode
@@ -64,9 +64,9 @@ module Utilities
         Link.from_response( *args )
     end
 
-    # @see Arachni::Element::Link.from_document
-    def links_from_document( *args )
-        Link.from_document( *args )
+    # @see Arachni::Element::Link.from_parser
+    def links_from_parser( *args )
+        Link.from_parser(*args )
     end
 
     # @see Arachni::Element::Cookie.from_response
@@ -74,9 +74,9 @@ module Utilities
         Cookie.from_response( *args )
     end
 
-    # @see Arachni::Element::Cookie.from_document
-    def cookies_from_document( *args )
-        Cookie.from_document( *args )
+    # @see Arachni::Element::Cookie.from_parser
+    def cookies_from_parser( *args )
+        Cookie.from_parser(*args )
     end
 
     # @see Arachni::Element::Cookie.parse_set_cookie

data/lib/arachni/version.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -28,12 +28,6 @@ module Arachni
             GC.start( full_mark: false )
         end
 
-        def tmpdir
-            # On MS Windows Dir.tmpdir can return the path with a shortname,
-            # better avoid that as it can be insonsistent with other paths.
-            get_long_win32_filename( Dir.tmpdir )
-        end
-
         def null_device
             Gem.win_platform? ? 'NUL' : '/dev/null'
         end

data/lib/version

@@ -1 +1 @@
-1.4
+1.6.0

data/spec/arachni/browser/element_locator_spec.rb

@@ -125,42 +125,66 @@ describe Arachni::Browser::ElementLocator do
                     }
                 ).css).to eq('a[stuff="blah"]')
             end
+
+            context 'with values that include double quotes' do
+                it 'escapes them' do
+                    expect(described_class.new(
+                        tag_name: :a,
+                        attributes: {
+                            stuff: 'bl"ah'
+                        }
+                    ).css).to eq('a[stuff="bl\"ah"]')
+                end
+            end
         end
 
         context 'when there are multiple attributes' do
             it 'returns a CSS locator with the attributes' do
                 expect(described_class.new(
-                           tag_name: :a,
-                           attributes: {
-                               stuff:  'blah',
-                               stuff2: 'blah2'
-                           }
-                       ).css).to eq('a[stuff="blah"][stuff2="blah2"]')
+                    tag_name: :a,
+                    attributes: {
+                        stuff:  'blah',
+                        stuff2: 'blah2'
+                    }
+                ).css).to eq('a[stuff="blah"][stuff2="blah2"]')
             end
 
             context 'and an ID' do
                 it 'only includes the ID' do
                     expect(described_class.new(
-                               tag_name: :a,
-                               attributes: {
-                                   stuff:  'blah',
-                                   stuff2: 'blah2',
-                                   id:     'my-id'
-                               }
-                           ).css).to eq('a[id="my-id"]')
+                        tag_name: :a,
+                        attributes: {
+                            stuff:  'blah',
+                            stuff2: 'blah2',
+                            id:     'my-id'
+                        }
+                    ).css).to eq('a[id="my-id"]')
+                end
+            end
+
+            context "and a #{described_class::ARACHNI_ID}" do
+                it "only includes the #{described_class::ARACHNI_ID}" do
+                    expect(described_class.new(
+                       tag_name: :a,
+                       attributes: {
+                           stuff:  'blah',
+                           stuff2: 'blah2',
+                           described_class::ARACHNI_ID =>     'my-id'
+                       }
+                   ).css).to eq("a[#{described_class::ARACHNI_ID}=\"my-id\"]")
                 end
             end
 
             context 'and includes data ones' do
-                it "excludes all but #{described_class::ARACHNI_ID}" do
+                it 'excludes them' do
                     expect(described_class.new(
                         tag_name: :a,
                         attributes: {
-                            'data-stuff'                => 'blah',
-                            'data-stuff2'               => 'blah2',
-                            described_class::ARACHNI_ID => 'blah3'
+                            'data-stuff'  => 'blah',
+                            'data-stuff2' => 'blah2',
+                            'class'       => 'blah3'
                         }
-                    ).css).to eq("a[#{described_class::ARACHNI_ID}=\"blah3\"]")
+                    ).css).to eq('a[class="blah3"]')
                 end
             end
         end