arachni 1.3.2 → 1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d0495113ec40359a8b7dbe109db091405eba059d
-  data.tar.gz: 977c1ba9d8aa9d6b3079501574b4014f68524373
+  metadata.gz: f4c2f78464615f487fac68b9e72685088d568977
+  data.tar.gz: db89773e54c4321dbb18bd9c18958c3263f4c952
 SHA512:
-  metadata.gz: 438315daa50629cd4fc823a77d7a9ed2a46e1e6bd4971f190f02a0d6b890d796b4e10e03a9b15c93d7486a21a77a1e237ead5ad0040114020baee6900cff3cb6
-  data.tar.gz: 6affe8aba04846a30e8673c4530c32a246782aa4ee4d54656cbf8ff99ff31b6e0b1eff1712839023233bad28d232ce6b9c927bcc3b2a5cb8a5fdfa0a95c452fd
+  metadata.gz: 5ed261c2eab8f545585e8ac2a733f59296823e8d3de533a6789ba871c4a4ff732108760c33169198fde6b8c240192cdbeba16a9241d8eba4a03da6b9ee9b176b
+  data.tar.gz: 50297d450fd4e8d53b1c556af729d889a94ea0c4cd2f7220e2703dd70f274bff373d6603fb57dfb99860d3fd90199b0258d0a6e271d672e92564884cf8be5a1f

data/CHANGELOG.md

@@ -1,5 +1,113 @@
 # ChangeLog
 
+## 1.4 _(February 7, 2016)_
+
+- Native MS Windows compatibility.
+- Options
+    - `--http-proxy-type` -- Added `socks5h`, enabling hostname resolution via the proxy.
+    - Added
+        - Scope
+            - `--scope-exclude-file-extensions` -- CSV of file extensions to exclude.
+        - Audit
+            - `--audit-with-raw-payloads` -- Injects both raw and HTTP encoded payloads.
+- `URI` -- Optimized and re-written to completely bypass Ruby's `URI` lib.
+- `Plugin::Manager`
+    - Run `#prepare` methods of plugins in the Framework thread, ordered by plugin priority.
+- `HTTP`
+    - `ProxyServer` -- Replaced the previous `WEBrick`-based one with a custom
+        written server with support for `keep-alive` and low-overhead SSL interception.
+    - `Client`
+        - Added default value for `Accept-Language` header.
+        - Updated to treat cookie-jar cookies as dumb storage and not encode/decode
+            names and values.
+        - `Dynamic404Handler` -- Check for excessive amounts of noise during
+            custom-404 signature generation and abort if an accurate reading is
+            impossible.
+- `Page`
+    - `DOM`
+        - `#restore` -- Don't preload the stored page to avoid stale nonces,
+            instead rely solely on browser for caching.
+- `Browser`
+    - Replaced internal use of `Watir` with direct access to `Selenium`, resulting
+        in much better performance and lower CPU utilization.
+    - Sped up process spawning,
+    - Switched to `Selenium`'s default HTTP client for `WebDriver` communications
+        in order to resolve JRuby and MS Windows issues.
+    - Added support for tracking event delegation.
+    - `#spawn_phantomjs` -- Use a Ruby lifeline process to kill the browser
+            if the parent dies for whatever reason.
+    - `#fire_event` -- Track changes in timers caused by event triggers to identify
+        and wait for effects and transitions.
+- `Support`
+    - `Signature` -- Optimized signature tokenization, deduplication and compression
+        to be less resource intensive when processing large data sets.
+    - `Cache` -- Minimized calls to `Base#make_key`.
+    - Added
+        - `Glob` -- Glob matcher.
+- `Session`
+    - Added `#check_options`, allowing login scripts to set advanced HTTP request
+        options for login checks.
+- `REST::Server` -- Added REST API.
+- `RPC`
+    - `Server`
+        - `ActiveOptions#set` -- Allow options to be set during runtime and adjust
+            the scan scope accordingly.
+- `Element`
+    - `UIInput::DOM` -- Updated coverage identifier calculation.
+    - `UIForm::DOM` -- Updated coverage identifier calculation.
+    - `Capabilities`
+        - `Analyzable`
+            - `Signature`
+                - Replaced `regexp` and `substring` options with `signature` --
+                    type of matching depends on `signature` type.
+                - Allow `signature` to be generated dynamically based on the
+                    `HTTP::Response` about to be checked, from a `#call`able object.
+            - `Differential`
+                - Abort on partial responses to avoid FPs caused by server stress
+                    or Firewall/IDS/IPS.
+            - `Timeout`
+                - Added one more verification phase to further reduce the possibility
+                    of random FPs.
+- Checks
+    - Active -- Updated all checks that make use of `Element::Capabilities::Analyzable::Signature`
+        to provide simple substring signatures whenever possible.
+        Alternatively, when a `Regexp` is necessary, they take advantage of dynamic
+        signature generation based on the current response and perform a lightweight
+        preliminary check for hints of vulnerability, only then is the more
+        resource intensive `Regexp` matched.
+        - `xss`, `xss_dom`, `xss_tag`, `xss_event`, `xss_script_context` --
+            Optimized identification of tainted responses to avoid parsing as
+                much as possible.
+        - `xss_dom` -- Updated payloads to improve coverage.
+        - `sql_injection_differential`
+            - Replaced `-1` control `false` value with `-1839`
+            - When using quotes, quote all parts of the conditional in the SQL query.
+        - `no_sql_injection_differential`
+            - Replaced `-1` control `false` value with `-1839`
+    - Passive
+        - `directory_listing` - Bail out on failed requests to avoid FPs.
+        - `backdoors`, `backup_directories`, `backup_files`, `common_admin_interfaces`,
+            `common_directories`, `common_files` -- Bail out if the seed resource
+            is already a 404.
+        - Grep
+            - `emails` -- Verify e-mail addresses by resolving the identified domains.
+            - `credit_card`, `ssn` -- Mark issues as untrusted by default since
+                there's no way to verify SSNs.
+            - `http_only_cookies`, `insecure_cookies` -- Only check current page
+                cookies, don't let the CookieJar ones sneak in.
+- Plugins
+    - `proxy`
+        - Removed injection of control toolbar to each response.
+        - Cleaned up control panel design.
+        - Updated description to list management URLs and SSL interception info.
+    - `email_notify` -- Made username and password optional.
+    - `defaults/meta/remedy/`
+        - `discovery` -- Updated similarity check to prevent analysis of singular issues.
+- Reporters
+    - `xml` -- Updated validation messages to point to relevant markup.
+- Path extractors
+    - `meta_refresh` -- Strip whitespaces from URLs when not in quotes.
+
 ## 1.3.2 _(October 19, 2015)_
 
 - `UI`

data/Gemfile

@@ -10,19 +10,15 @@ end
 group :spec do
     gem 'simplecov', require: false, group: :test
 
-    gem 'rspec', '2.99'
+    gem 'rspec', '2.99.0'
     gem 'faker'
-
-    gem 'puma' if !Gem.win_platform? || RUBY_PLATFORM == 'java'
-
-    gem 'sinatra'
-    gem 'sinatra-contrib'
 end
 
 group :prof do
     gem 'stackprof'
     gem 'sys-proctable'
     gem 'ruby-mass'
+    gem 'benchmark-ips'
 end
 
 gemspec

data/LICENSE.md

@@ -1,6 +1,6 @@
 # License
 
-Copyright 2010-2015 [Tasos Laskos](mailto:tasos.laskos@arachni-scanner.com).
+Copyright 2010-2016 [Tasos Laskos](mailto:tasos.laskos@arachni-scanner.com).
 
 ```
                     Arachni Public Source License

data/README.md

@@ -3,7 +3,7 @@
 <table>
     <tr>
         <th>Version</th>
-        <td>1.3.2</td>
+        <td>1.4</td>
     </tr>
     <tr>
         <th>Homepage</th>
@@ -38,7 +38,7 @@
     </tr>
     <tr>
         <th>Copyright</th>
-        <td>2010-2015 Tasos Laskos</td>
+        <td>2010-2016 Tasos Laskos</td>
     </tr>
     <tr>
         <th>License</th>
@@ -236,20 +236,38 @@ Arachni is able to extract and audit the following elements and their inputs:
 
 ### Open [distributed architecture](https://github.com/Arachni/arachni/wiki/Distributed-components)
 
-- High-performance/low-bandwidth [communication protocol](https://github.com/Arachni/arachni-rpc).
-    - `MessagePack` serialization for performance, efficiency and ease of
-        integration with 3rd party systems.
-- Remote monitoring and management of Dispatchers and Instances.
-- Parallel scans -- Each scan is compartmentalized to its own OS process to take
-    advantage of:
+Arachni is designed to fit into your workflow and easily integrate with your
+existing infrastructure.
+
+Depending on the level of control you require over the process, you can either
+choose the REST service or the custom RPC protocol.
+
+Both approaches allow you to:
+
+- Remotely monitor and manage scans.
+- Perform multiple scans at the same time -- Each scan is compartmentalized to
+    its own OS process to take advantage of:
     - Multi-core/SMP architectures.
     - OS-level scheduling/restrictions.
     - Sandboxed failure propagation.
-- Multi-Instance scans for parallelization of _individual scans_ using multiple
-    Instances to:
-    - Take advantage of multi-core/SMP architectures.
-    - Greatly diminish scan-times.
-- Dispatcher Grid:
+- Communicate over a secure channel.
+
+#### [REST API](https://github.com/Arachni/arachni/wiki/REST-API)
+
+- Very simple and straightforward API.
+- Easy interoperability with non-Ruby systems.
+    - Operates over HTTP.
+    - Uses JSON to format messages.
+- Stateful scan monitoring.
+    - Unique sessions automatically only receive updates when polling for progress,
+        rather than full data.
+
+#### [RPC API](https://github.com/Arachni/arachni/wiki/RPC-API)
+
+- High-performance/low-bandwidth [communication protocol](https://github.com/Arachni/arachni-rpc).
+    - `MessagePack` serialization for performance, efficiency and ease of
+        integration with 3rd party systems.
+- Grid:
     - Self-healing.
     - Scale up/down by hot-plugging/hot-unplugging nodes.
         - Can scale up infinitely by adding nodes to increase scan capacity.
@@ -259,7 +277,6 @@ Arachni is able to extract and audit the following elements and their inputs:
     - _(Optional)_ High-Performance mode -- Combines the resources of
         multiple nodes to perform multi-Instance scans.
         - Enabled on a per-scan basis.
-- SSL encryption (with optional peer authentication).
 
 ### Scope configuration
 
@@ -301,8 +318,9 @@ Arachni is able to extract and audit the following elements and their inputs:
     - JSON request data.
     - XML request data.
  - Can ignore binary/non-text pages.
- - Can optionally audit elements using both `GET` and `POST` HTTP methods.
- - Can optionally submit all links and forms of the page along with the cookie
+ - Can audit elements using both `GET` and `POST` HTTP methods.
+ - Can inject both raw and HTTP encoded payloads.
+ - Can submit all links and forms of the page along with the cookie
     permutations to provide extensive cookie-audit coverage.
  - Can exclude specific input vectors by name.
  - Can include specific input vectors by name.

data/Rakefile

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/arachni.gemspec

@@ -1,6 +1,6 @@
 # coding: utf-8
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -35,39 +35,54 @@ Gem::Specification.new do |s|
     s.files            += %w(Gemfile Rakefile arachni.gemspec)
     s.test_files        = Dir.glob( 'spec/**/**' )
 
-    s.executables       = [ 'arachni', 'arachni_rpcd_monitor', 'arachni_rpcd',
-                            'arachni_rpc', 'arachni_console', 'arachni_script',
-                            'arachni_multi', 'arachni_reporter', 'arachni_restore' ]
+    s.executables       = Dir.glob( 'bin/*' ).map { |e| File.basename e }
 
     s.extra_rdoc_files  = %w(README.md ACKNOWLEDGMENTS.md LICENSE.md
                             AUTHORS.md CHANGELOG.md CONTRIBUTORS.md)
 
     s.rdoc_options      = [ '--charset=UTF-8' ]
 
+    s.add_dependency 'awesome_print'
+
     s.add_dependency 'rack'
 
     s.add_dependency 'bundler'
 
+    s.add_dependency 'concurrent-ruby',     '1.0.0'
+    s.add_dependency 'concurrent-ruby-ext', '1.0.0'
+
     # For compressing/decompressing system state archives.
     s.add_dependency 'rubyzip',           '1.1.6'
 
+    # HTTP proxy server
+    s.add_dependency 'http_parser.rb'
+
     # HTML report
     s.add_dependency 'coderay',           '1.1.0'
 
     s.add_dependency 'childprocess',      '0.5.3'
 
     # RPC serialization.
-    if RUBY_PLATFORM == 'java'
-        s.add_dependency 'msgpack-jruby', '1.4.0'
-    else
-        s.add_dependency 'msgpack',       '0.5.8'
+    s.add_dependency 'msgpack',           '0.7.0'
+
+    if RUBY_PLATFORM != 'java'
+        # Optimized JSON.
+        s.add_dependency 'oj',            '~> 2.14.3'
+        s.add_dependency 'oj_mimic_json'
     end
 
+    # Web server
+    s.add_dependency 'puma',              '2.14.0'
+
+    # REST API
+    s.add_dependency 'sinatra',           '1.4.6'
+    s.add_dependency 'sinatra-contrib',   '1.4.6'
+
     # RPC client/server implementation.
-    s.add_dependency 'arachni-rpc',       '0.2.1.2'
+    s.add_dependency 'arachni-rpc',       '0.2.1.3'
 
     # HTTP client.
-    s.add_dependency 'typhoeus',          '0.6.9'
+    s.add_dependency 'typhoeus',          '1.0.1'
 
     # Fallback URI parsing and encoding utilities.
     s.add_dependency 'addressable',       '2.3.6'
@@ -75,24 +90,17 @@ Gem::Specification.new do |s|
     # E-mail plugin.
     s.add_dependency 'pony',              '1.8'
 
-    # Printing complex objects.
-    s.add_dependency 'awesome_print',     '~> 1.2.0'
-
-    # Optimized JSON.
-    s.add_dependency 'oj',                '~> 2.12.9'
-    s.add_dependency 'oj_mimic_json'
-
     # For the Arachni console (arachni_console).
     s.add_dependency 'rb-readline',       '0.5.1'
 
     # Markup parsing.
-    s.add_dependency 'nokogiri',          '~> 1.6.5'
+    s.add_dependency 'nokogiri',          '1.6.8rc2'
 
     # Outputting data in table format (arachni_rpcd_monitor).
     s.add_dependency 'terminal-table',    '1.4.5'
 
     # Browser support for DOM/JS/AJAX analysis stuff.
-    s.add_dependency 'watir-webdriver',   '0.6.9'
+    s.add_dependency 'watir-webdriver',   '0.8.0'
 
     # Markdown to HTML conversion, used by the HTML report for component
     # descriptions.
@@ -116,7 +124,7 @@ License            - Arachni Public Source License v1.0
                         (https://github.com/Arachni/arachni/blob/master/LICENSE.md)
 Author             - Tasos "Zapotek" Laskos (http://twitter.com/Zap0tek)
 Twitter            - http://twitter.com/ArachniScanner
-Copyright          - 2010-2015 Tasos Laskos
+Copyright          - 2010-2016 Tasos Laskos
 
 Please do not hesitate to ask for assistance (via the support portal)
 or report a bug (via GitHub Issues) if you come across any problem.

data/bin/arachni

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/bin/arachni_console

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/bin/arachni_multi

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/bin/arachni_reporter

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/bin/arachni_rest_server

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+=begin
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+
+    This file is part of the Arachni Framework project and is subject to
+    redistribution and commercial restrictions. Please see the Arachni Framework
+    web site for more information on licensing and terms of use.
+=end
+
+require_relative '../lib/arachni'
+require_relative '../ui/cli/rest/server'
+
+Arachni::UI::CLI::Rest::Server.new

data/bin/arachni_restore

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/bin/arachni_rpc

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/bin/arachni_rpcd

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/bin/arachni_rpcd_monitor

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/bin/arachni_script

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework