arachni 0.4.2 → 0.4.3

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ZGEyZTQwN2Y1MDBmYjVmMWUwNmI0MmFmMjBlNGRjYTRjYzMzYjc2Yg==
+    ZWZmNTMzNDdmNmFlYjYzMTQyMjk3MzcyYzlmNWM1NzIwNTM3ZDE4Yg==
   data.tar.gz: !binary |-
-    Zjk3MzE2NzJlZTFjMjUzZTgzODhiYTQwMTc0NzQ1ODMxMmUyMTMzZg==
+    ZGE1Y2EyYTAxZjU0Y2I3YmRlOGIxZmFlNzRjODFiNjg0NWVhMTliZg==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    NDMwZjQ3YjM0ODExY2U3N2RkNmFiNmJlZjNhYTdjNjI2ZjkyNzFhYjM4ZWM5
-    MWVjMmI3NTFlMGRhZjQzMDZjZTJmMGFhNDM2NDkwNmFhZWMyYmIwZTJlMjY4
-    Y2U2ZTA5YmUxMDk5MGZiZTQyMWY4MTc3MTU4ZjU5OTMxYjZmZGY=
+    ZThkOWMzZDQ5MzY4ODUxN2UzMWY0ODk2MDFkMTYxZmZkZWIwZTA5ZGFhOTdm
+    MmYyN2NkNzMyNjU4NWJlOTdiOGUzMDRjN2NkZDhlZTcyNGYwZWRkM2Y5MTEw
+    NTA2MWFjZWQzMDkxNjQ3MDRiZjE3ZmQ1MWE4NzE3NGI5YWI3YjE=
   data.tar.gz: !binary |-
-    ZWUzN2FkOTc2N2MzNTg1OGJlNDc1N2U0ZTgyYjgxM2RhN2JmYWRiZjI2ODc2
-    ZGYzNjUwYTMzYzM5MDY3ODI1MGIxYWJkMDZiOTA1MGIyZjhkZDhlZmI2ZDM4
-    N2RmY2JlZWY5ZjM3YmJkZTIxYWE1NDZhNzRjODcwOTI4NzVlYWI=
+    ZTFjNmVmZjM4ZjA4MjZkNGE4NjYzYjdhYWJlZDc2YjQyNmU1ZmM2MGVkNmY0
+    ZTc2NmMyMGNjMmQ1MWQyMDI4ODI3NzM3OTlhYWI3NDMzZWUwYTMzZTQ5ZDJi
+    ZjNiZDM2OThmNzFlNWQwODAxZmFkYTBhYjJhZjE0MGY3ZmFiODI=

data/ACKNOWLEDGMENTS.md

@@ -16,3 +16,6 @@ All the people on:
 * [GitHub](http://github.com/Arachni/arachni/issues) who have submitted bugs and
     given constructive feedback.
 * The `CONTRIBUTORS.md` file.
+
+Finally, a big thanks to the [RubyMine](http://www.jetbrains.com/ruby/) people
+for providing their wonderful IDE to the Arachni project for free.

data/CHANGELOG.md

@@ -1,5 +1,186 @@
 # ChangeLog
 
+## Version 0.4.3 _(July 06, 2013)_
+
+- RPC protocol
+    - YAML serialization switched from `Syck` to `Psych` (the current Ruby default).
+- Executables:
+    - Added `arachni_multi`
+        - Spawns and controls an `RPC::Server::Instance` process in order to
+            provide access to RPC-only features such as multi-Instance scans
+            **without** requiring a Dispatcher.
+- CLI
+    - Added platform fingerprinting options:
+        - `--lsplat` -- Lists all available platforms.
+        - `--no-fingerprinting` -- Disables platform fingerprinting.
+        - `--platforms` -- Allows for user specified platforms.
+    - RPC client
+        - Added the `--grid-mode` option to allow the user to choose between:
+            - Load-balancing -- Slaves will be provided by the least burdened
+                Grid Dispatchers.
+            - Load balancing **with** line-aggregation -- In addition to balancing,
+                slaves will all be from Dispatchers with unique bandwidth Pipe-IDs
+                to result in application-level line-aggregation.
+- Added modular `Page` fingeprinting, via `fingerprinter` components, identifying:
+    - Operating systems
+        - BSD
+        - Linux
+        - Unix
+        - Windows
+        - Solaris
+    - Web servers
+        - Apache
+        - IIS
+        - Nginx
+        - Tomcat
+        - Jetty
+    - Programming languages
+        - PHP
+        - ASP
+        - ASPX
+        - JSP
+        - Python
+        - Ruby
+    - Frameworks
+        - Rack
+- `HTTP`
+    - `Accept-Encoding` set to `gzip, deflate` by default.
+- `Parser`
+    - Now fingerprints the pages it returns.
+- `Framework`
+    - Removed the following deprecated aliases:
+        - `:resume!` -- Only use `resume` from now on.
+        - `:pause!` -- Only use `pause` from now on.
+        - `:clean_up!` -- Only use `clean_up` from now on.
+    - Added `#list_platforms`.
+- `Spider`
+    - Optimized path de-duplication.
+    - Paths-list synchronized using a `Mutex` to prevent issues when running as
+        part of a multi-Instance operation.
+- `RPC::Server::Instance`
+    - Removed the following deprecated aliases:
+        - `:shutdown!` -- Only use `shutdown` from now on.
+    - Added preliminary support for UNIX sockets.
+    - Added `#list_platforms`.
+- `Module::Auditor`
+    - Having access to the `Framework` is now required and guaranteed.
+- `Element::Capabilities::Auditable`
+    - Out of scope elements are now visible in order to allow access to 3rd
+        party resources like Single Sign-On services.
+    - All audit methods return `false` when the element is out of the scan's scope.
+    - `#anonymous_auditor` now instantiates a `Framework`.
+    - Added `#skip_like` method to be passed blocks deciding what elements should
+        not be audited.
+    - `#audit`
+        - Updated to support the following payload types:
+            - `Array` -- Array of payloads to be injected.
+            - `Hash` -- Array of payloads to be injected per platform.
+- Grid
+    - `RPC::Server::Dispatcher#dispatch`
+        - When the Dispatcher is a Grid member, it returns an Instance from the least
+            burdened Grid member by default, thus allowing for easy load-balancing.
+- Multi-Instance scans
+    - Instances now communicate via UNIX domain sockets when all of them are on
+        the same host, to avoid TCP/IP overhead for IPC.
+    - `RPC::Server::Instance#scan`
+        - Added `grid_mode` option:
+            - `:balance` -- Slaves will be provided by the least burdened
+                Grid Dispatchers.
+            - `:aggregate` -- In addition to balancing, slaves will all be from
+                Dispatchers with unique bandwidth Pipe-IDs to result in
+                application-level line-aggregation.
+    - `RPC::Server::Framework`
+        - No longer performs a multi-Instance scan when its Dispatcher is a Grid
+            member. The `grid` or `grid_mode` options need to be set explicitly,
+            along with a `spawns` option value of 1 and higher.
+        - General code cleanup.
+            - Multi-Instance code moved under the `RPC::Server::Framework::MultiInstance`
+                module which concentrates multi-Instance code and pulls in the
+                following modules:
+                - `RPC::Server::Framework::Slave` -- Holds API and utilities for
+                    slave instances.
+                - `RPC::Server::Framework::Master` -- Holds API and utilities for
+                    master instances.
+        - Master seed crawl runs in its own thread to avoid blocking during the
+            initial seeding process.
+        - Removed the concept of issue summaries -- were used for lightweight
+            transmission of issue data for real-time feedback. Instead, full issues
+            are being buffered and flushed to the master after each page is audited.
+    - `RPC::Server::Framework::Distributor#distribute_elements`
+        - Optimized to handle large data sets.
+    - `RPC::Server::Spider`
+        - Updated buffering strategy to reduce RPC calls.
+- Cleaned up and removed `@@` vars from:
+    - `Module::Manager`
+    - `Module::KeyFiller`
+    - `Plugin::Manager`
+    - `Parser`
+- Moved supporting classes under `Arachni::Support`.
+    - `Support::Cache` classes now store `#hash` values of keys to preserve space.
+    - Added:
+        - `Support::LookUp` namespace to hold look-up optimized data structures with:
+            - `Support::LookUp::HashSet` -- Stores hashed items in a `Set`.
+            - `Support::LookUp::Moolb` -- Reverse of a Bloom-filter.
+        - `Support::Queue::Disk` -- Disk Queue with in-memory buffer.
+- Added:
+    - `Arachni::Platform` -- Holds resources relevant to platform identification,
+        storage, and filtering.
+        - `Fingerprinters` -- Namespace under which all fingerprinter
+            components reside.
+        - `List` - List structure holding applicable platforms for a given WWW resource.
+        - `Manager` - Collection of `Lists`s for easy management of platforms of
+            different types.
+    - `IO#tail` -- Returns a specified amount of lines from the bottom of a file.
+    - Process helpers for RPC Instance and Dispatcher servers.
+        - `Arachni::Processes::Dispatchers` -- Spawns and kills Dispatchers.
+        - `Arachni::Processes::Instances` -- Spawns and kills Instances.
+        - `Arachni::Processes::Manager` -- Forks and kills processes.
+- RSpec tests
+    - Major cleanup, using the aforementioned process helpers to remove duplicate code.
+    - Moved supporting components under `spec/support/`.
+- Modules
+    - Audit
+        - `code_injection`
+            - Removed `Ruby` payload since it wasn't applicable.
+            - Updated to categorize payloads by platform.
+        - `code_injection_timing`
+            - Code cleanup.
+            - Removed `payloads.txt`, payloads are now in the module.
+            - Updated to categorize payloads by platform.
+        - `os_cmd_injection`
+            - Code cleanup.
+            - Removed `payloads.txt`, payloads are now in the module.
+            - Updated to categorize payloads by platform.
+        - `os_cmd_injection_timing`
+            - Code cleanup.
+            - Removed `payloads.txt`, payloads are now in the module.
+            - Updated to categorize payloads by platform.
+        - `path_traversal`
+            - Code cleanup.
+            - Updated to categorize payloads by platform.
+        - `sqli_blind_timing`
+            - Code cleanup.
+            - Split `payloads.txt`, to individual files per platform.
+            - Updated to categorize payloads by platform.
+    - Recon
+        - `html_objects`
+            - Updated description.
+- Plugins
+    - Proxy
+        - Out-of-scope pages no longer return a _403 Forbidden_ error but are
+            instead loaded, though ignored.
+        - Fixed bug causing the `Content-Length` header to sometimes hold an
+            incorrect value.
+        - Fixed bug causing the control panel to be injected in a loop.
+        - Added support for `PUT` and `DELETE` methods.
+        - Supports exporting of discovered vectors in YAML format suitable for
+            use with the `vector_feed` plugin.
+        - Fixed bug with `POST` requests resulting in timed-out connections due
+            to forwarding a `Content-Length` request header to the origin server.
+    - AutoLogin
+        - Updated to allow access to out-of-scope resources like Single Sign-On
+            services.
+
 ## Version 0.4.2 _(April 26, 2013)_
 
 - Options
@@ -34,7 +215,6 @@
   - Protocol -- Now supports both ```Marshal``` and ```YAML``` automatically.
       - ```Marshal``` by default since it's many times faster than ```YAML```.
       - ```YAML``` as an automatic fallback in order to maintain backwards compatibility and ease of integration with 3rd parties.
-          - Updated to use the Ruby-default ```Psych``` engine.
   - ```Framework```
       - Updated gathering of slave status -- once a slave is done it reports back to the master.
       - Clean-up happens automatically, clients no longer need to call ```#clean_up``` (like previously mentioned).

data/README.md

@@ -3,18 +3,18 @@
 <table>
     <tr>
         <th>Version</th>
-        <td>0.4.2</td>
+        <td>0.4.3</td>
     </tr>
     <tr>
         <th>Homepage</th>
-        <td><a href="http://arachni-scanner.com">http://arachni-scanner.com</a></td>
+        <td><a href="http://www.arachni-scanner.com">http://arachni-scanner.com</a></td>
     </tr>
     <tr>
         <th>Blog</th>
-        <td><a href="http://arachni-scanner.com/blog">http://arachni-scanner.com/blog</a></td>
+        <td><a href="http://www.arachni-scanner.com/blog">http://arachni-scanner.com/blog</a></td>
     <tr>
         <th>Github</th>
-        <td><a href="http://github.com/Arachni/arachni">http://github.com/Arachni/arachni</a></td>
+        <td><a href="https://github.com/Arachni/arachni">http://github.com/Arachni/arachni</a></td>
      <tr/>
     <tr>
         <th>Documentation</th>
@@ -133,17 +133,29 @@ you with its findings.
 
 ### Open [distributed architecture](https://github.com/Arachni/arachni/wiki/Distributed-components)
 
-- High-performance/low-bandwidth [communication protocol](https://github.com/Arachni/arachni-rpc).
-- Multiple clients.
+- High-performance/low-bandwidth [communication protocol](https://github.com/Arachni/arachni-rpc-em).
+    - `Marshal` serialization for performance and efficiency.
+        - Automatically falls back to `YAML` for ease of integration with 3rd party systems.
+    - TCP/IP for general network communications.
+    - UNIX domain sockets for multi-Instance IPC.
+- Remote monitoring and management of Dispatchers and Instances.
 - Parallel scans -- Each scan is compartmentalized to its own OS process to take
     advantage of:
     - Multi-core/SMP architectures.
     - OS-level scheduling/restrictions.
     - Sandboxed failure propagation.
+- Multi-Instance scans for parallelization of _individual scans_ using multiple
+    Instances to:
+    - Take advantage of multi-core/SMP architectures.
+    - Greatly diminish scan-times.
+- Dispatcher Grids supporting:
+    - _(Optional)_ High-Performance mode -- Combines the resources of
+        multiple nodes to perform multi-Instance scans.
+        - Enabled on a per-scan basis.
+    - _(Always-on)_ Load-balancing -- All Instances are automatically provided
+        by the least burdened Grid member.
+        - With optional per-scan opt-out/override.
 - SSL encryption (with optional peer authentication).
-- Remote monitoring and management.
-- Experimental support for High Performance Grid configuration, combining the
-    resources of multiple nodes to perform faster scans.
 
 ### Crawler
 
@@ -176,6 +188,43 @@ you with its findings.
     permutations to provide extensive cookie-audit coverage.
  - Can exclude specific input vectors by name.
 
+### Platform fingerprinter
+
+In order to make efficient use of the available bandwidth, Arachni performs some
+basic platform fingerprinting and tailors the audit process to the server-side
+deployed platforms by only injecting applicable payloads.
+
+Currently, the following platforms can be identified:
+
+- Operating systems
+    - BSD
+    - Linux
+    - Unix
+    - Windows
+    - Solaris
+- Web servers
+    - Apache
+    - IIS
+    - Nginx
+    - Tomcat
+    - Jetty
+- Programming languages
+    - PHP
+    - ASP
+    - ASPX
+    - JSP
+    - Python
+    - Ruby
+- Frameworks
+    - Rack
+
+The user also has the option of specifying extra platforms (like a DB server)
+in order to help the system be as efficient as possible. Alternatively, fingerprinting
+can be disabled altogether.
+
+Finally, Arachni will always err on the side of caution and send all available
+payloads when it fails to identify specific platforms.
+
 ### HTML Parser
 
 Can extract and analyze:
@@ -400,7 +449,10 @@ You can run `rake spec` to run **all** specs or you can run them selectively usi
     rake spec:reports         # for the reports
     rake spec:path_extractors # for the path extractors
 
-**Note**: _The module specs will take about 90 minutes due to the ones which perform timing attacks._
+**Please be warned**, the core specs will require a beast of a machine due to the
+necessity to test the Grid/multi-Instance features of the system.
+
+**Note**: _The module specs will take about 90 minutes due to the timing-attack tests._
 
 ## Bug reports/Feature requests
 
@@ -409,19 +461,21 @@ get support via the [Support Portal](http://support.arachni-scanner.com).
 
 ## Contributing
 
+(Before starting any work, please read the [instructions](https://github.com/Arachni/arachni/tree/experimental#source)
+for working with the source code.)
+
 We're happy to accept help from fellow code-monkeys and these are the steps you
 need to follow in order to contribute code:
 
-* [Fork the project](https://github.com/Arachni/arachni/fork_select).
-* Start a feature branch based on the `experimental` branch (`git checkout -b <feature-name> experimental`).
+* Fork the project.
+* Start a feature branch based on the [experimental](https://github.com/Arachni/arachni-ui-web/tree/experimental)
+    branch (`git checkout -b <feature-name> experimental`).
 * Add specs for your code.
 * Run the spec suite to make sure you didn't break anything (`rake spec:core`
     for the core libs or `rake spec` for everything).
 * Commit and push your changes.
 * Issue a pull request and wait for your code to be reviewed.
 
-_PS: You may want to setup a [development environment](https://github.com/Arachni/arachni/wiki/Development-environment) first._
-
 ## License
 
 Arachni is licensed under the Apache License Version 2.0.<br/>

data/Rakefile

@@ -157,11 +157,7 @@ task :clean do
     sh "rm *.marshal || true"
     sh "rm *.gem || true"
     sh "rm logs/*.log || true"
-    sh "rm spec/logs/*.log || true"
-    sh "rm lib/arachni/ui/web/server/db/*.* || true"
-    sh "rm lib/arachni/ui/web/server/db/welcomed || true"
-    sh "rm lib/arachni/ui/web/server/public/reports/*.* || true"
-    sh "rm lib/arachni/ui/web/server/tmp/*.* || true"
+    sh "rm spec/support/logs/*.log || true"
 end
 
 

data/arachni.gemspec

@@ -27,7 +27,7 @@ Gem::Specification.new do |s|
         ' Ruby framework aimed towards helping penetration testers and' +
         ' administrators evaluate the security of web applications.'
 
-    s.homepage          = 'https://github.com/Arachni/arachni'
+    s.homepage          = 'https://www.arachni-scanner.com'
     s.email             = 'tasos.laskos@gmail.com'
     s.authors           = [ 'Tasos Laskos' ]
 
@@ -38,6 +38,7 @@ Gem::Specification.new do |s|
     s.files            += Dir.glob( 'lib/**/**' )
     s.files            += Dir.glob( 'logs/**/**' )
     s.files            += Dir.glob( 'modules/**/**' )
+    s.files            += Dir.glob( 'fingerprinters/**/**' )
     s.files            += Dir.glob( 'path_extractors/**/**' )
     s.files            += Dir.glob( 'plugins/**/**' )
     s.files            += Dir.glob( 'profiles/**/**' )
@@ -45,9 +46,11 @@ Gem::Specification.new do |s|
     s.files            += Dir.glob( 'rpcd_handlers/**/**' )
     s.files            += Dir.glob( 'spec/**/**' )
     s.files            += %w(Gemfile Rakefile arachni.gemspec)
+    s.test_files        = Dir.glob( 'spec/**/**' )
 
     s.executables       = [ 'arachni', 'arachni_rpcd_monitor', 'arachni_rpcd',
-                          'arachni_rpc', 'arachni_console', 'arachni_script' ]
+                            'arachni_rpc', 'arachni_console', 'arachni_script',
+                            'arachni_multi' ]
 
     s.extra_rdoc_files  = %w(README.md ACKNOWLEDGMENTS.md LICENSE.md
                             AUTHORS.md CHANGELOG.md CONTRIBUTORS.md
@@ -55,7 +58,8 @@ Gem::Specification.new do |s|
 
     s.rdoc_options      = [ '--charset=UTF-8' ]
 
-    s.add_dependency 'bundler',         '>= 1.0.0'
+    # RPC client/server implementation.
+    s.add_dependency 'arachni-rpc-em',  '0.2'
 
     # HTTP interface.
     s.add_dependency 'typhoeus',        '~> 0.3.3'
@@ -84,16 +88,15 @@ Gem::Specification.new do |s|
     # Outputting data in table format (arachni_rpcd_monitor).
     s.add_dependency 'terminal-table',  '>= 1.4.2'
 
-    # RPC client/server implementation.
-    s.add_dependency 'arachni-rpc-em',  '~> 0.1.3'
-
     # For CLI interfaces.
     s.add_dependency 'highline'
 
+    s.add_development_dependency 'bundler'
     s.add_development_dependency 'rake'
     s.add_development_dependency 'rspec'
 
     # For test mini-servers.
+    s.add_development_dependency 'thin'
     s.add_development_dependency 'sinatra',         '~> 1.3.2'
     s.add_development_dependency 'sinatra-contrib', '~> 1.3.1'
 

data/bin/arachni_multi

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+=begin
+    Copyright 2010-2013 Tasos Laskos <tasos.laskos@gmail.com>
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+=end
+
+require_relative '../lib/arachni'
+
+Arachni::Options.parse
+require Arachni::Options.dir['lib'] + 'ui/cli/rpc/local'
+
+Arachni::UI::CLI::RPC::Local.new