metasploit_data_models 0.6.13

data/.gitignore

@@ -0,0 +1,24 @@
+# bundler configuration
+.bundle
+# Mac OS X folder attributes
+.DS_Store
+# built gems
+*.gem
+# Rubymine project configuration
+.idea
+# logs
+*.log
+# Don't check in rvmrc since this is a gem
+.rvmrc
+# YARD database
+.yardoc
+# coverage report directory for simplecov/Rubymine
+coverage
+# generated yardocs
+doc
+# Installed gem versions.  Not stored for the same reasons as .rvmrc
+Gemfile.lock
+# Packaging directory for builds
+pkg/*
+# Database configuration (with passwords) for specs
+spec/dummy/config/database.yml

data/.rspec

@@ -0,0 +1,3 @@
+--format nested
+--colour
+--drb

data/.simplecov

@@ -0,0 +1,38 @@
+# RM_INFO is set when using Rubymine.  In Rubymine, starting SimpleCov is
+# controlled by running with coverage, so don't explicitly start coverage (and
+# therefore generate a report) when in Rubymine.  This _will_ generate a report
+# whenever `rake spec` is run.
+unless ENV['RM_INFO']
+	SimpleCov.start
+end
+
+SimpleCov.configure do
+  load_adapter('rails')
+
+  # ignore this file
+	add_filter '.simplecov'
+
+	#
+	# Changed Files in Git Group
+	# @see http://fredwu.me/post/35625566267/simplecov-test-coverage-for-changed-files-only
+	#
+
+	untracked = `git ls-files --exclude-standard --others`
+	unstaged = `git diff --name-only`
+	staged = `git diff --name-only --cached`
+	all = untracked + unstaged + staged
+	changed_filenames = all.split("\n")
+
+	add_group 'Changed' do |source_file|
+		changed_filenames.detect { |changed_filename|
+			source_file.filename.end_with?(changed_filename)
+		}
+	end
+
+	#
+	# Specs are reported on to ensure that all examples are being run and all
+	# lets, befores, afters, etc are being used.
+	#
+
+	add_group 'Specs', 'spec'
+end

data/.yardopts

@@ -0,0 +1,4 @@
+--markup markdown
+--protected
+{app,lib}/**/*.rb
+db/migrate/*.rb

data/Gemfile

@@ -0,0 +1,27 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in metasploit_data_models.gemspec
+gemspec
+
+# used by dummy application
+group :development, :test do
+  # supplies factories for producing model instance for specs
+  # Version 4.1.0 or newer is needed to support generate calls without the 'FactoryGirl.' in factory definitions syntax.
+  gem 'factory_girl', '>= 4.1.0'
+  # auto-load factories from spec/factories
+  gem 'factory_girl_rails'
+  # rails is only used for the dummy application in spec/dummy
+  gem 'rails'
+end
+
+group :test do
+  # In a full rails project, factory_girl_rails would be in both the :development, and :test group, but since we only
+  # want rails in :test, factory_girl_rails must also only be in :test.
+  # add matchers from shoulda, such as validates_presence_of, which are useful for testing validations
+  gem 'shoulda-matchers'
+  # code coverage of tests
+  gem 'simplecov', :require => false
+  # need rspec-rails >= 2.12.0 as 2.12.0 adds support for redefining named subject in nested context that uses the
+  # named subject from the outer context without causing a stack overflow.
+  gem 'rspec-rails', '>= 2.12.0'
+end

data/LICENSE

@@ -0,0 +1,27 @@
+Copyright (C) 2012, Rapid7, Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice,
+	  this list of conditions and the following disclaimer.
+
+    * Redistributions in binary form must reproduce the above copyright notice,
+	  this list of conditions and the following disclaimer in the documentation
+	  and/or other materials provided with the distribution.
+
+    * Neither the name of Rapid7 LLC nor the names of its contributors
+	  may be used to endorse or promote products derived from this software
+	  without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.md

@@ -0,0 +1,72 @@
+#MetasploitDataModels
+
+The database layer for Metasploit
+
+
+## Purpose
+__MetasploitDataModels__ exists to do several key things:
+
+1. Allow code sharing between Metasploit Framework (MSF) and the commercial versions of Metasploit (Community, Express, Pro -- usually referred to collectively as "Pro")
+
+2. Give developers a lightweight entry point to MSF's backend for use in developing tools that gather data intended for later use with Metasploit (e.g. specialized scanners).
+
+3. Make it easy to keep commercial stuff private while increasing the functionality of the open-source tools we provide to the community.
+
+
+## Usage
+
+### Rails
+
+In a Rails application, MetasploitDataModels acts a
+[Rails Engine](http://edgeapi.rubyonrails.org/classes/Rails/Engine.html) and the models are available to application
+just as if they were defined under app/models.  If your Rails appliation needs to modify the models, this can be done
+using ActiveSupport.on_load hooks in initializers.  The block passed to on_load hook is evaluated in the context of the
+model class, so defining method and including modules will work just like reopeninng the class, but
+ActiveSupport.on_load ensures that the monkey patches will work after reloading in development mode.  Each class has a
+different on_load name, which is just the class name converted to an underscored symbol, so Mdm::ApiKey runs the
+:mdm_api_key load hooks, etc.
+
+    # Gemfile
+    gem :metasploiit_data_models, :git => git://github.com/rapid7/metasploit_data_models.git, :tag => 'v0.3.0'
+
+    # config/initializers/metasploit_data_models.rb
+    ActiveSupport.on_load(:mdm_api_key) do
+        # Returns the String obfuscated token for display. Meant to avoid CSRF
+        # api-key stealing attackes.
+        def obfuscated_token
+          token[0..3] + "****************************"
+        end
+    end
+
+### Metasploit Framework
+
+In Metasploit Framework, `MetasploitDataModels.require_models` is called by the `Msf::DbManager` to use the data models
+only if the user wants to use the database.
+
+### Elsewhere
+
+__NOTE: This isn't in RubyGems yet.  Using a Gemfile entry pointing to this repo (i.e., using
+[Bundler](http://gembundler.com)) is the suggested option for now.__
+
+Usage outside of Rapid7 is still alpha, as reflected in the pre-1.0.0 version, and we're not making many promises.  That
+being said, usage is easy:
+
+    connection_info = YAML.load_file("path/to/rails-style/db_config_file")
+    ActiveRecord::Base.establish_connection(connection_info['development'])
+    MetasploitDataModels.require_models
+
+Basically you need to do the following things:
+
+1. Establish an ActiveRecord connection.  A Rails __config/database.yml__ is ideal for this.
+2. `MetasploitDataModels.require_models`
+
+
+## Developer Info
+
+### Console
+The gem includes a console based on [Pry](https://github.com/pry/pry/)
+
+Give it a path to a working MSF database.yml file for full
+ActiveRecord-based access to your data.
+
+__Note:__ "development" mode is hardcoded into the console currently.

data/Rakefile

@@ -0,0 +1,53 @@
+#!/usr/bin/env rake
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+print_without = false
+APP_RAKEFILE = File.expand_path('../spec/dummy/Rakefile', __FILE__)
+
+begin
+  load 'rails/tasks/engine.rake'
+rescue LoadError
+  puts "railties not in bundle, so can't load engine tasks."
+  print_without = true
+end
+
+Bundler::GemHelper.install_tasks
+
+#
+# load rake files like a normal rails app
+# @see http://viget.com/extend/rails-engine-testing-with-rspec-capybara-and-factorygirl
+#
+
+pathname = Pathname.new(__FILE__)
+root = pathname.parent
+rakefile_glob = root.join('lib', 'tasks', '**', '*.rake').to_path
+
+Dir.glob(rakefile_glob) do |rakefile|
+  load rakefile
+end
+
+begin
+  require 'rspec/core'
+rescue LoadError
+  puts "rspec not in bundle, so can't set up spec tasks.  " \
+       "To run specs ensure to install the development and test groups."
+  print_without = true
+else
+  require 'rspec/core/rake_task'
+
+  # Depend on app:db:test:prepare so that test database is recreated just like in a full rails app
+  # @see http://viget.com/extend/rails-engine-testing-with-rspec-capybara-and-factorygirl
+  RSpec::Core::RakeTask.new(:spec => 'app:db:test:prepare')
+
+  task :default => :spec
+end
+
+if print_without
+  puts "Bundle currently installed '--without #{Bundler.settings.without.join(' ')}'."
+  puts "To clear the without option do `bundle install --without ''` (the --without flag with an empty string) or " \
+       "`rm -rf .bundle` to remove the .bundle/config manually and then `bundle install`"
+end

data/app/models/mdm/api_key.rb

@@ -0,0 +1,20 @@
+class Mdm::ApiKey < ActiveRecord::Base
+  #
+  # Validators
+  #
+
+  validate :supports_api
+  validates :token, :presence => true, :length => { :minimum => 8 }
+
+  protected
+
+  def supports_api
+    license = License.get
+
+    if license and not license.supports_api?
+      errors[:license] = " - this product does not support API access"
+    end
+  end
+
+  ActiveSupport.run_load_hooks(:mdm_api_key, self)
+end

data/app/models/mdm/client.rb

@@ -0,0 +1,9 @@
+class Mdm::Client < ActiveRecord::Base
+  #
+  # Relations
+  #
+  belongs_to :campaign, :class_name => 'Mdm::Campaign'
+  belongs_to :host, :class_name => 'Mdm::Host'
+
+  ActiveSupport.run_load_hooks(:mdm_client, self)
+end

data/app/models/mdm/cred.rb

@@ -0,0 +1,80 @@
+class Mdm::Cred < ActiveRecord::Base
+  #
+  # CONSTANTS
+  #
+  KEY_ID_REGEX = /([0-9a-fA-F:]{47})/
+  PTYPES = {
+      'read/write password' => 'password_rw',
+      'read-only password' => 'password_ro',
+      'SMB hash' => 'smb_hash',
+      'SSH private key' => 'ssh_key',
+      'SSH public key' => 'ssh_pubkey'
+  }
+
+  #
+  # Relations
+  #
+  belongs_to :service, :class_name => "Mdm::Service"
+
+  def ptype_human
+    humanized = PTYPES.select do |k, v|
+      v == ptype
+    end.keys[0]
+
+    humanized ? humanized : ptype
+  end
+
+  # Returns its key id. If this is not an ssh-type key, returns nil.
+  def ssh_key_id
+    return nil unless self.ptype =~ /^ssh_/
+    return nil unless self.proof =~ KEY_ID_REGEX
+    $1.downcase # Can't run into NilClass problems.
+  end
+
+  def ssh_key_matches?(other_cred)
+    return false unless other_cred.kind_of? self.class
+    return false unless self.ptype == other_cred.ptype
+    case self.ptype
+      when "ssh_key"
+        matches = self.ssh_private_keys
+      when "ssh_pubkey"
+        matches = self.ssh_public_keys
+      else
+        false
+    end
+    matches.include?(self) and matches.include?(other_cred)
+  end
+
+  # Returns all keys with matching key ids, including itself
+  # If this is not an ssh-type key, always returns an empty array.
+  def ssh_keys
+    (self.ssh_private_keys | self.ssh_public_keys)
+  end
+
+  # Returns all private keys with matching key ids, including itself
+  # If this is not an ssh-type key, always returns an empty array.
+  def ssh_private_keys
+    return [] unless self.ssh_key_id
+    matches = self.class.all(
+        :conditions => ["creds.ptype = ? AND creds.proof ILIKE ?", "ssh_key", "%#{self.ssh_key_id}%"]
+    )
+    matches.select {|c| c.workspace == self.workspace}
+  end
+
+  # Returns all public keys with matching key ids, including itself
+  # If this is not an ssh-type key, always returns an empty array.
+  def ssh_public_keys
+    return [] unless self.ssh_key_id
+    matches = self.class.all(
+        :conditions => ["creds.ptype = ? AND creds.proof ILIKE ?", "ssh_pubkey", "%#{self.ssh_key_id}%"]
+    )
+    matches.select {|c| c.workspace == self.workspace}
+  end
+
+  # Returns its workspace
+  def workspace
+    self.service.host.workspace
+  end
+
+  ActiveSupport.run_load_hooks(:mdm_cred, self)
+end

data/app/models/mdm/event.rb

@@ -0,0 +1,30 @@
+class Mdm::Event < ActiveRecord::Base
+  #
+  # Relations
+  #
+
+  belongs_to :host, :class_name => 'Mdm::Host'
+  belongs_to :workspace, :class_name => 'Mdm::Workspace'
+
+  #
+  # Scopes
+  #
+
+  scope :flagged, where(:critical => true, :seen => false)
+  scope :module_run, where(:name => 'module_run')
+
+  #
+  # Serializations
+  #
+
+  serialize :info, MetasploitDataModels::Base64Serializer.new
+
+  #
+  # Validations
+  #
+
+  validates :name, :presence => true
+
+  ActiveSupport.run_load_hooks(:mdm_event, self)
+end
+

data/app/models/mdm/exploit_attempt.rb

@@ -0,0 +1,14 @@
+class Mdm::ExploitAttempt < ActiveRecord::Base
+  #
+  # Relations
+  #
+  belongs_to :host, :class_name => 'Mdm::Host', :counter_cache => :exploit_attempt_count
+
+  #
+  # Validations
+  #
+
+  validates :host_id, :presence => true
+
+  ActiveSupport.run_load_hooks(:mdm_exploit_attempt, self)
+end

data/app/models/mdm/exploited_host.rb

@@ -0,0 +1,11 @@
+class Mdm::ExploitedHost < ActiveRecord::Base
+  #
+  # Relations
+  #
+
+  belongs_to :host, :class_name => 'Mdm::Host'
+  belongs_to :service, :class_name => 'Mdm::Service'
+  belongs_to :workspace, :class_name => 'Mdm::Workspace'
+
+  ActiveSupport.run_load_hooks(:mdm_exploited_host, self)
+end

data/app/models/mdm/host.rb

@@ -0,0 +1,134 @@
+
+class Mdm::Host < ActiveRecord::Base
+  include Mdm::Host::OperatingSystemNormalization
+
+  #
+  # Callbacks
+  #
+
+  before_destroy :cleanup_tags
+
+  #
+  # CONSTANTS
+  #
+
+  # Fields searched for the search scope
+  SEARCH_FIELDS = [
+      'address::text',
+      'hosts.name',
+      'os_name',
+      'os_flavor',
+      'os_sp',
+      'mac',
+      'purpose',
+      'comments'
+  ]
+
+  #
+  # Relations
+  #
+
+  has_many :exploit_attempts, :dependent => :destroy, :class_name => 'Mdm::ExploitAttempt'
+  has_many :exploited_hosts, :dependent => :destroy, :class_name => 'Mdm::ExploitedHost'
+  has_many :clients, :dependent => :delete_all, :class_name => 'Mdm::Client'
+  has_many :host_details, :dependent => :destroy, :class_name => 'Mdm::HostDetail'
+  # hosts_tags are cleaned up in before_destroy:
+  has_many :hosts_tags, :class_name => 'Mdm::HostTag'
+  has_many :loots, :dependent => :destroy, :class_name => 'Mdm::Loot', :order => 'loots.created_at desc'
+  has_many :notes, :dependent => :delete_all, :class_name => 'Mdm::Note', :order => 'notes.created_at'
+  has_many :services, :dependent => :destroy, :class_name => 'Mdm::Service', :order => 'services.port, services.proto'
+  has_many :sessions, :dependent => :destroy, :class_name => 'Mdm::Session', :order => 'sessions.opened_at'
+  has_many :vulns, :dependent => :delete_all, :class_name => 'Mdm::Vuln'
+  belongs_to :workspace, :class_name => 'Mdm::Workspace'
+
+  #
+  # Through host_tags
+  #
+  has_many :tags, :through => :hosts_tags, :class_name => 'Mdm::Tag'
+
+  #
+  # Through services
+  #
+  has_many :creds, :through => :services, :class_name => 'Mdm::Cred'
+  has_many :service_notes, :through => :services
+  has_many :web_sites, :through => :services, :class_name => 'Mdm::WebSite'
+
+  #
+  # Nested Attributes
+  # @note Must be declared after relations being referenced.
+  #
+
+  accepts_nested_attributes_for :services, :reject_if => lambda { |s| s[:port].blank? }, :allow_destroy => true
+
+  #
+  # Validations
+  #
+
+  validates :address,
+            :exclusion => {
+                :in => ['127.0.0.1']
+            },
+            :ip_format => true,
+            :presence => true,
+            :uniqueness => {
+                :scope => :workspace_id,
+                :unless => :ip_address_invalid?
+            }
+  validates :workspace, :presence => true
+
+  #
+  # Scopes
+  #
+
+  scope :alive, where({'hosts.state' => 'alive'})
+  scope :flagged, where('notes.critical = true AND notes.seen = false').includes(:notes)
+  scope :search,
+        lambda { |*args|
+          # @todo replace with AREL
+          terms = SEARCH_FIELDS.collect { |field|
+            "#{field} ILIKE ?"
+          }
+          disjunction = terms.join(' OR ')
+          formatted_parameter = "%#{args[0]}%"
+          parameters = [formatted_parameter] * SEARCH_FIELDS.length
+          conditions = [disjunction] + parameters
+
+          {
+              :conditions => conditions
+          }
+        }
+  scope :tag_search,
+        lambda { |*args| where("tags.name" => args[0]).includes(:tags) }
+
+  def attribute_locked?(attr)
+    n = notes.find_by_ntype("host.updated.#{attr}")
+    n && n.data[:locked]
+  end
+
+  def cleanup_tags
+    # No need to keep tags with no hosts
+    tags.each do |tag|
+      tag.destroy if tag.hosts == [self]
+    end
+    # Clean up association table records
+    Mdm::HostTag.delete_all("host_id = #{self.id}")
+  end
+
+  # This is replicated by the IpAddressValidator class. Had to put it here as well to avoid
+  # SQL errors when checking address uniqueness.
+  def ip_address_invalid?
+    begin
+      potential_ip = IPAddr.new(address)
+      return true unless potential_ip.ipv4? || potential_ip.ipv6?
+    rescue ArgumentError
+      return true
+    end
+  end
+
+  def is_vm?
+    !!self.virtual_host
+  end
+
+  ActiveSupport.run_load_hooks(:mdm_host, self)
+end
+