honeycomb 0.0.1

data/README.rdoc

@@ -0,0 +1,72 @@
+= honeycomb
+Josh Grunzweig - jgrunzweig at trustwave dot com
+
+== Introduction
+
+Tool to manage and analyze data from the Dionaea Honeypot Project
+
+The Dionaea Project is a great honeypot which originated from the Google Summer
+of Code. The project aims at obtaining malware samples by emulating Microsoft
+Windows services, however, has branched out since its create to emulate other
+services as well, such as MySQL. This project was created out of the necessity
+to monitor and manage multiple instances of the Dionaea on honeypots located 
+around the world. 
+
+You can view more information about the Dionaea at the following address:
+http://dionaea.carnivore.it/
+
+
+== Usage
+
+  # basic example... See how much disk space is located on all honeypots
+  
+  require 'honeycomb'
+
+  all_pots = Honeycomb::Interact.new
+
+  all_pots.check_diskspace
+
+ 
+  # another example... See how many instances of a specific md5 there are
+
+  require 'honeycomb'
+
+  all_pots = Honeycomb::Interact.new
+
+  all_pots.all{Honeycomb::Download.all(:download_md5_hash => "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")}
+
+== Versioning
+The current version of honeycomb at the time of writing is 0.0.1. Code will be 
+buggy. Features may be lacking. However, in time I hope to make this a pretty
+functional/useful tool. 
+
+
+== Requirements
+There is, unfortunately, a number of gem pre-requisites. Apologies for this,
+but it's the only way I could do it. 
+
+* bundler
+
+After bundler is installed, do a "bundle install" to get everything installed.
+
+
+== Copyright
+honeycomb - Tool to manage and analyze data from the Dionaea Honeypot Project
+Josh Grunzweig
+Copyright (C) 2011 Trustwave Holdings
+
+This program is free software: you can redistribute it and/or modify it 
+under the terms of the GNU General Public License as published by the 
+Free Software Foundation, either version 3 of the License, or (at your
+option) any later version.
+
+This program is distributed in the hope that it will be useful, but 
+WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program. If not, see <http://www.gnu.org/licenses/>.
+
+See LICENSE.txt
+

data/Rakefile

@@ -0,0 +1,52 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+require 'rake'
+
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+
+Dir["tasks/*.rake"].each { |taskfile| load taskfile }
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "honeycomb"
+  gem.homepage = "http://github.com/spiderlabs/honeycomb"
+  gem.license = "GNU v3"
+  gem.summary = %Q{Tool to manage and analyze data from the Dionaea Honeypot Project}
+  gem.description = %Q{Tool to manage and analyze data from the Dionaea Honeypot Project}
+  gem.email = "jgrunzweig at trustwave dot com"
+  gem.authors = ["Josh Grunzweig"]
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+
+RSpec::Core::RakeTask.new(:rcov) do |spec|
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :default => :spec
+
+require 'rdoc/task'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "honeycomb #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/etc/config.yml.example

@@ -0,0 +1,11 @@
+honey_config:
+   servers:  ['honeypot1', 'honeypot2']
+   username: 'r00t'
+   key:      'path/to/key'
+   password: 'sekret'
+   path:     'path/to/dionaea'
+   # Uncomment to following to change default directory of 
+   # honeycomb/data/binaries
+   # honeycomb/data/logsql
+   #download_binaries: 'where/to/store/binaries'
+   #download_databases: 'where/to/store/databases'

data/lib/honeycomb.rb

@@ -0,0 +1,24 @@
+#    honeycomb - Tool to manage and analyze data from the Dionaea Honeypot 
+#                Project
+#    Josh Grunzweig
+#    Copyright (C) 2011 Trustwave Holdings
+#
+#    This program is free software: you can redistribute it and/or modify it 
+#    under the terms of the GNU General Public License as published by the 
+#    Free Software Foundation, either version 3 of the License, or (at your
+#    option) any later version.
+#
+#    This program is distributed in the hope that it will be useful, but 
+#    WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+#    or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+#    for more details.
+#
+#    You should have received a copy of the GNU General Public License along
+#    with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+require 'honeycomb/interact'
+require 'honeycomb/environment'
+require 'honeycomb/default_setup'
+require 'honeycomb/model'
+

data/lib/honeycomb/default_setup.rb

@@ -0,0 +1,25 @@
+#    honeycomb - Tool to manage and analyze data from the Dionaea Honeypot 
+#                Project
+#    Josh Grunzweig
+#    Copyright (C) 2011 Trustwave Holdings
+#
+#    This program is free software: you can redistribute it and/or modify it 
+#    under the terms of the GNU General Public License as published by the 
+#    Free Software Foundation, either version 3 of the License, or (at your
+#    option) any later version.
+#
+#    This program is distributed in the hope that it will be useful, but 
+#    WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+#    or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+#    for more details.
+#
+#    You should have received a copy of the GNU General Public License along
+#    with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+require 'honeycomb'
+require 'honeycomb/model'
+
+Honeycomb::Env.read_config
+Honeycomb::Model.setup!
+

data/lib/honeycomb/environment.rb

@@ -0,0 +1,64 @@
+#    honeycomb - Tool to manage and analyze data from the Dionaea Honeypot 
+#                Project
+#    Josh Grunzweig
+#    Copyright (C) 2011 Trustwave Holdings
+#
+#    This program is free software: you can redistribute it and/or modify it 
+#    under the terms of the GNU General Public License as published by the 
+#    Free Software Foundation, either version 3 of the License, or (at your
+#    option) any later version.
+#
+#    This program is distributed in the hope that it will be useful, but 
+#    WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+#    or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+#    for more details.
+#
+#    You should have received a copy of the GNU General Public License along
+#    with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+require 'pathname'
+require 'yaml'
+
+module Honeycomb
+
+  module Environment
+    CONFIG = {}
+    CFGFILE = Pathname.new(__FILE__).dirname.dirname.dirname.expand_path.join('etc').join('config.yml')
+    
+    # TODO: Comment
+    def self.load_config(hash)
+      hash.each do |k,v|
+        if v.is_a?(String)
+          v = v.gsub(/\$([A-Z][A-Z0-9_]*)\$/) do |v|
+            var = $1
+            if const_defined?(var)
+              const_get(var).to_s
+            else
+              raise("Invalid variable referenced in configuration: #{v}")
+            end
+          end
+        end
+        CONFIG[k.to_s] = v
+      end
+    end
+
+    # TODO: Comment
+    def self.read_config(file=CFGFILE)
+      if h = YAML.load_file(file) 
+        if h.is_a?(Hash)
+          load_config(h)
+        else
+          raise("invalid honeycomb config file format")
+        end
+      end
+    end
+
+  end
+
+  Env = Environment
+    
+end
+
+
+

data/lib/honeycomb/interact.rb

@@ -0,0 +1,20 @@
+#    honeycomb - Tool to manage and analyze data from the Dionaea Honeypot 
+#                Project
+#    Josh Grunzweig
+#    Copyright (C) 2011 Trustwave Holdings
+#
+#    This program is free software: you can redistribute it and/or modify it 
+#    under the terms of the GNU General Public License as published by the 
+#    Free Software Foundation, either version 3 of the License, or (at your
+#    option) any later version.
+#
+#    This program is distributed in the hope that it will be useful, but 
+#    WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+#    or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+#    for more details.
+#
+#    You should have received a copy of the GNU General Public License along
+#    with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+require 'honeycomb/interact/interact'

data/lib/honeycomb/interact/interact.rb

@@ -0,0 +1,392 @@
+#    honeycomb - Tool to manage and analyze data from the Dionaea Honeypot 
+#                Project
+#    Josh Grunzweig
+#    Copyright (C) 2011 Trustwave Holdings
+#
+#    This program is free software: you can redistribute it and/or modify it 
+#    under the terms of the GNU General Public License as published by the 
+#    Free Software Foundation, either version 3 of the License, or (at your
+#    option) any later version.
+#
+#    This program is distributed in the hope that it will be useful, but 
+#    WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+#    or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+#    for more details.
+#
+#    You should have received a copy of the GNU General Public License along
+#    with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+require 'net/ssh'
+require 'net/scp'
+require 'open3'
+
+module Honeycomb
+
+  class Interact
+
+      attr_accessor :db_path, :bin_path, :base_path, :username, :servers, :key
+
+      # This initializes a Honeycomb::Interact object and sets all the necessary
+      # variables which are used by other methods of the object.
+      #
+      # Variables and their purpose:
+      # * db_path - Path where databases are stored/saved
+      # * bin_path - Path where binaries are stored/saved
+      # * username - Username to connect to remote honeypot servers
+      # * key - Path to private key which is used for connections to honeypot 
+      # servers
+      # * servers - Array of servers to connect to
+      # * base_path - Base location where Dionaea is installed to (Default per 
+      #   installation instructions: /opt/dionaea)
+      def initialize(db_path = nil, bin_path = nil, username = nil, key = nil, 
+                     servers = nil, base_path = nil)
+        self.db_path = Honeycomb::Env::CONFIG[:download_databases] ||
+          self.db_path = Pathname.new(__FILE__).dirname.dirname.dirname.dirname.expand_path.join('data').join('logsql/').to_s ||
+            db_path
+        self.bin_path = Honeycomb::Env::CONFIG[:download_binaries] ||
+          self.bin_path = Pathname.new(__FILE__).dirname.dirname.dirname.dirname.expand_path.join('data').join('binaries/').to_s ||
+            bin_path
+        self.username = Honeycomb::Env::CONFIG["honey_config"]["username"] || 
+          username
+        self.key = Honeycomb::Env::CONFIG["honey_config"]["key"] || key
+        self.servers = Honeycomb::Env::CONFIG["honey_config"]["servers"] || 
+          servers
+        self.base_path = Honeycomb::Env::CONFIG["honey_config"]["path"] ||
+          base_path
+      end
+
+      # This method will attempt to download all binaries from all servers 
+      # specified in Honeycomb::Interact.servers.
+      #
+      # It will attempt to store all binaries into the folder specified in
+      # Honeycomb::Interact.bin_path.
+      #
+      # Additionally, rsync is utilized to transfer these files. It was 
+      # chosen to use rsync over scp in order to limit the amount of 
+      # bandwidth used between the client and servers.
+      #
+      # Arguments:
+      # * server - Array of servers to query
+      def download_binaries(server = self.servers)
+        server.each do |server|
+          tries = 0
+          puts "Downloading binaries from #{server} ..."
+          begin
+            Open3::popen3("rsync -v --force --ignore-errors --times -r -u -e \"ssh -i #{self.key}\" #{self.username}@#{server}:#{self.base_path}/var/dionaea/binaries/ #{self.bin_path}") { |stdin, stdout, stderr|
+              puts stdout.read.strip
+              puts stderr.read.strip
+            }
+          rescue
+            tries += 1
+            retry if tries <= 3
+            puts "Unable to connect. Moving on ..."
+            next
+          end
+        end
+      end 
+
+      # This method will attempt to download all databases from all servers 
+      # specified in Honeycomb::Interact.servers.
+      #
+      # It will attempt to store all binaries into the folder specified in
+      # Honeycomb::Interact.db_path.
+      #
+      # Additionally, scp is utilized to transfer these files. During tests,
+      # it was discovered that rsync had less than ideal results when 
+      # downloading these files. While the transfer would appear to occur 
+      # without error, the databases were often found to be corrupt.
+      #
+      # Arguments:
+      # * server - Array of servers to query
+      def download_databases(server = self.servers)
+        server.each do |server|
+          tries = 0
+          begin
+            Net::SSH.start(server, self.username, :keys => self.key) do |session|
+              puts "Downloading database from #{server} ..."
+              session.scp.download!(base_path + "/var/dionaea/logsql.sqlite", 
+                                    self.db_path + "#{server}.sqlite")
+            end
+          rescue Errno::ETIMEDOUT
+            tries += 1
+            retry if tries <= 3
+            puts "Unable to connect. Moving on ..."
+            next
+          rescue Exception => e
+            puts "Error encountered: #{e.message}"
+            next
+          end
+        end
+      end
+
+      # This method will execute a command via ssh on all servers specified in
+      # the Honeycomb::Interact.servers variable. This command calls the internal
+      # ssh_command method in order to properly function.
+      #
+      # Argument:
+      # * command - Command to execute
+      #
+      # Returns:
+      # * Nothing
+      #
+      # Multiple strings with the results are outputted to the screen.
+      def execute_command(command)
+        response = self.ssh_command(command)
+        response.each do |server_hash|
+          puts "Executing #{command} on #{server_hash[:server]}:"
+          puts "\t#{server_hash[:result].gsub!(/\n/,"\n\t")}"
+        end
+      end # end execute_command
+
+      # This method will query the diskspace on all remote servers by calling
+      # the internal ssh_command method. It executes the command 'df -h /' and
+      # parses the results. The response is then parsed to return the total 
+      # percentage of diskspace being used currently on each host.
+      #
+      # Arguments:
+      # * None
+      #
+      # Returns:
+      # * [ {:server => "Server Hostname", :result => 
+      #
+      # Multiple strings with the results are outputted to the screen.
+      def check_diskspace
+        response = self.ssh_command("df -h /") 
+        all_usage = []
+        response.each do |server_hash|
+          usage = server_hash[:result]
+          if usage =~ /^(\/\w+)+.+\S+\s+\S+\s+\S+\s+(([0-9]+)%)/m
+            all_usage << {:server => server_hash[:server], :result => $2}
+          end
+        end
+        all_usage
+      end # end check_diskspace
+
+      # Only leaving this in here because it does a decent job of showing some 
+      # of the stuff you can do with these DataMapper database bindings.
+      #
+      # This method was mainly created to be utilized for another project. It 
+      # takes the md5 as in argument and performs a series of queries against
+      # all of the database to retrieve a large amount of data about that
+      # provided binary (in md5 checksum)
+      #
+      # Multiple encounters can be returned. These encounters are uniqued based
+      # on the URL of the download. Additionally, if duplicates of a given url
+      # are discovered, the encounter with the earliest timestamp is added to
+      # the hash.
+      #
+      # Argument:
+      # * md5 - MD5 String of the binary to be examined.
+      #
+      # Returns:
+      # * Hash - 
+      #   {:md5 => <md5_provided>, 
+      #    :encounters = [{ :url => <url_discovered>,
+      #                     :original_filename => <md5_provided>,
+      #                     :remote_host => <ip_address>,
+      #                     :source_timestamp => <Time>,
+      #                     :source => "Honeypot - <ip_address>",
+      #                     :virustotal => {:url => <url>,
+      #                       :timestamp => <timestamp_of_vt_scan>,
+      #                       :results => {:scanner => <scanner>,
+      #                                    :result => <result>}
+      #                       }
+      #                   }
+      def self.get_md5_info(md5)
+        all_encounters = []
+        all_instances = self.all{Honeycomb::Download.all(:download_md5_hash => md5)}
+        all_instances.each do |instance|
+          all_encounters << {:url => instance.download_url}
+        end
+        num = 0 
+        all_encounters.uniq! {|e| e[:url] }
+        all_encounters.each do |url|
+          all_connections = self.all{Honeycomb::Download.all(:download_md5_hash => md5, 
+                                                             :download_url => url[:url]).connections}
+          connection = {}
+          all_connections.each do |conn|
+            if not connection[:source_timestamp].nil?
+              if Time.at(connection[:source_timestamp]) > Time.at(conn.connection_timestamp.to_i)
+                connection[:source_timestamp] = Time.at(conn.connection_timestamp.to_i)
+                connection[:remote_host] = conn.remote_host
+                connection[:source] = conn.local_host
+              end
+            else
+              connection[:source_timestamp] = Time.at(conn.connection_timestamp.to_i)
+              connection[:remote_host] = conn.remote_host
+              connection[:source] = conn.local_host
+            end
+          end
+          all_encounters[num][:original_filename] = md5
+          all_encounters[num][:remote_host] = connection[:remote_host]
+          all_encounters[num][:source_timestamp] = connection[:source_timestamp]
+          all_encounters[num][:source] = "Honeypot - #{connection[:source]}"
+          virustotal_links = self.all{Honeycomb::Virustotal.first(:virustotal_md5_hash => md5).virustotal_permalink}
+          virustotal_timestamp = self.all{Honeycomb::Virustotal.first(:virustotal_md5_hash => md5).virustotal_timestamp}
+          virustotal_results = {}
+          self.all{Honeycomb::Virustotal.first(:virustotal_md5_hash => md5).virustotalscans}.each do |vtscan|
+            virustotal_results[:scanner] = vtscan.virustotalscan_scanner
+            virustotal_results[:result] = vtscan.virustotalscan_result
+          end
+          all_encounters[num][:virustotal] = {:url => virustotal_links, 
+            :timestamp => virustotal_timestamp, :results => virustotal_results} if virustotal_links
+          num += 1
+        end
+        return {:md5 => md5, :encounters => all_encounters}
+      end
+
+
+
+      # This method will call the get_md5_info method on all binaries located
+      # in the provided directory. This information is then outputted to screen 
+      # (soon to be changed).
+      #
+      # Again, only leaving this in here because it does a decent job of showing
+      # the kind of information you can pull. 
+      #
+      # Argument:
+      # * dif - Directory of binaries 
+      #
+      # Retruns:
+      # * Nothing
+      #
+      # Multiple items are outputted to the screen. 
+      def self.get_all_md5_info(dir = Pathname.new(__FILE__).dirname.dirname.dirname.dirname.expand_path.join('data').join('binaries/').to_s)
+        results = []
+        all_binaries = Dir.entries(dir)
+        all_binaries.each do |bin|
+          if bin =~ /\w{32}/
+            require 'pp'
+            pp self.get_md5_info(bin)
+          end
+        end
+      end
+
+      # Used for executing a query against all databases at once. 
+      def all(&block)
+        all_values = []
+        ::DataMapper::Repository.adapters.each do |repo|
+          if repo[0] == :default
+            next
+          end
+          begin
+            response = DataMapper.repository(repo[0]) {yield}
+            if response.kind_of?(DataMapper::Collection)
+              response.each do |x|
+                all_values << x 
+              end
+            else
+              all_values << response if response
+            end
+          rescue Exception => e
+            #puts e.message
+          end
+        end
+        all_values
+      end
+
+      # Used for executing a query against a single database. 
+      def individual(repo, &block)
+        all_values = []
+        begin
+          response = DataMapper.repository(repo[0]) {yield}
+          if response.kind_of?(DataMapper::Collection)
+            response.each do |x|
+            all_values << x 
+            end
+          else
+            all_values << response if response
+          end
+        rescue Exception => e
+          #puts e.message
+        end
+        all_values
+      end
+
+      # This method was created mainly for my own benefit, but I figured I'd 
+      # leave it in here in case anyone else would like to use it. Was doing 
+      # some statistics for a co-worker who was looking for information about
+      # the IP addresses which has connected to my honeypots.
+      #
+      # It's a little clunky right now, and I ended up just performing actual
+      # SQL queries due to memory issues (Some of my databases are over 10 gigs
+      # in size). It will write a list of IP addresses and the number of times 
+      # encountered to the following log file:
+      #
+      # ip_reputation.txt
+      #
+      # The format of the data is:
+      #
+      # <ip_address>,<count>
+      #
+      # In order from most encountered to least encountered. I also ignore 
+      # people that have connected to these honeypots less than 200 times. 
+      #
+      def self.ip_reputation_rule
+        all_connections_hash = {}
+
+        ::DataMapper::Repository.adapters.each do |repo|
+          if repo[0] == :default
+            next
+          end
+          response = repo[1].select("SELECT COUNT(remote_host), remote_host FROM connections WHERE connection_type = \"accept\" GROUP BY remote_host ORDER BY COUNT(remote_host)")
+          response.each do |struct|
+            ip = struct.to_a[1]
+            count = struct.to_a[0]
+            if all_connections_hash[ip]
+              all_connections_hash[ip] = all_connections_hash[ip].to_i + 
+                count.to_i
+            else
+              all_connections_hash[ip] = count.to_i
+            end
+          end
+        end
+        all_connections_hash = all_connections_hash.sort_by { |k,v| -1*v }
+        File.open("ip_reputation.txt", 'w') do |f|
+          all_connections_hash.each do |ip,count|
+            if count > 200
+              f.write("#{ip},#{count}\n") 
+            end
+          end
+        end   
+      end
+
+      # This method is used internally by the execute_command method.
+      # It will take a command as an argument and execute it on ever server
+      # that is stored in Honeycomb::Interact.servers. The results are 
+      # stored in a hash which is returned in an Array.
+      #
+      # Argument:
+      # * command - Command to be executed
+      #
+      # Returns:
+      # * Array of hashes -
+      #   [{:server => <server_name>, :result => <result_of_command>}]
+      def ssh_command(command)
+        results = []
+        self.servers.each do |server|
+          begin
+            Net::SSH.start(server, self.username, :keys => self.key) do |session|
+              session.exec command do |ch, stream, data|
+              if stream == :stderr
+                  results << {:server => server, :result => "ERROR: #{data}"}
+                else
+                  results << {:server => server, :result => data}
+                end
+              end
+            end
+          rescue 
+            next
+          end
+        end
+        return results
+      end
+
+
+  end # end Interact
+
+end # end Honeycomb
+
+
+