dorothy2 1.2.0 → 2.0.0

data/etc/sources.yml.example

@@ -4,7 +4,7 @@
 ###
 ### type means the communication channel used
 ### to download the binaries, possible values
-### are: system | ssh
+### are: system | ssh |  mail
 ###
 ### typeid defines the type of the source, it
 ### depends on a userdefined-type in
@@ -17,10 +17,20 @@
 ### 2  - unknown
 #############################################
 ---
+#dont change the name of the webgui source, or if wont work. Change only the local path, the priority and the profile
+webgui:
+    type: web
+    localdir: /opt/dorothy2/opt/bins/web
+    typeid: 1
+
+#Examples below
 malwarefolder:
     type: system
-    localdir: /Users/m4rco/dorothy2/opt/bins/manual
+    localdir: /opt/dorothy2/opt/bins/manual
     typeid: 2
+    priority: 1
+    profile: test
+
 honeypot1:
     type: ssh
     ip: 1.2.3.4
@@ -30,3 +40,18 @@ honeypot1:
     remotedir: /asda/bins
     localdir: /opt/bins/honeypot
     typeid: 0
+    priority: 1
+    profile: test
+
+mailpot@example.com:
+    type: mail
+    localdir: /opt/dorothy2/opt/bins/mailpot@example.com
+    typeid: 1
+    address: pop-mail.outlook.com
+    username:  mailpot@example.com
+    password: myPASS
+    port: 995
+    enable_ssl: true
+    metodo_analisi: 1
+    priority: 2
+    profile: test

data/lib/doroGUI.rb

@@ -0,0 +1,232 @@
+require 'sinatra'
+require 'sinatra/activerecord'
+require 'sinatra/namespace'
+
+
+module Dorothy
+  class DoroGUI < Sinatra::Base
+    register Sinatra::Namespace
+
+    enable :logging, :dump_errors
+    file = File.new(DoroSettings.wgui[:logfile], 'a+')
+    file.sync = true
+    use Rack::CommonLogger, file
+    before { env['rack.errors'] = file  }
+
+    set :app_file, __FILE__
+    root = File.expand_path(File.dirname(__FILE__))
+    set :public_folder, File.join(root, 'www/public')
+    set :views, Proc.new { File.join(root, 'www/views') }
+
+
+    ActiveRecord::Base.establish_connection(
+        adapter:    'postgresql',
+        host:       DoroSettings.dorothive[:dbhost],
+        database:   DoroSettings.dorothive[:dbname],
+        username:   DoroSettings.dorothive[:dbuser],
+        password:   DoroSettings.dorothive[:dbpass],
+        port:       5432,
+        schema_search_path: 'dorothy' )
+
+    sources = YAML.load_file(DoroSettings.env[:home] + '/etc/sources.yml')
+
+
+    class Analyses < ActiveRecord::Base
+      self.table_name = "analyses"
+    end
+
+    class Samples < ActiveRecord::Base
+      self.table_name = "samples"
+    end
+
+    class Flows < ActiveRecord::Base
+      self.table_name = "flows"
+    end
+
+    class Sandboxes < ActiveRecord::Base
+      self.table_name = "sandboxes"
+    end
+
+    class AnalysisQueue < ActiveRecord::Base
+      self.table_name = "analysis_queue"
+    end
+
+    class TrafficDumps < ActiveRecord::Base
+      self.table_name = "traffic_dumps"
+    end
+
+    class Sources < ActiveRecord::Base
+      self.table_name = "sources"
+    end
+
+    class Emails < ActiveRecord::Base
+      self.table_name = "emails"
+    end
+
+    class Receivers < ActiveRecord::Base
+      self.table_name = "email_receivers"
+    end
+
+    class Sightings < ActiveRecord::Base
+      self.table_name = "sightings"
+    end
+
+    class SystemProcs < ActiveRecord::Base
+      self.table_name = "sys_procs"
+    end
+
+    class Malwares < ActiveRecord::Base
+      self.table_name = "malwares"
+    end
+
+    class AvSigns < ActiveRecord::Base
+      self.table_name = "av_signs"
+    end
+
+    get '/' do
+      @title = "Analyses"
+      @analyses = Analyses.all
+      @samples  = Samples.all
+      @queue = AnalysisQueue.all
+      @sightings = Sightings.all
+      @emails = Emails.all
+
+
+
+      erb :analyses
+    end
+
+
+
+    get '/queue' do
+      @title = "Queue Status"
+      @queue = AnalysisQueue.all.order(id: :asc, priority: :desc)
+      @sources = Sources.all
+      @analyses = Analyses.all
+      @emails = Emails.all
+      erb :queue
+    end
+
+
+    get '/resume/:analid' do
+      @title = "Sample Analysis Resume"
+      @analysis_dir = DoroSettings.env[:analysis_dir]
+      @analid = params[:analid]
+      @sample = Samples.where(:sha256 => Analyses.where(:id => @analid).first.sample).first
+      @sys_procs = SystemProcs.where(:analysis_id => params[:analid])
+      @malware = Malwares.where(:bin => Analyses.where(:id => @analid).first.sample).first
+      @sophos = @malware.nil? ? nil : AvSigns.where(:id => @malware.id).where(:av_name => 'Sophos').first
+
+      @mailid = Sightings.where(:id => AnalysisQueue.where( :id => Analyses.where(:id => @analid).first.queue_id).first.sighting).first.src_email
+
+
+
+      @net_dumps = TrafficDumps.where(:sha256 => Analyses.where(:id => @analid).first.traffic_dump)
+      @flows= Flows.where(:traffic_dump => Analyses.where(:id => @analid).first.traffic_dump)
+
+      @imgs = []
+      Dir[DoroSettings.env[:analysis_dir] + "/#{@analid}/screens/*.png"].each  {|file| @imgs.push(File.basename(file))  }
+
+      erb :resume
+    end
+
+    get '/screens/:analid/:name' do
+      full_path = DoroSettings.env[:analysis_dir] + "/" + params[:analid] + "/screens/" + params[:name]
+      send_file full_path.strip, :filename => params[:name].strip, :disposition => 'inline'
+    end
+
+
+    get '/profile/:profile' do
+      @profile = Util.load_profile(params[:profile])
+
+      erb :profile
+    end
+
+
+    get '/upload' do
+      @sandboxes = Sandboxes.all
+      @profiles = YAML.load_file(DoroSettings.env[:home] + '/etc/profiles.yml')
+
+      erb :upload
+    end
+
+    post '/upload' do
+      localpath = sources["webgui"]["localdir"] + "/#{params[:uploaded_data][:filename]}"
+      FileUtils.mv(params[:uploaded_data][:tempfile].path, localpath) unless params[:uploaded_data].nil?
+      id = QueueManager.add(localpath, 'webgui', params[:profile], params[:priority])
+
+      #entry = AnalysisQueue.create(date: get_time, binary: localpath, filename: filename, source: "webgui", priority: params[:priority], profile: params[:OS], user: "webuser")
+      #entry.save
+      erb "Upload Complete. Prio #{params[:priority]} - OS #{params[:OS]} - Scheduled with Queue ID #{id}"
+    end
+
+
+
+    namespace '/email' do
+      get '/view/:mail_id' do
+        @email = Emails.where(:id => params[:mail_id]).first
+        @receivers = Receivers.where(:email_id => params[:mail_id])
+        erb :email
+      end
+
+      get '/download/:mail_id' do
+        email = Emails.where(:id => params[:mail_id]).first
+
+        content_type 'Application/octet-stream'
+        attachment( "Message_#{email.id}" + '.eml')
+        PG::Connection.unescape_bytea(email.data)
+      end
+
+    end
+
+
+
+    namespace '/samples/' do
+
+      get '/' do
+        @title = "Sample Information"
+        @samples  = Samples.all
+        erb :samples
+      end
+
+      get ':sha256' do |sha256|
+        @samples = Samples.where(:sha256 => params[:sha256])
+
+        #list all the analysis that have been done on this file , including timestamp, OS, etc
+        erb :samples
+      end
+
+
+      get 'download/:sha256' do |sha256|
+        first_id = Analyses.where(:sample => params[:sha256]).first.id
+        filename = Samples.where(:sha256 => params[:sha256]).first.filename
+        full_path = DoroSettings.env[:analysis_dir] + "/#{first_id}/bin/" + filename
+        send_file full_path.strip, :filename => filename, :type => 'Application/octet-stream'
+      end
+
+    end
+
+
+    namespace '/net' do
+
+      namespace '/:dump_sha1' do
+
+        get '/' do
+          @title = "Network Flows"
+          @net_dumps = TrafficDumps.where(:sha256 => params[:dump_sha1])
+          @flows= Flows.where(:traffic_dump => params[:dump_sha1])
+
+          erb :flows
+        end
+
+      end
+    end
+
+
+    after do
+      ActiveRecord::Base.connection.close
+    end
+
+  end
+
+end

data/lib/doroParser.rb

@@ -13,26 +13,18 @@
 ## Data Definition Module ##
 ############################
 
-require 'digest'
-require 'rbvmomi'
-require 'rest_client'
+
 require 'net/dns'
 require 'net/dns/packet'
 require 'ipaddr'
-require 'colored'
-require 'filemagic'
 require 'geoip'
-require 'pg'
-require 'tmail'
 require 'ipaddr'
 require 'net/http'
 require 'json'
 
 require File.dirname(__FILE__) + '/mu/xtractr'
 require File.dirname(__FILE__) + '/dorothy2/DEM'
-require File.dirname(__FILE__) + '/dorothy2/do-utils'
-require File.dirname(__FILE__) + '/dorothy2/do-logger'
-require File.dirname(__FILE__) + '/dorothy2/deep_symbolize'
+
 
 
 module DoroParser
@@ -41,6 +33,7 @@ module DoroParser
   CCIRC = 1
   CCDROP = 3
   CCSUPPORT = 5
+  NONETBIOS = true
 
   def search_irc(streamdata)
 
@@ -53,14 +46,13 @@ module DoroParser
 
       ircvalues.push "default, currval('dorothy.connections_id_seq'), E'#{Insertdb.escape_bytea(m[0])}', #{direction_bool}"
     end
-    return ircvalues
+    ircvalues
   end
 
   def analyze_bintraffic(pcaps)
 
     dns_list = Hash.new
     hosts = []
-    @insertdb.begin_t
 
     pcaps.each do |dump|
       #RETRIVE MALWARE FILE INFO
@@ -92,12 +84,12 @@ module DoroParser
       #The following section is to avoid a crash while quering such (still-empty instance)
       #In addition, an added check is inserted, to see if the pcapr instance really match the pcap filename
       begin
-      pcapr_query = URI.parse "http://#{DoroSettings.pcapr[:host]}:#{DoroSettings.pcapr[:port]}/pcaps/1/about/#{dump['pcapr_id'].rstrip}"
-      pcapr_response = Net::HTTP.get_response(pcapr_query)
-      pcapname = File.basename(JSON.parse(pcapr_response.body)["filename"], ".pcap")
+        pcapr_query = URI.parse "http://#{DoroSettings.pcapr[:host]}:#{DoroSettings.pcapr[:port]}/pcaps/1/about/#{dump['pcapr_id'].rstrip}"
+        pcapr_response = Net::HTTP.get_response(pcapr_query)
+        pcapname = File.basename(JSON.parse(pcapr_response.body)["filename"], ".pcap")
 
-      t ||= $1 if pcapname =~ /[0-9]*\-(.*)$/
-      raise NameError.new if  t != dump['sample'].rstrip
+        t ||= $1 if pcapname =~ /[0-9]*\-(.*)$/
+        raise NameError.new if  t != dump['sample'].rstrip
 
       rescue NameError
         LOGGER_PARSER.error "PARSER", "The pcapr filename mismatchs the one present in Dorothive!. Skipping."
@@ -111,6 +103,9 @@ module DoroParser
 
       LOGGER_PARSER.info "PARSER", "Scanning network flows and searching for unknown host IPs".yellow
 
+      @insertdb.begin_t
+
+
       xtractr.flows.each { |flow|
 
         begin
@@ -139,7 +134,7 @@ module DoroParser
 
 
             #insert Geoinfo
-            unless(localnet.include?(flow.dst.address) || multicast.include?(flow.dst.address))
+            unless localnet.include?(flow.dst.address) || multicast.include?(flow.dst.address)
 
               geo = Geoinfo.new(flow.dst.address.to_s)
               geoval = ["default", geo.coord, geo.country, geo.city, geo.updated, geo.asn]
@@ -163,9 +158,9 @@ module DoroParser
             #Insert host info
             #ip - geoinfo -  sbl - uptime - is_online - whois - zone - last-update - id - dns_name
             hostname = (dns_list[dest].nil? ? "null" : dns_list[dest])
-            hostval = [dest, geoval, "null", "null", true, "null", "null", get_time, "default", hostname]
+            hostval = [dest, geoval, "null", "null", true, "null", "null", Util.get_time, "default", hostname]
 
-            if	!@insertdb.insert("host_ips",hostval)
+            unless @insertdb.insert("host_ips",hostval)
               LOGGER_PARSER.debug "DB", " Skipping flow #{flow.id}: #{flow.src.address} > #{flow.dst.address}"  if VERBOSE
               next
             end
@@ -179,7 +174,7 @@ module DoroParser
 
           flowvals = [flow.src.address, flow.dst.address, flow.sport, flow.dport, flow.bytes, dump['sha256'], flow.packets, "default", flow.proto, flow.service.name, title, "null", flow.duration, flow.time, flow.id ]
 
-          if	!@insertdb.insert("flows",flowvals)
+          unless @insertdb.insert("flows",flowvals)
             LOGGER_PARSER.info "PARSER", "Skipping flow #{flow.id}: #{flow.src.address} > #{flow.dst.address}"
             next
           end
@@ -424,72 +419,33 @@ module DoroParser
 
     LOGGER_PARSER.info "PARSER", "Started, looking for network dumps into Dorothive.."
 
-    if daemon
-      check_pid_file DoroSettings.env[:pidfile_parser]
-      puts "[PARSER]".yellow + " Going in backround with pid #{Process.pid}"
-      puts "[PARSER]".yellow + " Logging on #{DoroSettings.env[:logfile_parser]}"
-      Process.daemon
-      create_pid_file DoroSettings.env[:pidfile_parser]
-      puts "[PARSER]".yellow + " Going in backround with pid #{Process.pid}"
-    end
 
-    @insertdb = Insertdb.new
     infinite = true
 
-    while infinite
-      pcaps = @insertdb.find_pcap
-      analyze_bintraffic(pcaps)
-      infinite = daemon
-      LOGGER.info "PARSER", "SLEEPING" if daemon
-      sleep DoroSettings.env[:dtimeout].to_i if daemon # Sleeping a while if -d wasn't set, then quit.
-    end
-    LOGGER_PARSER.info "PARSER" , "There are no more pcaps to analyze.".yellow
-    @insertdb.close
-    exit(0)
-  end
+    @insertdb = Insertdb.new
 
-  def check_pid_file file
-    if File.exist? file
-      # If we get Errno::ESRCH then process does not exist and
-      # we can safely cleanup the pid file.
-      pid = File.read(file).to_i
-      begin
-        Process.kill(0, pid)
-      rescue Errno::ESRCH
-        stale_pid = true
-      end
 
-      unless stale_pid
-        puts "[PARSER]".yellow + " Dorothy is already running (pid=#{pid})"
-        exit(1)
+    begin
+      while infinite
+        begin
+          pcaps = @insertdb.find_pcap
+          analyze_bintraffic(pcaps)
+        rescue SignalException #, RuntimeError
+          LOGGER.warn "PARSER", "SIGINT".red + " Catched [1], exiting gracefully."
+        end
+        infinite = daemon
+        LOGGER.debug "PARSER", "SLEEPING" if daemon && DEBUG
+        sleep DoroSettings.env[:sleeptime].to_i if daemon # Sleeping a while if -d wasn't set, then quit.
       end
-    end
-  end
+      LOGGER_PARSER.info "PARSER" , "There are no more pcaps to analyze.".yellow if DEBUG
+      @insertdb.close
+      exit(0)
 
-  def create_pid_file file
-    File.open(file, "w") { |f| f.puts Process.pid }
-
-    # Remove pid file during shutdown
-    at_exit do
-      LOGGER_PARSER.info "PARSER", "Shutting down." rescue nil
-      if File.exist? file
-        File.unlink file
-      end
+    rescue SignalException #, RuntimeError
+      LOGGER.warn "PARSER", "SIGINT".red + " Catched [2], exiting gracefully."
     end
-  end
 
-  # Sends SIGTERM to process in pidfile. Server should trap this
-  # and shutdown cleanly.
-  def self.stop
-    LOGGER_PARSER.info "PARSER", "Shutting down.."
-    pid_file = DoroSettings.env[:pidfile_parser]
-    if pid_file and File.exist? pid_file
-      pid = Integer(File.read(pid_file))
-      Process.kill -15, -pid
-      LOGGER_PARSER.info "PARSER", "Process #{DoroSettings.env[:pidfile_parser]} terminated"
-    else
-      LOGGER_PARSER.info "PARSER", "Can't find PID file, is DoroParser really running?"
-    end
   end
 
+
 end