thm 0.4.5 → 0.5.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 40bbc67ea101f1fc34f2f85fdb06c13677451fd0
-  data.tar.gz: 78a08e73c11da31adf4834dbd5913fbfbb93eb35
+  metadata.gz: 8b169ed01c8ea3600372b26c6586e0a972f4c3e8
+  data.tar.gz: 20b7c47617ee242e15560f44b245dcfde937b3c7
 SHA512:
-  metadata.gz: fcd88da68f3f44c7072378134bd3c2ab644b1db8f07db52d43a1644009d76ba92d8787b4b4c625b47b5199a922b7e80e72e69624766e496abcedae914910c3c4
-  data.tar.gz: 8ce4134539e86fae9dac11d7e9cfedde5310c80a9088eafb7a94f02d575b3ee5e6c4bb4e183b8409569d329e7aa8b048c21edd1ff2e4938be87381e10777cf0b
+  metadata.gz: ddedda3ae04f8f916f557e5e1080cdf2173a1aff66756f1b3d467fa495ee8f1fcf743fdffdee8833014f89466798028831e96eb326e0133705fb53544d1d21be
+  data.tar.gz: 3ce9517ec13223cb8573d5ef9026185b37535c63dd56ed86d3e65f013605f3571e48a76c007e272670153dfbcebb5b571c9877d0982c398056bf8d1d6c754360

data/README.md

@@ -1,6 +1,6 @@
 
 Threatmonitor - Packet Analysis suite with MonetDB / MySQL - RabbitMQ & PCap integration
-
+========================================================================================
 
  `,`,`,`
  ------

data/Rakefile

@@ -44,6 +44,8 @@ Gem::Specification.new do |spec|
     "lib/thm/version.rb",
     "lib/thm/dataservices/geolocation/geolocation.rb",
     "lib/thm/dataservices/trafviz/trafviz.rb",
+    "lib/thm/dataservices/external.rb",
+    "lib/thm/dataservices/safebrowsing_api.rb",
     "js/jquery.min.js",
     "js/chartkick.js",
     "js/JSXTransformer.js",

data/bin/thm-session

@@ -16,6 +16,7 @@ require 'chartkick'
 require 'sinatra/base'
 require 'slim'
 require 'colorize'
+require 'keycounter'
 
 require File.expand_path(File.join(
         File.dirname(__FILE__),
@@ -39,42 +40,6 @@ class Sinatra::Base
   
 end
 
-
-class Geocounter
-
-  # Create / Add to instance variable
-  def geocount(country)
-    country.gsub!(" ", "_") # no spaces or case
-    if !instance_variable_get("@#{country}")
-      instance_variable_set("@#{country}", 1)
-    else
-      instance_variable_set("@#{country}", instance_variable_get("@#{country}") + 1)
-    end
-    puts "Country: #{country} Value: "+instance_variable_get("@#{country}").to_s
-  end
-
-  # Read a single country
-  def geocount_reader(country)
-    country.gsub!(" ", "_") # no spaces or case
-    valnum = instance_variable_get("@#{country}")
-    return valnum
-  end
-
-  # Compile in array with the totals of all instance variables
-  def geocount_compile
-    countrycounts = Array.new
-    # You can't really inherit this class as the other class may also contain instance variables
-    # its not really an exact logic this class only works alone.
-    instance_variables.each {|n|
-      t = n.to_s.gsub("@", "")
-      countrycounts << ["#{t}", instance_variable_get("#{n}")]
-    }
-    return countrycounts
-    countrycounts
-  end
-
-end
-
 module ThmUI extend self
 
   class Deedrah < Sinatra::Base
@@ -204,14 +169,14 @@ module ThmUI extend self
       query = "select count(*) as num2,  ip_dst from wifi_ippacket a JOIN wifi_#{proto}packet b on (a.guid = b.guid) JOIN service_definitions s on (s.num = b.#{proto}_dport) where #{proto}_dport > 0 and #{proto}_dport < 10000 and s.protocol = '#{proto.upcase}' and ip_dst not in ('255.255.255.255') group by b.#{proto}_dport, a.ip_dst;"
       resusrcnt = @sessobj.query("#{query}")
       rowusrcnt = Array.new
-      @gcnt = Geocounter.new
+      @gcnt = Keycounter.new
       while row = resusrcnt.fetch_hash do
         num2 = row["num2"].to_s
         ip_dst = row["ip_dst"].to_s
         location = geoiplookup(ip_dst)
         locfix = location.to_s.gsub("(", "").gsub(")", "") # Yawn
         if location != nil or location == "" # You can't have a blank or nil instance_variable
-          @gcnt.geocount("#{locfix}")
+          @gcnt.keycount("#{locfix}")
         end
         rowusrcnt << ["#{ip_dst} #{location}", "#{num2}"]
       end    
@@ -259,7 +224,7 @@ module ThmUI extend self
       @rowusrcnt5 = toptalkers("udp")
       # Top UDP/IP Talkers
       @rowusrcnt6 = toptalkers("tcp")
-      @rowgeocount = @gcnt.geocount_compile
+      @rowgeocount = @gcnt.keycount_compile
       puts "Geo Data:"
       @rowgeocount.each {|n, x|
         puts "#{n} #{x}"

data/bin/thm-trafviz

@@ -15,6 +15,7 @@ require 'walltime'
 require 'readline'
 require 'mymenu'
 require 'pp'
+require 'cgi'
 
 require File.expand_path(File.join(
           File.dirname(__FILE__),
@@ -80,6 +81,7 @@ opts = GetoptLong.new(
   [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
   [ '--interface', '-i', GetoptLong::OPTIONAL_ARGUMENT],
   [ '--snaplength', '-s', GetoptLong::OPTIONAL_ARGUMENT],
+  [ '--safebrowsing', '-b', GetoptLong::OPTIONAL_ARGUMENT],
   [ '--debug', '-d', GetoptLong::NO_ARGUMENT ],
 )
 
@@ -94,6 +96,8 @@ opts.each do |opt, arg|
 
 -s, --snaplength - Snaplength [ OPTIONAL_ARGUMENT ]
 
+-b, --safebrowsing - Enable Google Safebrowsing API for traffic categorization
+
 -d --debug
 
 ]
@@ -106,6 +110,8 @@ opts.each do |opt, arg|
       @interface = nil || arg
     when '--snaplength'
       @snaplength = nil || arg
+    when '--safebrowsing'
+      @safebrowsing_enabled = nil || arg
   end
 end
 
@@ -113,13 +119,14 @@ puts banner
 
 # Trafviz DataServices
 tv = Thm::DataServices::Trafviz.new
+tv.debug = @debug
 tv.reqtable = HTTP_REQUEST_TABLE
 tv.reqtableua = HTTP_REQUEST_TABLE_UA
 # Connect to Datastore
 gloc = Thm::DataServices::Geolocation.new
 gloc.datastore = DATASTORE
-gloc.debug = false
-gloc.autocommit = true
+gloc.debug = 0
+gloc.autocommit = false
 gloc.dbhost = DBHOST
 gloc.dbuser = DBUSER
 gloc.dbpass = DBPASS
@@ -128,152 +135,12 @@ gloc.dbconnect
 
 use_const_defined_unless?("INTERFACE")
 use_const_defined_unless?("SNAPLENGTH")
+use_const_defined_unless?("SAFEBROWSING_ENABLED")
 
 startup = "-s #{@snaplength} -n -i #{@interface}"
 puts "Trafviz - Startup Parameters: #{startup}"
+puts "Safebrowsing URL: #{SAFEBROWSING_URL}" unless @safebrowsing_enabled == "false"
 
-module Thm
-
-  class DataServices::Trafviz::FilterManager
-
-    attr_reader :bookmarks, :pcapsetfilter
-    
-    def initialize
-      @bookmarks = Array.new
-      @bkm = MyMenu.new
-      @bkm.settitle("Welcome to Trafviz")
-      @bkm.mymenuname = "Trafviz"
-      @bkm.prompt = "Trafviz"
-      @pcapsetfilter = String.new
-    end
-
-    def read(file)
-      b = 0
-      File.open("#{Dir.home}/.thm/#{file}", 'r') {|n|
-        n.each_line {|l|
-          puts "\e[1;36m#{b})\e[0m\ #{l}"
-          @bookmarks[b] = l
-          b += 1
-        }
-      }
-    end
-
-    def write(file)
-      @bkm.mymenuname = "Filters"
-      @bkm.prompt = "\e[1;33m\Set filter>\e[0m\ "
-      pcapfilter = @bkm.definemenuitem("selectfilter", true) do
-        # Just needs value returned via readline block into addfilter
-      end
-      fltvalid = validate_filter?("#{pcapfilter}")
-      if fltvalid == true
-        File.open("#{Dir.home}/.thm/#{file}", 'a') {|n| # Append to filter file
-          n.puts("#{addfilter}")
-        }
-      end
-    end
-    
-    def set_defaults(file)
-      # Add default example filters
-      File.open("#{Dir.home}/.thm/#{file}", 'w') {|n|
-        n.puts("webtraffic: tcp dst port 80")
-        n.puts("sourceportrange: tcp src portrange 1024-65535")
-      }
-    end
-
-    def validate_filter?(filter)
-      begin
-        Pcap::Filter.compile("#{filter}")
-        puts "Filter Compile #{filter}"
-        return true
-      rescue Pcap::PcapError => e
-        pp e
-        return false
-      end
-    end
-    
-    def build_filter_menu
-      @bkm.settitle("Welcome to Trafviz")
-      @bkm.mymenuname = "Trafviz"
-      @bkm.prompt = "Trafviz"
-      @bkm.debug = 3
-      pp @bookmarks
-      @bookmarks.each {|n|
-        func_name = n.split(":")[0]
-        pcap_filter = n.split(":")[1].lstrip
-        puts "#{pcap_filter}"
-        # Instance Eval probably nicer
-        fltvalid = validate_filter?("#{pcap_filter}") # Because validate_filter? won't exist inside instance_eval
-        @bkm.instance_eval do
-          pp fltvalid
-          if fltvalid == true
-            definemenuitem("#{func_name}") do
-              @pcapsetfilter = "#{pcap_filter}"
-              #thm = DataServices::Trafviz::Main.new
-            end
-            additemtolist("#{func_name}: #{pcap_filter}", "#{func_name};")
-          end
-        end
-      }
-      @bkm.instance_eval do
-        definemenuitem("showfilter") do
-          puts "Filter: #{@pcapsetfilter}"
-        end
-        additemtolist("Show Current Filter", "showfilter;")
-      end
-      @bkm.additemtolist("Display Menu", "showmenu;")
-      @bkm.additemtolist("Toggle Menu", "togglemenu;")
-      @bkm.additemtolist("Exit Trafviz", "exit;")
-      @bkm.menu!
-    end
-        
-    def load_filters(file)
-      if File.exists?("#{Dir.home}/.thm/#{file}")
-        read(file)
-      else
-        set_defaults(file)
-        read(file)
-      end
-      build_filter_menu
-    end
-    
-  end
-  
-end
-
-# Main class / Startup
-
-module Thm
-
-  class DataServices::Trafviz::Main
-
-    attr_accessor :startup
-    
-    def initialize
-      @filter_const = Array.new
-      @startup = String.new
-      @thm = Thm::DataServices::Trafviz::FilterManager.new
-    end
-
-    def addfilter(const, filter)
-      if @thm.validate_filter?(filter) == true
-        filtercode = %Q{#{const} = Pcap::Filter.new('#{filter}', @trafviz.capture)}
-        @filter_const << "#{const})"
-        eval(filtercode)
-      end
-    end
-    
-    def commitfilters
-      flts = @filter_const.join(" | ") # Build string of CONST names
-      commitcode = %Q{@trafviz.add_filter(#{flts})}
-      eval(flts)
-    end
-    
-    def run!
-      @trafviz = Pcaplet.new(@startup)
-    end
-  end
-
-end
 =begin
 FILTERLIST = 'filters.lst'
 a = Thm::DataServices::Trafviz::FilterManager.new
@@ -291,6 +158,8 @@ a.menu!
 HTTP_REQUEST = Pcap::Filter.new('tcp dst port 80', @trafviz.capture)
 HTTP_RESPONSE = Pcap::Filter.new('tcp src portrange 1024-65535', @trafviz.capture)
 
+@sb = Thm::DataServices::Safebrowsing.new unless @safebrowsing_enabled == "false"
+
 @trafviz.add_filter(HTTP_REQUEST | HTTP_RESPONSE)
 @trafviz.each_packet {|pkt|
     data = pkt.tcp_data.to_s
@@ -308,6 +177,11 @@ HTTP_RESPONSE = Pcap::Filter.new('tcp src portrange 1024-65535', @trafviz.captur
         puts "\e[4;36mGeo Location:\e[0m\ \n\e[0;35m#{geo} \e[0m\ "
         puts "\e[4;36mRequest Data:\e[0m\ \n\e[0;32m#{data_highlight} \e[0m\ "
         tv.makeurl(data_orig)
+        makeurl_last = CGI.escape(tv.makeurl_last)
+        if instance_variable_defined?("@sb")
+          @sb.debug = @debug        
+          @sb.lookup("#{SAFEBROWSING_URL}#{makeurl_last}")
+        end
         # Process data and prepare then send elsewhere
         query_return_sql = tv.request_filter(data)
         # Store data into Datastore
@@ -319,7 +193,7 @@ HTTP_RESPONSE = Pcap::Filter.new('tcp src portrange 1024-65535', @trafviz.captur
             end
           }
         rescue
-          Tools::log_errors("/tmp/thm-sql-errors.log", "SQL Error - #{Time.now} - #{query_return_sql}") # Catch them all
+          Tools::log_errors("/tmp/thm-sql-errors.log", "SQL Error - #{Time.now} - #{query_return_sql}") unless query_return_sql == "SELECT 1;"
         end
         stwt.watch('stop')
         stwt.print_stats
@@ -338,4 +212,11 @@ HTTP_RESPONSE = Pcap::Filter.new('tcp src portrange 1024-65535', @trafviz.captur
       end
     end
   puts s.gsub("GET", "\e[1;36mGET\e[0m").gsub("POST", "\e[1;36mPOST\e[0m") if s
+  
+  # Just so we don't loose any data between commits on exiting...
+  trap("INT") {
+    gloc.commit
+    exit
+  }
+
 }

data/bin/thm-useradmin

@@ -78,9 +78,8 @@ if obj.user_exists?("admin") == true
   end  
 else
   # Create admin user if none exists
-  #obj.add_user
-  #obj.delete_user
-  #obj.set_password
+  obj.add_user
+  obj.set_password
 end
 
 

data/config.rb

@@ -31,11 +31,24 @@ module Thm
     HTTP_REQUEST_TABLE = "http_traffic_json"
     HTTP_RESPONSE_TABLE = "http_traffic_json"
     HTTP_REQUEST_TABLE_UA = "http_traffic_ua"
-    
+    # Common HTTP / HTTPS data ports for Trafviz    
+    HTTP_PORTS = [80, 3128, 8000, 8080, 8088, 8888]
+    HTTPS_PORTS = [443, 444, 3129, 8443]
+
     # Misc
     SNAPLENGTH = 65536
     INTERFACE = "eth0"
+    
+    # Google Safe Browsing API
+    SAFEBROWSING_ENABLED = "false"
+    SAFEBROWSING_API_KEY = "12345"
+    GOOGLE_API_PROJECTNAME = "myproject"
+    SAFEBROWSING_URL = "https://sb-ssl.google.com/safebrowsing/api/lookup?client=#{GOOGLE_API_PROJECTNAME}&key=#{SAFEBROWSING_API_KEY}&appver=1.5.2&pver=3.1&url="
 
+    GEOCODING_ENABLED = "false"
+    GEOCODING_API_KEY = "12345"
+    GEOCODING_URL = "https://maps.googleapis.com/maps/api/geocode/json?key=#{GEOCODING_API_KEY}&" # Format: "latlng=40.714224,-73.961452"
+    
   end
 
 end

data/lib/thm.rb

@@ -27,6 +27,126 @@ class String
 
 end
 
+
+# Slight patch here
+# Needs to moving to monkeypatches.rb
+
+class MonetDBConnection
+
+  # perform a real connection; retrieve challenge, proxy through merovinginan, build challenge and set the timezone
+  def real_connect
+    
+    server_challenge = retrieve_server_challenge()
+    if server_challenge != nil
+      salt = server_challenge.split(':')[0]
+      @server_name = server_challenge.split(':')[1]
+      @protocol = server_challenge.split(':')[2].to_i
+      @supported_auth_types = server_challenge.split(':')[3].split(',')
+      @server_endianness = server_challenge.split(':')[4]
+=begin 
+Causes issues with Threatmonitor
+      #if @@SUPPORTED_PROTOCOLS.include?(@protocol) == False
+      #  raise MonetDBProtocolError, "Protocol not supported. The current implementation of ruby-monetdb works with MAPI protocols #{@@SUPPORTED_PROTOCOLS} only."
+      #end
+=end
+      @pwhash = server_challenge.split(':')[5]
+    else
+      raise MonetDBConnectionError, "Error: server returned an empty challenge string."
+    end
+    
+    # The server supports only RIPMED160 or crypt as an authentication hash function, but the driver does not.
+    if @supported_auth_types.length == 1
+      auth = @supported_auth_types[0]
+      if auth.upcase == "RIPEMD160"
+        raise MonetDBConnectionError, auth.upcase + " " + ": algorithm not supported by ruby-monetdb."
+      end
+    end
+
+    reply = build_auth_string_v9(@auth_type, salt, @database)
+
+    if @socket != nil
+      @connection_established = true
+
+      send(reply)
+      monetdb_auth = receive
+      
+      if monetdb_auth.length == 0
+        # auth succedeed
+        true
+      else
+        if monetdb_auth[0].chr == MSG_REDIRECT
+        #redirection
+          
+          redirects = [] # store a list of possible redirects
+          
+          monetdb_auth.split('\n').each do |m|
+            # strip the trailing ^mapi:
+            # if the redirect string start with something != "^mapi:" or is empty, the redirect is invalid and shall not be included.
+            if m[0..5] == "^mapi:"
+              redir = m[6..m.length]
+              # url parse redir
+              redirects.push(redir)  
+            else
+              $stderr.print "Warning: Invalid Redirect #{m}"
+            end          
+          end
+          
+          if redirects.size == 0  
+            raise MonetDBConnectionError, "No valid redirect received"
+          else
+            begin 
+              uri = URI.split(redirects[0])
+              # Splits the string on following parts and returns array with result:
+              #
+              #  * Scheme
+              #  * Userinfo
+              #  * Host
+              #  * Port
+              #  * Registry
+              #  * Path
+              #  * Opaque
+              #  * Query
+              #  * Fragment
+              server_name = uri[0]
+              host   = uri[2]
+              port   = uri[3]
+              database   = uri[5].gsub(/^\//, '') if uri[5] != nil
+            rescue URI::InvalidURIError
+              raise MonetDBConnectionError, "Invalid redirect: #{redirects[0]}"
+            end
+          end
+          
+          if server_name == MONETDB_MEROVINGIAN
+            if @auth_iteration <= MEROVINGIAN_MAX_ITERATIONS
+              @auth_iteration += 1
+              real_connect
+            else
+              raise MonetDBConnectionError, "Merovingian: too many iterations while proxying."
+            end
+          elsif server_name == MONETDB_MSERVER
+            begin
+              @socket.close
+            rescue
+              raise MonetDBConnectionError, "I/O error while closing connection to #{@socket}"
+            end
+            # reinitialize a connection
+            @host = host
+	          @port = port
+            
+            connect(database, @auth_type)
+          else
+            @connection_established = false
+            raise MonetDBConnectionError, monetdb_auth
+          end
+        elsif monetdb_auth[0].chr == MSG_INFO
+          raise MonetDBConnectionError, monetdb_auth
+        end
+      end
+    end
+  end
+
+end
+
 module Tools
 
   class << self
@@ -56,12 +176,12 @@ module Tools
   def use_const_defined_unless?(const)
     const_down = const.downcase
     if Kernel.const_defined?("#{const}")
-      if instance_variable_get("@#{const_down}") == nil
+      unless instance_variable_defined?("@#{const_down}")
         instance_variable_set("@#{const_down}", Kernel.const_get("#{const}"))
         puts "Config Constant #{const}: #{Kernel.const_get("#{const}")}"
         puts "Instance Variable @#{const_down}: #{instance_variable_get("@#{const_down}")}"
       else
-        puts "Param via Getoptlong: Instance Variable #{@const_down}: #{instance_variable_get("@#{const_down}")}"
+        puts "Param via Getoptlong: Instance Variable @#{const_down}: #{instance_variable_get("@#{const_down}")}"
       end
     else
       raise "No Config option set add #{const} to your config.rb"
@@ -70,6 +190,24 @@ module Tools
   
 end
 
+module TextProcessing
+
+
+    def text_highlighter(text)
+      keys = ["Linux", "Java", "Android", "iPhone", "Mobile", "Chrome", 
+               "Safari", "Mozilla", "Gecko", "AppleWebKit", "Windows", 
+               "MSIE", "Win64", "Trident", "wispr", "PHPSESSID", "JSESSIONID",
+               "AMD64", "Darwin", "Macintosh", "Mac OS X", "Dalvik", "text/html", "xml"]
+      cpicker = [2,3,4,1,7,5,6] # Just a selection of colours
+      keys.each {|n|
+        text.gsub!("#{n}", "\e[4;3#{cpicker[rand(cpicker.size)]}m#{n}\e[0m\ \e[0;32m".strip)
+      }
+      return text
+    end
+    
+
+end
+
 # Load Database drivers
 require File.expand_path(File.join(
         File.dirname(__FILE__),