thm 0.1.8

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 91428c60a9b74ea883550ea758c66868d9dff5df
+  data.tar.gz: 78114801f27854edd1133d5b9d1852594040fe01
+SHA512:
+  metadata.gz: 4669fd20a613c0ebd2434671d1abc5f1eaa8f8177edc0bfd17f572fd014feba1d0d2d0714589117a1054fb9154152b41cdbe11ce02a2503cdd6776eb4b7c4660
+  data.tar.gz: c976aa7fa719c269906e22fdbb2008c4f634e379e13549792c08190592eaa6573e4cd53219b42eddd9294665b1b4b949e1c2fdd17cc4d0155dd719e09f74a769

data/README.1ST

@@ -0,0 +1,38 @@
+Threatmonitor Overview
+======================
+
+
+Sending data to a RabbitMQ queue so you can grab the data later.
+
+Please note that the queue prefix -q the queue will end up named wifi_iptables etc ...
+
+ruby thm-producer.rb -m capture -q wifi
+
+Loading your data into a database.
+
+You can change the queue it reads from / databases tables it loads into in the code just make sure you've created the database tables.
+
+obj = Thm::Consumer.new
+obj.queueprefix = "wifi"
+obj.tblname_ippacket = "wifi_ippacket"
+obj.tblname_tcppacket = "wifi_tcppacket"
+obj.tblname_udppacket = "wifi_udppacket"
+
+ruby thm-consumer.rb
+
+How to load a pcap file.
+
+obj = Thm::Localmachine.new
+obj.tblname_ippacket = "wifi_ippacket"
+obj.tblname_tcppacket = "wifi_tcppacket"
+obj.tblname_udppacket = "wifi_udppacket"
+obj.dbconnect
+
+ruby thm-pcap.rb -f Example2.pcap
+
+Thats really all the is to it.
+
+Enjoy!
+
+Brian Hood
+

data/README.md

@@ -0,0 +1,59 @@
+
+Threatmonitor - Packet Analysis suite  with MonetDB / MySQL - RabbitMQ & PCap integration
+
+![GeoIP](https://raw.githubusercontent.com/puppetpies/threatmonitor/master/screenshot-3-geo.jpg)
+
+Getting Started
+
+Things you need
+
+RabbitMQ
+
+MonetDB or MySQL
+
+Ruby
+
+Pcaplet - https://github.com/ahobson/ruby-pcap
+
+PCAPRUB - https://github.com/puppetpies/pcaprub - For Interface Realtime capture / dumping to disk
+
+![Dashboard](https://raw.githubusercontent.com/puppetpies/threatmonitor/master/screenshot-1.jpg)
+
+To build:
+
+You'll need PCAP / Development Header files
+
+gem build pcap.gemspec
+gem install pcap-0.7.7.gem 
+
+GEMS: AMQP, Bunny, Eventmachine, guid, MonetDB, mysql, pcaplet
+
+This is all experimental but i believe it works well so far.
+
+Features ability to push pcap data to Message Queue.
+
+Read data from disk into message queue
+
+Write data from message queue to disk.
+
+Allows you to easily move IP / TCP / UDP data with ease and analyze else where
+
+Creating your database once you've installed MonetDB
+
+monetdbd create /path/to/dbfarm
+
+monetdbd start /path/to/dbfarm
+
+monetdb create threatmonitor
+
+monetdb release threatmonitor
+
+monetdb start threatmonitor
+
+Import the schema from the SQL provided now moved to sql/
+
+I've now included MySQL Database support also however if your going to create big data sets i think i would use MonetDB
+
+![Dashboard Other](https://raw.githubusercontent.com/puppetpies/threatmonitor/master/screenshot-2.jpg)
+
+Have fun !

data/bin/thm-consumer

@@ -0,0 +1,89 @@
+#!/usr/bin/ruby
+########################################################################
+#
+# Author: Brian Hood
+#
+# Description: Threatmonitor Consumer
+#
+# Producer - Save data from queue to Database
+#
+########################################################################
+#puts Dir.getwd
+require 'getoptlong'
+require File.expand_path(File.join(
+          File.dirname(__FILE__),
+          "../lib/thm.rb", "../../config.rb"))
+
+include Thm::Defaults
+
+ARGV[0] = "--help" if ARGV[0] == nil
+
+banner = "\e[1;34mWelcome to Threatmonitor Consumer \e[0m\ \n"
+banner << "\e[1;34m=================================\e[0m\ \n"
+
+opts = GetoptLong.new(
+  [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
+  [ '--infinite', '-i', GetoptLong::NO_ARGUMENT ],
+  [ '--passes', '-p', GetoptLong::REQUIRED_ARGUMENT ],
+  [ '--runonce', '-r', GetoptLong::NO_ARGUMENT ]
+)
+
+opts.each do |opt, arg|
+  case opt
+    when '--help'
+      helper = %q[
+-h, --help:
+   show help
+   
+-i, --infinite [ For consuming live captures ]
+  
+-p, --passes number of passes  [ For medium / larger queues ]
+
+-r, --runonce [ Just as it says ]
+
+NOTES:
+
+  Press CTRL+C to exit Reactor thread at anytime to move on to next queue.
+  There are currently 3 queues ippacket, tcppacket, udppacket
+  
+    ]
+      puts banner
+      puts helper
+      exit
+    when '--infinite'
+      @infinite = arg
+    when '--passes'
+      @passes = arg
+    when '--runonce'
+      @runonce = arg
+  end
+end
+
+puts banner
+# See thmmq.rb for list for variables
+obj = Thm::Consumer.new
+obj.datastore = DATASTORE
+obj.mqhost = MQHOST
+obj.mquser = MQUSER
+obj.mqpass = MQPASS
+obj.mqvhost = MQVHOST
+obj.dbhost = DBHOST
+obj.dbuser = DBUSER
+obj.dbpass = DBPASS
+obj.dbname = DBNAME
+obj.queueprefix = QUEUEPREFIX
+obj.tblname_ippacket = TBLNAME_IPPACKET
+obj.tblname_tcppacket = TBLNAME_TCPPACKET
+obj.tblname_udppacket = TBLNAME_UDPPACKET
+obj.mqconnect
+obj.dbconnect
+# Send data to database from queue
+if @infinite.nil? == false
+  obj.infinite
+elsif @passes.nil? == false
+  obj.passes(@passes.to_i)
+elsif @runonce.nil? == false
+  obj.from_mq_to_db
+end
+obj.mqclose
+

data/bin/thm-pcap

@@ -0,0 +1,67 @@
+#!/usr/bin/ruby
+########################################################################
+#
+# Author: Brian Hood
+#
+# Description: Threatmonitor Pcap
+#
+# Producer - Save data from Pcap file(s) to Database
+#
+########################################################################
+
+require 'getoptlong'
+require File.expand_path(File.join(
+          File.dirname(__FILE__),
+          "../lib/thm.rb", "../../config.rb"))
+
+include Thm::Defaults
+
+ARGV[0] = "--help" if ARGV[0] == nil
+
+banner = "\e[1;34mWelcome to Threatmonitor PCap Loader \e[0m\ \n"
+banner << "\e[1;34m===================================\e[0m\ \n"
+
+opts = GetoptLong.new(
+  [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
+  [ '--pcapfile', '-f', GetoptLong::OPTIONAL_ARGUMENT ]
+)
+
+opts.each do |opt, arg|
+  case opt
+    when '--help'
+      helper = %q[
+-h, --help:
+   show help
+   
+-f, --pcapfile [ REQUIRED ] 
+]
+      puts banner
+      puts helper
+      exit
+    when '--pcapfile'
+      @pcapfile = arg
+  end
+end
+
+puts banner
+# See thmmq.rb for list for variables
+obj = Thm::Localmachine.new
+obj.datastore = DATASTORE
+obj.mqhost = MQHOST
+obj.mquser = MQUSER
+obj.mqpass = MQPASS
+obj.mqvhost = MQVHOST
+obj.dbhost = DBHOST
+obj.dbuser = DBUSER
+obj.dbpass = DBPASS
+obj.dbname = DBNAME
+obj.queueprefix = QUEUEPREFIX
+obj.tblname_ippacket = TBLNAME_IPPACKET
+obj.tblname_tcppacket = TBLNAME_TCPPACKET
+obj.tblname_udppacket = TBLNAME_UDPPACKET
+obj.dbconnect
+# Send data to database from queue
+if @pcapfile.nil? == false
+  obj.from_pcap_db("#{@pcapfile}")
+end
+

data/bin/thm-producer

@@ -0,0 +1,102 @@
+#!/usr/bin/ruby
+########################################################################
+#
+# Author: Brian Hood
+#
+# Description: Threatmonitor Producer
+#
+# Producer - Load data Database to RabbitMQ for distrubition or Packet 
+#            Trace live data and send to RabbitMQ
+########################################################################
+
+require 'getoptlong'
+require File.expand_path(File.join(
+          File.dirname(__FILE__),
+          "../lib/thm.rb", "../../config.rb"))
+
+include Thm::Defaults
+
+ARGV[0] = "--help" if ARGV[0] == nil
+
+opts = GetoptLong.new(
+  [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
+  [ '--mode', '-m', GetoptLong::REQUIRED_ARGUMENT ],
+  [ '--queueprefix', '-q', GetoptLong::OPTIONAL_ARGUMENT ],
+  [ '--interface', '-i', GetoptLong::REQUIRED_ARGUMENT ],
+  [ '--filter', '-f', GetoptLong::REQUIRED_ARGUMENT ]
+)
+
+banner = "\e[1;34mWelcome to Threatmonitor Producer \e[0m\ \n"
+banner << "\e[1;34m=================================\e[0m\ \n"
+
+opts.each do |opt, arg|
+  case opt
+    when '--help'
+      helper = %q[
+-h, --help:
+   show help
+   
+-m, --mode database / capture [ REQUIRED ]
+  
+-q, --queueprefix queue name [ OPTIONAL ] [ Uses config.rb ]
+
+-i --interface network interface [ REQUIRED ]
+ 
+-f --filter your pcap / tcpdump style filter [ OPTIONAL ]
+    ]
+      puts banner
+      puts helper
+      exit
+    when '--mode'
+      @modeparam = arg
+    when '--queueprefix'
+      @queueprefix = arg
+    when '--interface'
+      @interface = arg
+    when '--filter'
+      @filter = arg
+  end
+end
+
+puts banner
+# See thmmq.rb for list for variables
+obj = Thm::Producer.new
+obj.datastore = DATASTORE
+obj.mqhost = MQHOST
+obj.mquser = MQUSER
+obj.mqpass = MQPASS
+obj.mqvhost = MQVHOST
+obj.dbhost = DBHOST
+obj.dbuser = DBUSER
+obj.dbpass = DBPASS
+obj.dbname = DBNAME
+if !defined? @queueprefix
+  obj.queueprefix = QUEUEPREFIX
+else
+  obj.queueprefix = @queueprefix
+end
+obj.tblname_ippacket = TBLNAME_IPPACKET
+obj.tblname_tcppacket = TBLNAME_TCPPACKET
+obj.tblname_udppacket = TBLNAME_UDPPACKET
+obj.mqconnect
+obj.dbconnect unless @modeparam == "capture"
+mode = @modeparam
+
+if mode == "database"
+  # Send data from Database to RabbitMQ for Fanout etc
+  ippacketsql = "SELECT * FROM \"threatmonitor\".#{obj.tblname_ippacket} LIMIT 300 OFFSET 400000"
+  obj.from_db_to_mq(ippacketsql)
+
+elsif mode == "capture"
+  # From Pcap to RabbitMQ for Fanout etc.
+  # 
+  # Standard Pcap / Wireshark filters 
+  # 'tcp port not 22 and not arp and udp port not 53 and udp portrange not 67-69 and tcp port not 8443 and host not 78.47.223.130'
+  if %x{id -u}.strip != "0"
+    puts "Require superuser privileges"
+    exit
+  end
+  obj.from_interface_to_mq("#{@interface}", "#{@filter}")
+  #obj.from_pcap_to_mq("wlo1", "tcp port not 22")
+  obj.mqclose
+end

data/bin/thm-session

@@ -0,0 +1,319 @@
+#!/usr/bin/ruby
+########################################################################
+#
+# Author: Brian Hood
+#
+# Description: Threatmonitor User Administration / Dashboard
+# Codename: Deedrah
+#
+# Extends the functionality of the Thm module adding Authorization
+# Adding Session Login / Sinatra / Web Interface functionality
+#
+########################################################################
+
+require 'rubygems' if RUBY_VERSION < "1.9"
+require 'chartkick'
+require 'sinatra/base'
+require 'slim'
+require 'colorize'
+
+require File.expand_path(File.join(
+          File.dirname(__FILE__),
+          "../thm-authentication.rb"))
+
+RELEASE = "Deedrah"
+
+class Sinatra::Base
+
+  class << self
+  
+    def getpost(path, opts={}, &block)
+      get(path, opts, &block)
+      post(path, opts, &block)
+    end
+    
+  end
+  
+end
+
+
+class Geocounter
+
+  # Create / Add to instance variable
+  def geocount(country)
+    country.gsub!(" ", "_") # no spaces or case
+    if !instance_variable_get("@#{country}")
+      instance_variable_set("@#{country}", 1)
+    else
+      instance_variable_set("@#{country}", instance_variable_get("@#{country}") + 1)
+    end
+    puts "Country: #{country} Value: "+instance_variable_get("@#{country}").to_s
+  end
+
+  # Read a single country
+  def geocount_reader(country)
+    country.gsub!(" ", "_") # no spaces or case
+    valnum = instance_variable_get("@#{country}")
+    return valnum
+  end
+
+  # Compile in array with the totals of all instance variables
+  def geocount_compile
+    countrycounts = Array.new
+    instance_variables.each {|n|
+      t = n.to_s.gsub("@", "")
+      countrycounts << ["#{t}", instance_variable_get("#{n}")]
+    }
+    return countrycounts
+    countrycounts
+  end
+
+end
+
+module ThmUI extend self
+
+  class Deedrah < Sinatra::Base
+
+    attr_reader :thmsession
+    
+    include Chartkick::Helper
+    
+    set :server, :puma
+    set :logging, :true
+    set :app_file, __FILE__
+
+    configure do
+      enable :session
+    end
+    
+    before do
+      content_type :html
+    end
+	
+    set :title, "Threatmonitor - Packet Analysis Suite #{RELEASE}"
+    set :port, 4567
+    set :bind, "0.0.0.0"
+    # Global helpers
+    #helpers ApplicationHelper
+
+    # Set folders for template to
+    set :root, File.expand_path(File.join(File.dirname(__FILE__), '../'))
+    puts root.green
+    
+    templateroot = File.expand_path(File.join(File.dirname(__FILE__), '../views'))
+    set :sessions,
+        :httponly       => true,
+        :secure         => production?,
+        :expire_after   => 3600, # 1 hour
+	#:views          => File.expand_path(File.expand_path('../../views/', __FILE__)),
+	:views		=> Proc.new { File.join(root, "views") },
+	#:layout_engine  => :slim,
+	:public_folder => File.dirname(__FILE__) + '/bin'
+
+    enable :method_override
+    
+    puts "TemplateRoot: #{templateroot}".to_s
+    def objstart!
+      @sessobj = Thm::Authorization::Authentication.new
+      @sessobj.datastore = "monetdb"
+      @sessobj.dbconnect
+    end
+    
+    # NOTE: Monkey patch Sinatra initialize
+    #       If you go to /dashboard without logging in @sessobj it won't have been created
+    #       So if you create it objstart! in the Sinatra's initialize your sorted.
+    #       Page requests will come in a @sessobj will always exist and be the same one no headaches 
+    def initialize(app = nil)
+      super()
+      objstart! # Little trick here patch so i can get objstart! out of the box!
+      @app = app 
+      @template_cache = Tilt::Cache.new
+      yield self if block_given?
+    end
+
+    def login_session?
+      if @sessobj.login_session? == false
+        puts "Not logged in"
+      else
+        return true
+      end
+    end
+    
+    def get_session_info?
+      puts "Sinatra: #{Sinatra::VERSION} / Thmsession: #{@sessobj.thmsession}"
+    end
+    
+    def login(username, password)
+      @sessobj.login("#{username}", "#{password}")
+      if login_session?
+        puts "\e[1;32m\Welcome to Threatmonitor \e[0m\ "
+        puts "Thm Session: #{@sessobj.thmsession}"
+        return true
+      else
+        return false
+      end
+    end
+     
+    def logout
+      @sessobj.thmsesslock = "DEADBEEF"
+    end
+    
+    def login_lock?
+      if @sessobj.thmsesslock != "OK"
+        puts "Session doesn't exist redirecting to Login..."
+        redirect '/'
+      end
+    end
+    
+    # Sinatra routings
+    
+    getpost '/' do
+      if params[:submit] != nil
+        @sessthm = login(params[:username], params[:password])
+      end
+      if @sessobj.thmsesslock == "OK"
+        puts "Session created redirecting to Dashboard ..."
+        redirect '/dashboard'
+      end
+      slim :authenticate
+    end
+    
+    def geoiplookup(ip)
+      query = "SELECT continent_name, country_name FROM geoipdata_ipv4blocks_country a JOIN geoipdata_locations_country b ON (a.geoname_id = b.geoname_id) WHERE network LIKE '#{ip.split(".")[0]}.#{ip.split(".")[1]}.%' GROUP BY b.continent_name, b.country_name LIMIT 1;"
+      resusrcnt = @sessobj.query("#{query}")
+      while row = resusrcnt.fetch_hash do
+        continent_name = row["continent_name"].to_s
+        country_name = row["country_name"].to_s
+        #if continent_name == ""
+          cname = "(#{country_name})"
+        #else
+        #  cname = "(#{continent_name})"
+        #end
+      end
+     # geoipcount(
+      return cname
+    end
+
+    def packetcounts(proto)
+      # Retrieve 
+      query = "select count(*) as num2,  ip_dst from wifi_ippacket a JOIN wifi_#{proto}packet b on (a.guid = b.guid) JOIN service_definitions s on (s.num = b.#{proto}_dport) where #{proto}_dport > 0 and #{proto}_dport < 10000 and s.protocol = '#{proto.upcase}' and ip_dst not in ('255.255.255.255') group by b.#{proto}_dport, a.ip_dst;"
+      resusrcnt = @sessobj.query("#{query}")
+      rowusrcnt = Array.new
+      @gcnt = Geocounter.new
+      while row = resusrcnt.fetch_hash do
+        num2 = row["num2"].to_s
+        ip_dst = row["ip_dst"].to_s
+        location = geoiplookup(ip_dst)
+        locfix = location.to_s.gsub("(", "").gsub(")", "") # Yawn
+        if location != nil or location == ""
+          @gcnt.geocount("#{locfix}")
+        end
+        rowusrcnt << ["#{ip_dst} #{location}", "#{num2}"]
+      end    
+      return rowusrcnt
+    end
+    
+    def servicecounts(proto)
+      query = "select num, description, count(*) as num2 from wifi_ippacket a JOIN wifi_#{proto}packet b on (a.guid = b.guid) JOIN service_definitions s on (s.num = b.#{proto}_dport) where #{proto}_dport > 0 and #{proto}_dport < 10000 and s.protocol = '#{proto.upcase}' and ip_dst not in ('255.255.255.255') group by b.#{proto}_dport, a.ip_dst, s.description, s.num;"
+      resusrcnt = @sessobj.query("#{query}")
+      rowusrcnt = Array.new
+      while row = resusrcnt.fetch_hash do
+        num = row["num"].to_s
+        desc = row["description"].to_s
+        count = row["num2"].to_s
+        rowusrcnt << ["#{desc} (#{num})", "#{count}"]
+      end
+      return rowusrcnt 
+    end
+    
+    def toptalkers(proto)
+      query = "select count(*) as num2, #{proto}_dport, ip_dst from wifi_ippacket a JOIN wifi_#{proto}packet b on (a.guid = b.guid) JOIN service_definitions s on (s.num = b.#{proto}_dport) where #{proto}_dport > 0 and #{proto}_dport < 10000 and s.protocol = '#{proto.upcase}' and ip_dst not in ('255.255.255.255') group by b.#{proto}_dport, a.ip_dst order by num2 desc;"
+      resusrcnt = @sessobj.query("#{query}")
+      rowusrcnt = Array.new
+      while row = resusrcnt.fetch_hash do
+        ip_dst = row["ip_dst"].to_s
+        num = row["#{proto}_dport"].to_s
+        count = row["num2"].to_s
+        location = geoiplookup(ip_dst)
+        rowusrcnt << ["#{ip_dst} #{location} (#{num})", "#{count}"]
+      end
+      return rowusrcnt
+    end
+    
+    getpost '/dashboard' do
+      login_lock?
+      # UDP
+      @rowusrcnt = packetcounts("udp")
+      # TCP
+      @rowusrcnt2 = packetcounts("tcp")
+      # Service chart UDP
+      @rowusrcnt3 = servicecounts("udp")
+      # Service chart TCP
+      @rowusrcnt4 = servicecounts("tcp")
+      # Top TCP/IP Talkers
+      @rowusrcnt5 = toptalkers("udp")
+      # Top UDP/IP Talkers
+      @rowusrcnt6 = toptalkers("tcp")
+      @rowgeocount = @gcnt.geocount_compile
+      puts "Geo Data:"
+      @rowgeocount.each {|n, x|
+        puts "#{n} #{x}"
+      }
+      erb :dashboard
+    end
+    
+    get '/status' do
+      # Informational / Debug
+      get_session_info?
+    end
+    
+    get '/logout' do
+      # Logout and remove thmsesslock
+      if @sessobj.thmsesslock == "OK"
+        logout
+        redirect '/'
+      end
+      slim :logout
+    end
+    
+    get '/stylesheets/screen.css' do
+      send_file 'stylesheets/screen.css', :type => :css
+    end
+    
+    get '/js/chartkick.js' do
+      send_file 'js/chartkick.js', :type => :js
+    end
+
+    get '/js/jquery.min.js' do
+      send_file 'js/jquery.min.js', :type => :js
+    end
+
+    get '/js/JSXTransformer.js' do
+      send_file 'js/JSXTransformer.js', :type => :js
+    end
+
+    get '/js/marked.min.js' do
+      send_file 'js/marked.min.js', :type => :js
+    end
+
+    get '/js/react.js' do
+      send_file 'js/react.js', :type => :js
+    end
+
+    get '/js/files/authenticate.jsx' do
+      send_file 'js/files/authenticate.jsx', :type => :js
+    end
+    
+    get '/js/jsapi.js' do
+      send_file 'js/jsapi.js', :type => :js
+    end
+    
+    run!
+    
+  end
+  
+end
+
+# Start Dashboard Release
+
+Deedrah.new