thm 0.2.2 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 14298155e86d49a7a209f664112f24a831608ab5
-  data.tar.gz: 36f3b7465a8bda21887bc76760d83e18e1f53aac
+  metadata.gz: 68cf9e24de9c37c3ddfc874510040610213e6e5e
+  data.tar.gz: da51e597440bdb58e9360da1e9e20738919b98a0
 SHA512:
-  metadata.gz: 4ea375fc0eeac8aa7c01f5849ab71b6994ef5a38f6f1756bbd7bc933a87e24d0dbc6a3330c575cb8243e4e3ac42c1a9ad1a54d541f66be0320174aa20dbfd776
-  data.tar.gz: 52f8c8f450769a074d3f4797dfe366a635ba307e50d13b76ddda15bde8ce86756bc8736fee2cdf5f08b5d710687425add72002f68207e6578bdf5e4fe078fb1f
+  metadata.gz: 009f8996404e193f786e030e08c777c9f2278c61442789a2080f331724fa1bcffaccbd96e2e089fbc53ceb5eff39314ae46760c9d022ee7bd0a168ef46dce6fd
+  data.tar.gz: 020a8de2955f539ef44170037594de9232b8df897caeec8b2fa0f7203584bea5eac680d6f220d69287576c96a7ae2a7c20d6efc3f49e1529458958de7ef4d717

data/README.1ST

@@ -1,5 +1,5 @@
-Threatmonitor Packet Analysis Suite HOWTO
-=========================================
+Threatmonitor Packet Analysis Suite - Docker Image HOWTO
+========================================================
 
 To import example data.
 
@@ -11,7 +11,7 @@ Will also need to import the GeoIP Data.
 
 The web interface admin login is admin - default password goblin
 
-You will also need the GeoIP Data importing as well for the Web interface Dashboard the operate
+You will also need the GeoIP Data importing as well for the Web interface Dashboard to operate
 
 mclient -u monetdb -d threatmonitor < geoip-import.sql
 
@@ -21,7 +21,7 @@ http://172.17.0.1:4567/dashboard
 
 Your defaults file is the config.rb in the gem directory for now.
 
-/usr/local/lib/ruby/gems/2.1.0/gems/thm-0.1.9/config.rb
+/home/<username>/config.rb
 
 To Capture data:
 

data/README.md

@@ -1,5 +1,13 @@
 
-Threatmonitor - Packet Analysis suite  with MonetDB / MySQL - RabbitMQ & PCap integration
+Threatmonitor - Packet Analysis suite with MonetDB / MySQL - RabbitMQ & PCap integration
+
+
+ `,`,`,`
+ ------
+( ͡° ͜ʖ ͡°)
+ \    /
+  \--/
+
 
 ![GeoIP](https://raw.githubusercontent.com/puppetpies/threatmonitor/master/screenshot-3-geo.jpg)
 
@@ -60,6 +68,10 @@ Import the schema from the SQL provided now moved to sql/
 
 I've now included MySQL Database support also however if your going to create big data sets i think i would use MonetDB
 
-![Dashboard Other](https://raw.githubusercontent.com/puppetpies/threatmonitor/master/screenshot-2.jpg)
+![Dashboard Other](https://raw.githubusercontent.com/puppetpies/threatmonitor/master/screenshot-4.jpg)
+
+We are also working on a Traffic Visualizer
+
+![Trafviz](https://raw.githubusercontent.com/puppetpies/threatmonitor/master/screenshot-trafviz-1.jpg)
 
 Have fun !

data/bin/thm-consumer

@@ -2,7 +2,7 @@
 ########################################################################
 #
 # Author: Brian Hood
-#
+# Email: <brianh6854@googlemail.com>
 # Description: Threatmonitor Consumer
 #
 # Producer - Save data from queue to Database

data/bin/thm-pcap

@@ -2,7 +2,7 @@
 ########################################################################
 #
 # Author: Brian Hood
-#
+# Email: <brianh6854@googlemail.com>
 # Description: Threatmonitor Pcap
 #
 # Producer - Save data from Pcap file(s) to Database

data/bin/thm-producer

@@ -2,7 +2,7 @@
 ########################################################################
 #
 # Author: Brian Hood
-#
+# Email: <brianh6854@googlemail.com>
 # Description: Threatmonitor Producer
 #
 # Producer - Load data Database to RabbitMQ for distrubition or Packet 

data/bin/thm-session

@@ -2,7 +2,7 @@
 ########################################################################
 #
 # Author: Brian Hood
-#
+# Email: <brianh6854@googlemail.com>
 # Description: Threatmonitor User Administration / Dashboard
 # Codename: Deedrah
 #
@@ -18,12 +18,12 @@ require 'slim'
 require 'colorize'
 
 require File.expand_path(File.join(
-          File.dirname(__FILE__),
-          "../thm-authentication.rb"))
+        File.dirname(__FILE__),
+        "../thm-authentication.rb"))
 
 require File.expand_path(File.join(
-          File.dirname(__FILE__),
-          "../lib/thm/version.rb"))
+        File.dirname(__FILE__),
+        "../lib/thm/version.rb"))
 
 
 class Sinatra::Base
@@ -110,10 +110,10 @@ module ThmUI extend self
         :httponly       => true,
         :secure         => production?,
         :expire_after   => 3600, # 1 hour
-	#:views          => File.expand_path(File.expand_path('../../views/', __FILE__)),
-	:views		=> Proc.new { File.join(root, "views") },
-	#:layout_engine  => :slim,
-	:public_folder => File.dirname(__FILE__) + '/bin'
+        #:views          => File.expand_path(File.expand_path('../../views/', __FILE__)),
+        :views		=> Proc.new { File.join(root, "views") },
+        #:layout_engine  => :slim,
+        :public_folder => File.dirname(__FILE__) + '/bin'
 
     enable :method_override
     
@@ -127,10 +127,10 @@ module ThmUI extend self
     # NOTE: Monkey patch Sinatra initialize
     #       If you go to /dashboard without logging in @sessobj it won't have been created
     #       So if you create it objstart! in the Sinatra's initialize your sorted.
-    #       Page requests will come in a @sessobj will always exist and be the same one no headaches 
+    #       Page requests will come in and @sessobj will always exist and be the same one no headaches 
     def initialize(app = nil)
       super()
-      objstart! # Little trick here patch so i can get objstart! out of the box!
+      objstart! # Little trick here patch so i can get @sessobj out of the box!
       @app = app 
       @template_cache = Tilt::Cache.new
       yield self if block_given?
@@ -210,7 +210,7 @@ module ThmUI extend self
         ip_dst = row["ip_dst"].to_s
         location = geoiplookup(ip_dst)
         locfix = location.to_s.gsub("(", "").gsub(")", "") # Yawn
-        if location != nil or location == ""
+        if location != nil or location == "" # You can't have a blank or nil instance_variable
           @gcnt.geocount("#{locfix}")
         end
         rowusrcnt << ["#{ip_dst} #{location}", "#{num2}"]

data/bin/thm-trafviz

@@ -0,0 +1,316 @@
+#!/usr/bin/ruby
+########################################################################
+#
+# Author: Brian Hood
+# Email: <brianh6854@googlemail.com>
+# Description: Threatmonitor HTTP Viz
+#
+# Analyze HTTP Traffic using Traffic Visualizer
+#
+########################################################################
+
+require 'pcaplet'
+require 'getoptlong'
+require 'walltime'
+require 'readline'
+require 'mymenu'
+require 'pp'
+
+require File.expand_path(File.join(
+          File.dirname(__FILE__),
+          "../lib/thm.rb"))
+
+conf = Thm::FileServices.new
+conf.thmhome?
+
+include Thm::Defaults
+include Tools
+
+class NilClass
+
+  def strip
+    # Slightly naughty as makeurl hits a bug.
+=begin
+exception when looping over each packet loop: #<NoMethodError: undefined method `strip' for nil:NilClass>
+./thm-httpseries:120:in `block in makeurl': undefined method `strip' for nil:NilClass (NoMethodError)
+	from ./thm-httpseries:118:in `each_line'
+	from ./thm-httpseries:118:in `makeurl'
+	from ./thm-httpseries:189:in `block in <main>'
+	from /usr/lib/ruby/gems/2.1.0/gems/pcap-0.7.7/lib/pcaplet.rb:94:in `loop'
+	from /usr/lib/ruby/gems/2.1.0/gems/pcap-0.7.7/lib/pcaplet.rb:94:in `each_packet'
+	from ./thm-httpseries:178:in `<main>'
+=end
+  end
+  
+end
+
+ARGV[0] = "--help" if ARGV[0] == nil
+@debug = false
+banner = "\e[1;34mWelcome to Threatmonitor HTTP Traffic Visualizer \e[0m\ \n"
+banner << "\e[1;34m=================================================\e[0m\ \n\n"
+m = %Q{  
+ /\
+/\/\
+\/\/
+ \/}
+
+opts = GetoptLong.new(
+  [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
+  [ '--interface', '-i', GetoptLong::OPTIONAL_ARGUMENT],
+  [ '--snaplength', '-s', GetoptLong::OPTIONAL_ARGUMENT],
+  [ '--debug', '-d', GetoptLong::NO_ARGUMENT ],
+)
+
+opts.each do |opt, arg|
+  case opt
+    when '--help'
+      helper = %q[
+-h, --help:
+   show help
+   
+-i, --interface - Network Interface to collect measurements from [ OPTIONAL_ARGUMENT ]
+
+-s, --snaplength - Snaplength [ OPTIONAL_ARGUMENT ]
+
+-d --debug
+
+]
+      puts banner
+      puts helper
+      exit
+    when '--debug'
+      @debug = true
+    when '--interface'
+      @interface = nil || arg
+    when '--snaplength'
+      @snaplength = nil || arg
+  end
+end
+
+puts banner
+
+# Trafviz DataServices
+tv = Thm::DataServices::Trafviz.new
+# Connect to Datastore
+gloc = Thm::DataServices::Geolocation.new
+gloc.datastore = DATASTORE
+gloc.debug = false
+gloc.autocommit = true
+gloc.dbhost = DBHOST
+gloc.dbuser = DBUSER
+gloc.dbpass = DBPASS
+gloc.dbname = DBNAME
+gloc.dbconnect
+
+use_const_defined_unless?("INTERFACE")
+use_const_defined_unless?("SNAPLENGTH")
+
+startup = "-s #{@snaplength} -n -i #{@interface}"
+puts "Trafviz - Startup Parameters: #{startup}"
+
+module Thm
+
+  class DataServices::Trafviz::FilterManager
+
+    attr_reader :bookmarks, :pcapsetfilter
+    
+    def initialize
+      @bookmarks = Array.new
+      @bkm = MyMenu.new
+      @bkm.settitle("Welcome to Trafviz")
+      @bkm.mymenuname = "Trafviz"
+      @bkm.prompt = "Trafviz"
+      @pcapsetfilter = String.new
+    end
+
+    def read(file)
+      b = 0
+      File.open("#{Dir.home}/.thm/#{file}", 'r') {|n|
+        n.each_line {|l|
+          puts "\e[1;36m#{b})\e[0m\ #{l}"
+          @bookmarks[b] = l
+          b += 1
+        }
+      }
+    end
+
+    def write(file)
+      @bkm.mymenuname = "Filters"
+      @bkm.prompt = "\e[1;33m\Set filter>\e[0m\ "
+      pcapfilter = @bkm.definemenuitem("selectfilter", true) do
+        # Just needs value returned via readline block into addfilter
+      end
+      fltvalid = validate_filter?("#{pcapfilter}")
+      if fltvalid == true
+        File.open("#{Dir.home}/.thm/#{file}", 'a') {|n| # Append to filter file
+          n.puts("#{addfilter}")
+        }
+      end
+    end
+    
+    def set_defaults(file)
+      # Add default example filters
+      File.open("#{Dir.home}/.thm/#{file}", 'w') {|n|
+        n.puts("webtraffic: tcp dst port 80")
+        n.puts("sourceportrange: tcp src portrange 1024-65535")
+      }
+    end
+
+    def validate_filter?(filter)
+      begin
+        Pcap::Filter.compile("#{filter}")
+        puts "Filter Compile #{filter}"
+        return true
+      rescue Pcap::PcapError => e
+        pp e
+        return false
+      end
+    end
+    
+    def build_filter_menu
+      @bkm.settitle("Welcome to Trafviz")
+      @bkm.mymenuname = "Trafviz"
+      @bkm.prompt = "Trafviz"
+      @bkm.debug = 3
+      pp @bookmarks
+      @bookmarks.each {|n|
+        func_name = n.split(":")[0]
+        pcap_filter = n.split(":")[1].lstrip
+        puts "#{pcap_filter}"
+        # Instance Eval probably nicer
+        fltvalid = validate_filter?("#{pcap_filter}") # Because validate_filter? won't exist inside instance_eval
+        @bkm.instance_eval do
+          pp fltvalid
+          if fltvalid == true
+            definemenuitem("#{func_name}") do
+              @pcapsetfilter = "#{pcap_filter}"
+              #thm = DataServices::Trafviz::Main.new
+            end
+            additemtolist("#{func_name}: #{pcap_filter}", "#{func_name};")
+          end
+        end
+      }
+      @bkm.instance_eval do
+        definemenuitem("showfilter") do
+          puts "Filter: #{@pcapsetfilter}"
+        end
+        additemtolist("Show Current Filter", "showfilter;")
+      end
+      @bkm.additemtolist("Display Menu", "showmenu;")
+      @bkm.additemtolist("Toggle Menu", "togglemenu;")
+      @bkm.additemtolist("Exit Trafviz", "exit;")
+      @bkm.menu!
+    end
+        
+    def load_filters(file)
+      if File.exists?("#{Dir.home}/.thm/#{file}")
+        read(file)
+      else
+        set_defaults(file)
+        read(file)
+      end
+      build_filter_menu
+    end
+    
+  end
+  
+end
+
+# Main class / Startup
+
+module Thm
+
+  class DataServices::Trafviz::Main
+
+    attr_accessor :startup
+    
+    def initialize
+      @filter_const = Array.new
+      @startup = String.new
+      @thm = Thm::DataServices::Trafviz::FilterManager.new
+    end
+
+    def addfilter(const, filter)
+      if @thm.validate_filter?(filter) == true
+        filtercode = %Q{#{const} = Pcap::Filter.new('#{filter}', @trafviz.capture)}
+        @filter_const << "#{const})"
+        eval(filtercode)
+      end
+    end
+    
+    def commitfilters
+      flts = @filter_const.join(" | ") # Build string of CONST names
+      commitcode = %Q{@trafviz.add_filter(#{flts})}
+      eval(flts)
+    end
+    
+    def run!
+      @trafviz = Pcaplet.new(@startup)
+    end
+  end
+
+end
+=begin
+FILTERLIST = 'filters.lst'
+a = Thm::DataServices::Trafviz::FilterManager.new
+a.load_filters("#{FILTERLIST}")
+a.instance_eval do
+  definemenuitem("addfilter") do
+    puts "Filter: #{@pcapsetfilter}"
+  end
+  additemtolist("Add New Filter", "write('#{FILTERLIST}');")
+end
+a.menu!
+=end
+
+@trafviz = Pcaplet.new(startup)
+HTTP_REQUEST  = Pcap::Filter.new('tcp dst port 80', @trafviz.capture)
+HTTP_RESPONSE = Pcap::Filter.new('tcp src portrange 1024-65535', @trafviz.capture)
+
+@trafviz.add_filter(HTTP_REQUEST | HTTP_RESPONSE)
+@trafviz.each_packet {|pkt|
+    data = pkt.tcp_data.to_s
+    data_orig = data.clone
+    data_highlight = tv.text_highlighter(data_orig)
+    case pkt
+    when HTTP_REQUEST
+      if data =~ HTTP_METHODS_REGEXP
+        stwt = Stopwatch.new
+        stwt.watch('start')
+        path = $1
+        host = pkt.dst.to_s
+        host << ":\e[1;33m#{pkt.dport}\e[0m\ "
+        s = "\e[1;33m#{pkt.src}:\e[1;31m#{pkt.sport}\e[0m\ > GET \e[1;33mhttp://#{host}\e[1;32mHTTP/1.1\e[0m "
+        geo = gloc.geoiplookup(host.split(":")[0])
+        puts "\e[4;36mGeo Location:\e[0m\ \n\e[0;35m#{geo} \e[0m\ "
+        puts "\e[4;36mRequest Data:\e[0m\ \n\e[0;32m#{data_highlight} \e[0m\ "
+        tv.makeurl(data_orig)
+        # Process data and prepare then send elsewhere
+        query_return_sql = tv.request_filter(HTTP_REQUEST_TABLE, data)
+        # Store data into InfluxDB API Capture if @mtable exists else Datastore
+        begin
+          ires = gloc.query("#{query_return_sql}")
+          if @debug == true
+            puts "\e[4;36mStructured Query:\e[0m\ #{query_return_sql} \e[4;36mResult:\e[0m\ #{ires}"
+          end
+        rescue
+          Tools::log_errors("/tmp/thm-sql-errors.log", "SQL Error - #{Time.now} - #{query_return_sql}") # Catch them all
+        end
+        stwt.watch('stop')
+        stwt.print_stats
+      elsif data =~ %r=^POST $=
+        puts data_highlight
+      end
+    when HTTP_RESPONSE
+      if data =~ HTTP_METHODS_REGEXP_RESPONSE
+        stwt = Stopwatch.new
+        stwt.watch('start')
+        status = $1
+        s = "#{pkt.dst}:#{pkt.dport} < #{status}"
+        puts "\e[1;31mResponse Data: #{data_highlight} \e[0m\ "
+        stwt.watch('stop')
+        stwt.print_stats
+      end
+    end
+  puts s.gsub("GET", "\e[1;36mGET\e[0m").gsub("POST", "\e[1;36mPOST\e[0m") if s
+}

data/bin/thm-useradmin

@@ -2,7 +2,7 @@
 ########################################################################
 #
 # Author: Brian Hood
-#
+# Email: <brianh6854@googlemail.com>
 # Description: Threatmonitor User Administration
 # 
 # Extends the functionality of the Thm module adding Authorization
@@ -23,9 +23,21 @@ trap("INT") {
   exit
 }
 
+conf = Thm::FileServices.new
+conf.thmhome?
+
+include Thm::Defaults
+
 obj = Thm::Authorization::Privileges.new
-obj.datastore = "monetdb"
+obj.datastore = DATASTORE
+obj.debug = false
+obj.autocommit = true
+obj.dbhost = DBHOST
+obj.dbuser = DBUSER
+obj.dbpass = DBPASS
+obj.dbname = DBNAME
 obj.dbconnect
+
 if obj.user_exists?("admin") == true
   # Menu
   while buf = Readline.readline("\e[1;32m\Threatmonitor>\e[0m\ ", true)

data/config.rb

@@ -1,7 +1,7 @@
 ########################################################################
 #
 # Author: Brian Hood
-#
+# Email: <brianh6854@googlemail.com>
 # Description: Threatmonitor User configuration
 #
 # Lets keep it simple and use constants
@@ -10,21 +10,31 @@
 module Thm
 
   module Defaults
+  
+    DATASTORE = "monetdb"
+    MQHOST = "127.0.0.1"
+    MQUSER = "traffic"
+    MQPASS = "dk3rbi9l"
+    MQVHOST = "/"
+    DBHOST = "127.0.0.1"
+    DBUSER = "threatmonitor"
+    DBPASS = "dk3rbi9l"
+    DBNAME = "threatmonitor"
+    QUEUEPREFIX = "wifi"
+    TBLNAME_IPPACKET = "#{QUEUEPREFIX}_ippacket"
+    TBLNAME_TCPPACKET = "#{QUEUEPREFIX}_tcppacket"
+    TBLNAME_UDPPACKET = "#{QUEUEPREFIX}_udppacket"
+
+    # Regexp strings first currently excludes POST Data
+    HTTP_METHODS_REGEXP = %r=^GET |^HEAD |^PUT |^TRACE |^CONNECT |^OPTIONS |^DELETE |^PROPFIND |^PROPPATCH |^MKCOL |^COPY |^MOVE |^LOCK |^UNLOCK =
+    HTTP_METHODS_REGEXP_RESPONSE = %r=^(HTTP\/.*)$=
+    HTTP_REQUEST_TABLE = "http_traffic_json"
+    HTTP_RESPONSE_TABLE = "http_traffic_json"
     
-      DATASTORE = "monetdb"
-      MQHOST = "127.0.0.1"
-      MQUSER = "traffic"
-      MQPASS = "dk3rbi9l"
-      MQVHOST = "/"
-      DBHOST = "127.0.0.1"
-      DBUSER = "threatmonitor"
-      DBPASS = "dk3rbi9l"
-      DBNAME = "threatmonitor"
-      QUEUEPREFIX = "wifi"
-      TBLNAME_IPPACKET = "#{QUEUEPREFIX}_ippacket"
-      TBLNAME_TCPPACKET = "#{QUEUEPREFIX}_tcppacket"
-      TBLNAME_UDPPACKET = "#{QUEUEPREFIX}_udppacket"
-          
+    # Misc
+    SNAPLENGTH = 65536
+    INTERFACE = "eth0"
+
   end
 
 end

data/lib/thm/consumer.rb

@@ -1,3 +1,12 @@
+########################################################################
+#
+# Author: Brian Hood
+# Email: <brianh6854@googlemail.com>
+# Description: Libraries
+#
+#   Pull data from required sources
+#
+########################################################################
 
 trap("INT") {
   

data/lib/thm/datalayerlight.rb

@@ -149,7 +149,7 @@ module DatalayerLight
     require "json"
     require "pp"
   
-    attr_accessor :dbhost, :dbuser, :dbpass, :dbport, :dburl
+    attr_accessor :dbhost, :dbuser, :dbpass, :dbport, :dburl, :dbname
 
     def initialize
       @dbhost = "127.0.0.1"
@@ -189,13 +189,16 @@ module DatalayerLight
       begin
         request.body = data unless data.empty?
         response = http.request(request)
-        if response.code == 204 # Good response
-          puts "OK"
-        elsif response.code == 200 or response.code == 400 # 200 can be an error in some cases !!
+        if response.code == "204" # Good response
+          # Be quiet
+          return response.code
+        elsif response.code =~ %r=[200,400,500]= # 200 can be an error in some cases !!
           puts "Error code #{response.code}"
+          return response.code
         end
       rescue
         puts "Error posting data"
+        return "404"
       end
     end
     

data/lib/thm/dataservices/geolocation/geolocation.rb

@@ -0,0 +1,92 @@
+########################################################################
+#
+# Author: Brian Hood
+# Email: <brianh6854@googlemail.com>
+# Description: Libraries - geolocation
+#
+#   Geo Data quick and simple
+#
+########################################################################
+
+require 'pp'
+
+module Thm
+
+  class DataServices::Geolocation < DataServices
+  
+    attr_writer :geodebug
+    
+    def initialize
+      @geodebug = false
+      @continent_name, @country_name, @city_name = Array.new, Array.new, Array.new
+    end
+    
+    def formatinet(ip, octets=2)
+      if octets == 2
+        "#{ip.split(".")[0]}.#{ip.split(".")[1]}"
+      elsif octets == 3
+        "#{ip.split(".")[0]}.#{ip.split(".")[1]}.#{ip.split(".")[2]}"
+      end
+    end
+    
+    def self.define_component(name)
+      name_func = name.to_sym
+      define_method(name_func) do |ip|
+        octets = formatinet(ip, 2)
+        geoquery = "SELECT count(*) as num FROM geoipdata_ipv4blocks_#{name_func} a "
+        geoquery << "JOIN geoipdata_locations_#{name_func} b ON (a.geoname_id = b.geoname_id) "
+        geoquery << "WHERE network LIKE '#{octets}.%';"
+        begin
+          res = @conn.query("#{geoquery}")
+          rowgeocount = res.fetch_hash
+          geocount = rowgeocount["num"].to_i
+        rescue => e
+          pp e
+        end
+        puts "Geo SELECT COUNT: #{geoquery}: Number: #{geocount}" if @geodebug == true
+        if geocount > 0;
+          geoquery = "SELECT continent_name, #{name_func}_name FROM geoipdata_ipv4blocks_#{name_func} a "
+          geoquery << "JOIN geoipdata_locations_#{name_func} b ON (a.geoname_id = b.geoname_id) "
+          geoquery << "WHERE network LIKE '#{octets}.%' GROUP BY b.continent_name, b.#{name_func}_name LIMIT 1;"
+          puts "Geo SELECT: #{geoquery}" if @geodebug == true
+          begin
+            resgeo = @conn.query("#{geoquery}")
+            while row = resgeo.fetch_hash do
+              populategeo = instance_variable_get("@#{name_func}_name")
+              populategeo << row["#{name_func}_name"].to_s
+              instance_variable_set("@#{name_func}_name", populategeo)
+              @continent_name = row["continent_name"].to_s
+            end
+          rescue => e
+            pp e
+          end
+        else
+          return false
+        end
+      end
+    end
+
+    define_component :country
+    define_component :city
+
+    def geoiplookup(ip)
+      t = country(ip)
+      city(ip)
+      unless t == false
+        res = "(#{@continent_name}) - \n"
+        @country_name.each {|n|
+          res << "[ #{n} ]"
+        }
+        @city_name.each {|n|
+          res << "[ #{n} ] "
+        }
+        initialize
+      else
+        res = "Not Available"
+      end
+      return res
+    end
+
+  end
+
+end

data/lib/thm/dataservices/trafviz/trafviz.rb

@@ -0,0 +1,152 @@
+########################################################################
+#
+# Author: Brian Hood
+# Email: <brianh6854@googlemail.com>
+# Description: Libraries - Trafviz
+#
+#  Data parsing functionality
+#
+########################################################################
+
+require 'json'
+
+module Thm
+
+  class DataServices::Trafviz
+    
+    def initialize
+      @debug = true
+    end
+    
+    def makeurl(data)
+      if !request_valid?(data)
+        return false
+      end
+      hdrs = data
+      hostn, requestn = ""
+      hdrs.each_line {|n|
+        if n.split(":")[0] == "Host"
+          hostn = n.split(":")[1].strip
+        elsif n.split(" ")[0] == "GET"
+          requestn = n.split(" ")[1]
+        end
+      }
+      puts "\e[1;37mURL: http://#{hostn}#{requestn} \e[0m\ "
+    end
+
+    # Check if a request isn't just a GET line without headers / single line
+    # Not sure if this is valid HTTP
+    def request_valid?(data)
+      ln = 0
+      data.each_line {|l|
+        ln += 1
+      }
+      if ln > 1
+        return true
+      else
+        puts "\e[1;31mCatch GET's without header information / Other \e[0m\ "
+        return false # Due to single GET Requests to no headers 
+      end
+    end
+    
+    # This is just an informal function when in debug mode
+    def hit_header(hdrs)
+      puts "Hit #{hdrs} header"
+    end
+    
+    # Cookie ommit as we don't want to steal cookie data and pointless to store.
+    def filter_header?(lkey)
+      puts "MY LKEY: |#{lkey}|" if @debug == true
+      case lkey.strip
+      when "cookie"
+        hit_header(lkey) if @debug == true
+        return true
+      when "range"
+        hit_header(lkey) if @debug == true
+        return true
+      else
+        return false
+      end
+    end
+    
+    # Right Cell eval
+    def rkey_decode(rkey)
+      rkeyenc = URI.decode(rkey)
+    end
+    
+    # Filter lkey = header, rkey = requestdata
+    def lkey_strip(hdrs)
+      hdrs.split(": ")[0].downcase.gsub("-", "").to_s.strip
+    end
+    
+    def rkey_strip(data)
+      data.split(": ")[1].to_s.strip #to_s.gsub(",", "").gsub(";", "").gsub("=", "").strip
+    end
+    
+    # Filter request data and build query
+    def request_filter(reqtable, data, keysamples=2000)
+      if !request_valid?(data)
+        sql = "SELECT 1;"
+        return sql
+      end
+      guid = Tools::guid
+      cols, vals = String.new, String.new
+      lkey, rkey = String.new, String.new
+      json_data_pieces = String.new
+      t = 0
+      json_data_hdr = "@json_template = { 'http' => { "
+      json_data_ftr = " } }"
+      sql = "INSERT INTO #{reqtable} (recv_time,recv_date,guid,json_data) "
+      data.each_line {|n|
+        unless n.strip == ""
+          if t > 0 # Don't processes GET / POST Line
+            lkey, rkey = lkey_strip(n), rkey_strip(n)
+            puts "LKEY: #{lkey} RKEY: #{rkey}" if @debug == true
+            rkeyenc = filter_header?(lkey)
+            if rkeyenc == false
+              rkeyenc = rkey_decode(rkey)
+            else 
+              rkey = "ommited"
+            end
+            if rkey.strip != "" or lkey.strip != ""
+              prerkeyins = rkey.gsub('"', '') # Strip Quotes
+              prerkeyins = "blank" if prerkeyins.strip == "" # Seems JSON values can't be "accept":""
+              puts "Found Blank Value!!!" if prerkeyins == "blank"
+              json_data_pieces << "'#{lkey}' => \"#{prerkeyins}\",\n"
+            end
+          end
+          t += 1
+        end
+      }
+      # SQL for Datastore
+      begin
+        # Remove last , to fix hash table
+        json_data_pieces.sub!(%r{,\n$}, "")
+        json_eval = %Q{#{json_data_hdr}#{json_data_pieces}#{json_data_ftr}}
+        puts "\e[4;36mJSON Data:\e[0m\ \n#{json_eval}"
+        eval(json_eval) # Unsure why a local variable works for this in IRB
+        json_data = @json_template.to_json
+        remove_instance_variable("@json_template") # Hence remove instance variable here
+        # Added GUID as i could extend TCP/IP capture suites in the future for HTTP traffic 
+        sql = "#{sql}VALUES (NOW(), NOW(), '#{guid}', '#{json_data}');"
+        return sql
+      rescue => e
+        pp e
+      end
+    end
+    
+    def text_highlighter(text)
+      keys = ["Linux", "Java", "Android", "iPhone", "Mobile", "Chrome", 
+              "Safari", "Mozilla", "Gecko", "AppleWebKit", "Windows", 
+              "MSIE", "Win64", "Trident", "wispr", "PHPSESSID", "JSESSIONID",
+              "AMD64", "Darwin", "Macintosh", "Mac OS X", "Dalvik", "text/html", "xml"]
+      cpicker = [2,3,4,1,7,5,6]
+      keys.each {|n|
+        text.gsub!("#{n}", "\e[4;3#{cpicker[rand(cpicker.size)]}m#{n}\e[0m\ \e[0;32m".strip)
+      }
+      return text
+    end
+
+  end
+
+end

data/lib/thm/dataservices.rb

@@ -13,10 +13,12 @@ module Thm
     #  obj.mqconnect
     #  obj.dbconnect
     
-    attr_accessor :datastore, :mqhost, :mquser, :mqpass, :mqvhost, :dbhost, :dbuser, :dbpass, :dbname, :queueprefix, :tblname_ippacket, :tblname_tcppacket, :tblname_udppacket
+    attr_accessor :autocommit, :datastore, :debug, :mqhost, :mquser, :mqpass, :mqvhost, :dbhost, :dbuser, :dbpass, :dbname, :queueprefix, :tblname_ippacket, :tblname_tcppacket, :tblname_udppacket
     
     def initialize
+      @autocommit = false
       @datastore = "monetdb"
+      @debug = false
       @mqhost = "127.0.0.1"
       @mquser = "traffic"
       @mqpass = "dk3rbi9l"
@@ -54,7 +56,8 @@ module Thm
       @conn.username = @dbuser
       @conn.password = @dbpass
       @conn.dbname = @dbname
-      @conn.autocommit = false
+      @conn.debug = @debug
+      @conn.autocommit = @autocommit
       begin
         @conn.connect
       rescue Errno::ECONNREFUSED
@@ -71,3 +74,7 @@ module Thm
   end
 
 end
+
+require_relative 'dataservices/trafviz/trafviz.rb'
+require_relative 'dataservices/geolocation/geolocation.rb'
+

data/lib/thm/fileservices.rb

@@ -1,3 +1,12 @@
+########################################################################
+#
+# Author: Brian Hood
+# Email: <brianh6854@googlemail.com>
+# Description: Libraries
+#
+#   File services for local / global settings
+#
+########################################################################
 
 module Thm
 
@@ -15,10 +24,10 @@ module Thm
         }
       end
       begin
-        if loadswitch == true
-          require Dir.home+"/.thm/#{file}"
+        if loadswitch == true # So original backup config doesn't change your settings
+          require "#{Dir.home}/.thm/#{file}"
         end
-       rescue
+      rescue
         puts "Failed to load something went wrong check permissions !"
       end
     end
@@ -28,7 +37,7 @@ module Thm
       if Dir.exists?("#{Dir.home}/.thm") == false
         Dir.mkdir("#{Dir.home}/.thm")
         puts "Creating .thm home subfolder copying config.rb"
-	#puts "#{File.getwd}"
+        #puts "#{File.getwd}"
         File.open(File.expand_path(File.join(File.dirname(__FILE__), "../../#{file}")), 'r') {|n|
           n.each_line {|l|
             @fdata << l
@@ -36,8 +45,8 @@ module Thm
         }
       end
       begin
-	#puts "FDATA: #{@fdata}"
-	#puts "Begin"
+        #puts "FDATA: #{@fdata}"
+        #puts "Begin"
         conf_loader("#{file}")
         conf_loader("config-original.rb", false)
       rescue

data/lib/thm/localmachine.rb

@@ -1,3 +1,12 @@
+########################################################################
+#
+# Author: Brian Hood
+# Email: <brianh6854@googlemail.com>
+# Description: Libraries
+#
+#   Remote to Local Pcap operations
+#
+########################################################################
 
 module Thm
 

data/lib/thm/producer.rb

@@ -1,3 +1,13 @@
+########################################################################
+#
+# Author: Brian Hood
+# Email: <brianh6854@googlemail.com>
+# Description: Libraries
+#
+#   Message queue data producer / data transport
+#
+########################################################################
+
 module Thm
   # Send data else were
   

data/lib/thm/version.rb

@@ -3,9 +3,9 @@ module Thm #:nodoc:
   module VERSION #:nodoc:
 
     MAJOR = 0
-    MINOR = 2
-    TINY = 2
-    CODENAME = "Deedrah"
+    MINOR = 3
+    TINY = 1
+    CODENAME = "Micel"
 
     STRING = [MAJOR, MINOR, TINY].join('.')
 

data/lib/thm.rb

@@ -1,7 +1,7 @@
 ########################################################################
 #
 # Author: Brian Hood
-#
+# Email: <brianh6854@googlemail.com>
 # Description: Threatmonitor Producer
 #
 # Producer / Consumer controller module
@@ -22,7 +22,15 @@ include Pcap
 #
 # Create def's for that packet SQL / Refactor to provent code duplication
 # Create def's for Hash table YAML same idea as above.
-    
+
+class String
+
+  def size_minus(min=1)
+    size - min
+  end
+
+end
+
 module Tools
 
   class << self
@@ -30,38 +38,59 @@ module Tools
     def guid
       guid = Guid.new # Generate GUID
     end
-  
+
+    def log_errors(file, data)
+      File.open("#{file}", 'a') {|n|
+        n.puts("#{data}")
+      }
+    end
+    
+  end
+
+  def use_const_defined_unless?(const)
+    const_down = const.downcase
+    if Kernel.const_defined?("#{const}")
+      if instance_variable_get("@#{const_down}") == nil
+        instance_variable_set("@#{const_down}", Kernel.const_get("#{const}"))
+        puts "Config Constant #{const}: #{Kernel.const_get("#{const}")}"
+        puts "Instance Variable @#{const_down}: #{instance_variable_get("@#{const_down}")}"
+      else
+        puts "Param via Getoptlong: Instance Variable #{@const_down}: #{instance_variable_get("@#{const_down}")}"
+      end
+    else
+      raise "No Config option set add #{const} to your config.rb"
+    end
   end
   
 end
 
 # Load Database drivers
 require File.expand_path(File.join(
-          File.dirname(__FILE__),
-          "../lib/thm/datalayerlight.rb"))
+        File.dirname(__FILE__),
+        "../lib/thm/datalayerlight.rb"))
           
 # Load Datasources / Services contains defaults
 require File.expand_path(File.join(
-          File.dirname(__FILE__),
-          "../lib/thm/dataservices.rb"))
+        File.dirname(__FILE__),
+        "../lib/thm/dataservices.rb"))
 
 require File.expand_path(File.join(
-          File.dirname(__FILE__), 
-	  "../lib/thm/producer.rb"))
+        File.dirname(__FILE__), 
+        "../lib/thm/producer.rb"))
 
 require File.expand_path(File.join(
-          File.dirname(__FILE__),
-	  "../lib/thm/consumer.rb"))
+        File.dirname(__FILE__),
+        "../lib/thm/consumer.rb"))
 
 require File.expand_path(File.join(
-          File.dirname(__FILE__), 
-	  "../lib/thm/localmachine.rb"))
+        File.dirname(__FILE__), 
+        "../lib/thm/localmachine.rb"))
 
 require File.expand_path(File.join(
-          File.dirname(__FILE__), 
-	  "../lib/thm/fileservices.rb"))
+        File.dirname(__FILE__), 
+        "../lib/thm/fileservices.rb"))
 
 # Versioning information
 require File.expand_path(File.join(
-          File.dirname(__FILE__),
-	  "../lib/thm/version.rb"))
+        File.dirname(__FILE__),
+        "../lib/thm/version.rb"))

data/sql/geoipdata-monetdb.sql

@@ -9,15 +9,15 @@
 
 DROP TABLE "threatmonitor".geoipdata_ipv4blocks_city;
 CREATE TABLE "threatmonitor".geoipdata_ipv4blocks_city (
-  network varchar(18),
-  geoname_id char(10),
-  registered_country_geoname_id char(30),
-  represented_country_geoname_id char(30),
-  is_anonymous_proxy char(30),
-  is_satellite_provider char(30),
-  postal_code char(30),
-  latitude char(10),
-  longitude char(10)
+  network varchar(18) NOT NULL,
+  geoname_id char(10) NOT NULL,
+  registered_country_geoname_id char(30) NOT NULL,
+  represented_country_geoname_id char(30) NOT NULL,
+  is_anonymous_proxy char(30) NOT NULL,
+  is_satellite_provider char(30) NOT NULL,
+  postal_code char(30) NOT NULL,
+  latitude char(10) NOT NULL,
+  longitude char(10) NOT NULL
 );
 
 CREATE INDEX cindex_ipv4_network ON "threatmonitor".geoipdata_ipv4blocks_city(network);
@@ -27,19 +27,19 @@ COPY 2519918 OFFSET 2 RECORDS INTO "threatmonitor".geoipdata_ipv4blocks_city FRO
 
 DROP TABLE "threatmonitor".geoipdata_locations_city;
 CREATE TABLE "threatmonitor".geoipdata_locations_city (
-  geoname_id char(10),
-  locale_code char(2),
-  continent_code char(2),
-  continent_name char(15),
-  country_iso_code char(2),
-  country_name char(50),
-  subdivision_1_iso_code char(70),
-  subdivision_1_name char(50),
-  subdivision_2_iso_code char(70),
-  subdivision_2_name char(50),
-  city_name char(70),
-  metro_code char(30),
-  time_zone char(30)
+  geoname_id char(10) NOT NULL,
+  locale_code char(2) NOT NULL,
+  continent_code char(2) NOT NULL,
+  continent_name char(15) NOT NULL,
+  country_iso_code char(2) NOT NULL,
+  country_name char(50) NOT NULL,
+  subdivision_1_iso_code char(70) NOT NULL,
+  subdivision_1_name char(50) NOT NULL,
+  subdivision_2_iso_code char(70) NOT NULL,
+  subdivision_2_name char(50) NOT NULL,
+  city_name char(70) NOT NULL,
+  metro_code char(30) NOT NULL,
+  time_zone char(30) NOT NULL
 );
 
 CREATE INDEX cindex_country_geoname_id ON "threatmonitor".geoipdata_locations_city(geoname_id);
@@ -48,12 +48,12 @@ COPY 80006 OFFSET 2 RECORDS INTO "threatmonitor".geoipdata_locations_city FROM '
 
 DROP TABLE "threatmonitor".geoipdata_ipv4blocks_country;
 CREATE TABLE "threatmonitor".geoipdata_ipv4blocks_country (
-  network varchar(18),
-  geoname_id char(10),
-  registered_country_geoname_id char(30),
-  represented_country_geoname_id char(30),
-  is_anonymous_proxy char(30),
-  is_satellite_provider char(30)
+  network varchar(18) NOT NULL,
+  geoname_id char(10) NOT NULL,
+  registered_country_geoname_id char(30) NOT NULL,
+  represented_country_geoname_id char(30) NOT NULL,
+  is_anonymous_proxy char(30) NOT NULL,
+  is_satellite_provider char(30) NOT NULL
 );
 
 CREATE INDEX index_ipv4_network ON "threatmonitor".geoipdata_ipv4blocks_country(network);
@@ -62,12 +62,12 @@ COPY 169357 OFFSET 2 RECORDS INTO "threatmonitor".geoipdata_ipv4blocks_country F
 
 DROP TABLE "threatmonitor".geoipdata_locations_country;
 CREATE TABLE "threatmonitor".geoipdata_locations_country (
-  geoname_id char(10),
-  locale_code char(2),
-  continent_code char(2),
-  continent_name char(15),
-  country_iso_code char(2),
-  country_name char(50)
+  geoname_id char(10) NOT NULL,
+  locale_code char(2) NOT NULL,
+  continent_code char(2) NOT NULL,
+  continent_name char(15) NOT NULL,
+  country_iso_code char(2) NOT NULL,
+  country_name char(50) NOT NULL
 -- FOREIGN KEY (geoname_id) REFERENCES "geoipdata".geoipdata_ipv4blocks_country (index_geoname_id)
 );
 

data/sql/threatmonitor-http.sql

@@ -0,0 +1,15 @@
+
+DROP TABLE "threatmonitor".http_traffic_json;
+CREATE TABLE "threatmonitor".http_traffic_json (
+id INT GENERATED ALWAYS AS 
+        IDENTITY (
+           START WITH 0 INCREMENT BY 1
+           NO MINVALUE NO MAXVALUE
+           CACHE 2 CYCLE
+) primary key,
+  guid char(36),
+  recv_date date,
+  recv_time time,
+  json_data JSON
+);
+

data/thm-authentication.rb

@@ -2,8 +2,9 @@
 #
 # Author: Brian Hood
 #
+# Email: <brianh6854@googlemail.com>
 # Description: Threatmonitor User Administration
-# 
+#
 # Extends the functionality of the Thm module adding Authorization
 # Adding Authentication to the Privileges model
 #
@@ -24,7 +25,7 @@ module Thm::Authorization
     
     def initialize
       super
-      @debug = 1
+      @debug = true
     end
     
     def login(username, password)

data/thm-authorization.rb

@@ -1,7 +1,7 @@
 ########################################################################
 #
 # Author: Brian Hood
-#
+# Email: <brianh6854@googlemail.com>
 # Description: Threatmonitor Authorization
 # 
 # Extends the functionality of the Thm module adding Authorization
@@ -21,7 +21,7 @@ module Thm::Authorization
     
     def initialize
       super
-      @debug = 1
+      @debug = true
     end
     
     def setup_privileges(name, obj)

data/thm-privileges.rb

@@ -1,7 +1,7 @@
 ########################################################################
 #
 # Author: Brian Hood
-#
+# Email: <brianh6854@googlemail.com>
 # Description: Threatmonitor User Administration
 # 
 # Extends the functionality of the Thm module adding Authorization

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: thm
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.1
 platform: ruby
 authors:
 - puppetpies
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-20 00:00:00.000000000 Z
+date: 2015-09-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -164,6 +164,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: keycounter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.0'
 description: Threatmonitor - Packet Capture / Analysis Suite
 email: brianh6854@googlemail.com
 executables:
@@ -171,6 +185,7 @@ executables:
 - thm-pcap
 - thm-producer
 - thm-session
+- thm-trafviz
 - thm-useradmin
 extensions: []
 extra_rdoc_files:
@@ -183,6 +198,7 @@ files:
 - bin/thm-pcap
 - bin/thm-producer
 - bin/thm-session
+- bin/thm-trafviz
 - bin/thm-useradmin
 - config.rb
 - js/JSXTransformer.js
@@ -196,12 +212,15 @@ files:
 - lib/thm/consumer.rb
 - lib/thm/datalayerlight.rb
 - lib/thm/dataservices.rb
+- lib/thm/dataservices/geolocation/geolocation.rb
+- lib/thm/dataservices/trafviz/trafviz.rb
 - lib/thm/fileservices.rb
 - lib/thm/localmachine.rb
 - lib/thm/producer.rb
 - lib/thm/version.rb
 - service_definitions.csv
 - sql/geoipdata-monetdb.sql
+- sql/threatmonitor-http.sql
 - sql/threatmonitor-monetdb.sql
 - sql/threatmonitor-mysql.sql
 - stylesheets/screen.css