thm 0.3.2 → 0.4.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 573f88eb1f845128f054c18907b9200a18e7cc11
-  data.tar.gz: 845b4ec58b467afe8b6ff70de6d761e83e66618c
+  metadata.gz: 40bbc67ea101f1fc34f2f85fdb06c13677451fd0
+  data.tar.gz: 78a08e73c11da31adf4834dbd5913fbfbb93eb35
 SHA512:
-  metadata.gz: 80d474d2ef0fe7ae85845e7c77028745da77e2d1b8daa71d1cedb4b5c487cb292ef8f86bd63d323d4ccd93a50855687a68db0d5379ddf852ad376006a5af765e
-  data.tar.gz: 98b61c0aa0e5d762aacb181c783d92ad2e188329e305877ec2728967ad4e5267bf9b08667308566e09e9e5b78b7a3dff505f7f70623b1c23df8b93f3f9ffc2ca
+  metadata.gz: fcd88da68f3f44c7072378134bd3c2ab644b1db8f07db52d43a1644009d76ba92d8787b4b4c625b47b5199a922b7e80e72e69624766e496abcedae914910c3c4
+  data.tar.gz: 8ce4134539e86fae9dac11d7e9cfedde5310c80a9088eafb7a94f02d575b3ee5e6c4bb4e183b8409569d329e7aa8b048c21edd1ff2e4938be87381e10777cf0b

data/Rakefile

@@ -0,0 +1,107 @@
+# coding: utf-8
+require './lib/thm/version.rb'
+
+def java?
+  /java/ === RUBY_PLATFORM
+end
+
+ENV['LANG'] = "en_US.UTF-8"
+
+VERSION = Thm::VERSION::STRING
+
+Gem::Specification.new do |spec|
+  spec.name          = "thm"
+  spec.version       = VERSION
+  spec.authors       = ["puppetpies"]
+  spec.email         = "brianh6854@googlemail.com"
+  spec.description   = "Threatmonitor - Packet Capture / Analysis Suite"
+  spec.summary       = "Packet Data Analysis"
+  spec.executables = ["thm-consumer", "thm-producer", "thm-session", "thm-useradmin", "thm-pcap", "thm-trafviz"]
+  spec.homepage      = "https://github.com/puppetpies/threatmonitor"
+  spec.requirements  = "libpcap"
+  spec.license       = "MIT"
+
+  spec.files = [
+    "config.rb",
+    "Rakefile",
+    "lib/thm/datalayerlight.rb",
+    "thm-authentication.rb",
+    "thm-authorization.rb",
+    "bin/thm-consumer",
+    "bin/thm-producer",
+    "bin/thm-session",
+    "bin/thm-useradmin",
+    "bin/thm-pcap",
+    "bin/thm-trafviz",
+    "thm-privileges.rb",
+    "service_definitions.csv",
+    "lib/thm.rb",
+    "lib/thm/consumer.rb",
+    "lib/thm/dataservices.rb",
+    "lib/thm/fileservices.rb",
+    "lib/thm/localmachine.rb",
+    "lib/thm/producer.rb",
+    "lib/thm/version.rb",
+    "lib/thm/dataservices/geolocation/geolocation.rb",
+    "lib/thm/dataservices/trafviz/trafviz.rb",
+    "js/jquery.min.js",
+    "js/chartkick.js",
+    "js/JSXTransformer.js",
+    "js/marked.min.js",
+    "js/react.js",
+    "js/jsapi.js",
+    "js/files/authenticate.jsx",
+    "stylesheets/screen.css",
+    "sql/geoipdata-monetdb.sql",
+    "sql/threatmonitor-monetdb.sql",
+    "sql/threatmonitor-mysql.sql",
+    "sql/threatmonitor-http.sql",
+    "views/authenticate.slim",
+    "views/dashboard.erb",
+    "views/logout.slim"
+  ]
+
+  spec.extra_rdoc_files = [
+    "README.md",
+    "README.1ST"
+  ]
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake", "~> 10.4"
+  spec.add_development_dependency "rake-compiler", "~> 0.9"
+  spec.add_runtime_dependency "bunny", "~> 1.7"
+  spec.add_runtime_dependency "amqp", "~> 1.5"
+  spec.add_runtime_dependency "pcap", "~> 0.7"
+  spec.add_runtime_dependency "guid", "~> 0.1"
+  spec.add_runtime_dependency "eventmachine", "~> 1.0"
+  spec.add_runtime_dependency "chartkick", "~> 1.3"
+  spec.add_runtime_dependency "sinatra", "~> 1.4"
+  spec.add_runtime_dependency "slim", "~> 3.0"
+  spec.add_runtime_dependency "keycounter", "~> 0.0.8"
+  spec.add_runtime_dependency "walltime", "~> 0.0.5"
+end
+
+require 'rubygems/tasks'
+Gem::Tasks.new
+task :default do
+
+end
+
+# Override standard release task
+require 'git'
+Rake::Task["release"].clear
+task :release do
+  version = "#{VERSION}"
+  remote = 'origin'
+  puts "Creating tag v#{version}"
+  git = Git.open(".")
+  git.add_tag("v#{version}")
+  puts "Pushing tag to #{remote}"
+  git.push(remote, 'master', true)
+  Rake::Task['gem'].invoke
+  gemtask = Gem::Tasks::Push.new
+  gemtask.push("pkg/thm-#{version}.gem")
+end

data/bin/thm-trafviz

@@ -26,6 +26,16 @@ conf.thmhome?
 include Thm::Defaults
 include Tools
 
+class FalseClass
+
+  def []
+=begin
+#<NoMethodError: undefined method `[]' for false:FalseClass>
+=end
+  end
+  
+end
+
 class NilClass
 
   def strip
@@ -42,6 +52,18 @@ exception when looping over each packet loop: #<NoMethodError: undefined method
 =end
   end
   
+  def > name=nil
+=begin
+exception when looping over each packet loop: #<NoMethodError: undefined method `>' for nil:NilClass>
+/data2/Projects/threatmonitor/lib/thm/dataservices/geolocation/geolocation.rb:47:in `block in define_component': undefined method `>' for nil:NilClass (NoMethodError)
+	from /data2/Projects/threatmonitor/lib/thm/dataservices/geolocation/geolocation.rb:73:in `geoiplookup'
+	from ./thm-trafviz:284:in `block in <main>'
+	from /usr/lib/ruby/gems/2.1.0/gems/pcap-0.7.7/lib/pcaplet.rb:94:in `loop'
+	from /usr/lib/ruby/gems/2.1.0/gems/pcap-0.7.7/lib/pcaplet.rb:94:in `each_packet'
+	from ./thm-trafviz:271:in `<main>'
+=end
+  end
+  
 end
 
 ARGV[0] = "--help" if ARGV[0] == nil
@@ -91,6 +113,8 @@ puts banner
 
 # Trafviz DataServices
 tv = Thm::DataServices::Trafviz.new
+tv.reqtable = HTTP_REQUEST_TABLE
+tv.reqtableua = HTTP_REQUEST_TABLE_UA
 # Connect to Datastore
 gloc = Thm::DataServices::Geolocation.new
 gloc.datastore = DATASTORE
@@ -264,13 +288,13 @@ a.menu!
 =end
 
 @trafviz = Pcaplet.new(startup)
-HTTP_REQUEST  = Pcap::Filter.new('tcp dst port 80', @trafviz.capture)
+HTTP_REQUEST = Pcap::Filter.new('tcp dst port 80', @trafviz.capture)
 HTTP_RESPONSE = Pcap::Filter.new('tcp src portrange 1024-65535', @trafviz.capture)
 
 @trafviz.add_filter(HTTP_REQUEST | HTTP_RESPONSE)
 @trafviz.each_packet {|pkt|
     data = pkt.tcp_data.to_s
-    data_orig = data.clone
+    data_orig = data.clone # Preserve copy in its own object_id
     data_highlight = tv.text_highlighter(data_orig)
     case pkt
     when HTTP_REQUEST
@@ -278,21 +302,22 @@ HTTP_RESPONSE = Pcap::Filter.new('tcp src portrange 1024-65535', @trafviz.captur
         stwt = Stopwatch.new
         stwt.watch('start')
         path = $1
-        host = pkt.dst.to_s
-        host << ":\e[1;33m#{pkt.dport}\e[0m\ "
+        host = "#{pkt.dst.to_s}:\e[1;33m#{pkt.dport}\e[0m\ "
         s = "\e[1;33m#{pkt.src}:\e[1;31m#{pkt.sport}\e[0m\ > GET \e[1;33mhttp://#{host}\e[1;32mHTTP/1.1\e[0m "
         geo = gloc.geoiplookup(host.split(":")[0])
         puts "\e[4;36mGeo Location:\e[0m\ \n\e[0;35m#{geo} \e[0m\ "
         puts "\e[4;36mRequest Data:\e[0m\ \n\e[0;32m#{data_highlight} \e[0m\ "
         tv.makeurl(data_orig)
         # Process data and prepare then send elsewhere
-        query_return_sql = tv.request_filter(HTTP_REQUEST_TABLE, data)
-        # Store data into InfluxDB API Capture if @mtable exists else Datastore
+        query_return_sql = tv.request_filter(data)
+        # Store data into Datastore
         begin
-          ires = gloc.query("#{query_return_sql}")
-          if @debug == true
-            puts "\e[4;36mStructured Query:\e[0m\ #{query_return_sql} \e[4;36mResult:\e[0m\ #{ires}"
-          end
+          query_return_sql.each {|sql|
+            ires = gloc.query("#{sql}")
+            if @debug == true
+              puts "\e[4;36mStructured Query:\e[0m\ #{sql} \e[4;36mResult:\e[0m\ #{ires}"
+            end
+          }
         rescue
           Tools::log_errors("/tmp/thm-sql-errors.log", "SQL Error - #{Time.now} - #{query_return_sql}") # Catch them all
         end

data/config.rb

@@ -30,6 +30,7 @@ module Thm
     HTTP_METHODS_REGEXP_RESPONSE = %r=^(HTTP\/.*)$=
     HTTP_REQUEST_TABLE = "http_traffic_json"
     HTTP_RESPONSE_TABLE = "http_traffic_json"
+    HTTP_REQUEST_TABLE_UA = "http_traffic_ua"
     
     # Misc
     SNAPLENGTH = 65536

data/lib/thm/dataservices/geolocation/geolocation.rb

@@ -54,7 +54,7 @@ module Thm
             while row = resgeo.fetch_hash do
               populategeo = instance_variable_get("@#{name_func}_name")
               populategeo << row["#{name_func}_name"].to_s
-              instance_variable_set("@#{name_func}_name", populategeo)
+              instance_variable_set("@#{name_func}_name", populategeo) # Only returns 1 row
               @continent_name = row["continent_name"].to_s
             end
           rescue => e

data/lib/thm/dataservices/trafviz/trafviz.rb

@@ -8,14 +8,37 @@
 #
 ########################################################################
 
+require 'pp'
 require 'json'
+require 'walltime'
 
-module Thm
+module TimeWarp
+
+  refine Stopwatch do
+
+    def print_stats
+      round = round_to(@t2 - @t1, 2)
+      puts "Start: #{Time.at(@t1)} Finish: #{Time.at(@t2)} Total time: #{round}"
+      diff = (Time.at(@t2) - Time.at(@t1))*1000
+      puts "Difference: #{diff.to_s.gsub(".", "")[0..2]}ms"
+    end
+    
+  end
+
+end
 
+module Thm
+    
   class DataServices::Trafviz
     
+    attr_writer :reqtable, :reqtableua
+    
+    # For refinement of print_stats 
+    using TimeWarp
+    
     def initialize
-      @debug = true
+      @debug = false
+      @reqtable, @reqtableua = String.new, String.new
     end
     
     def makeurl(data)
@@ -50,20 +73,25 @@ module Thm
     end
     
     # This is just an informal function when in debug mode
-    def hit_header(hdrs)
-      puts "Hit #{hdrs} header"
+    def hit_header(hdrs, comment="")
+      puts "Hit #{hdrs} header #{comment}"
     end
     
+    
     # Cookie ommit as we don't want to steal cookie data and pointless to store.
+    # Other useless headers / slight issues
     def filter_header?(lkey)
       puts "MY LKEY: |#{lkey}|" if @debug == true
-      case lkey.strip
-      when "cookie"
+      case 
+      when lkey == "cookie"
         hit_header(lkey) if @debug == true
         return true
-      when "range"
+      when lkey == "range"
         hit_header(lkey) if @debug == true
         return true
+      when lkey =~ /^get |^post /
+        hit_heaer(lkey, "Seen this unsure why it even occurs yet !") if @debug == true
+        return true
       else
         return false
       end
@@ -84,19 +112,22 @@ module Thm
     end
     
     # Filter request data and build query
-    def request_filter(reqtable, data, keysamples=2000)
+    def request_filter(data, keysamples=2000)
       if !request_valid?(data)
         sql = "SELECT 1;"
         return sql
       end
+      flt = Stopwatch.new
+      flt.watch('start')
       guid = Tools::guid
       cols, vals = String.new, String.new
       lkey, rkey = String.new, String.new
+      sql_ua = String.new
       json_data_pieces = String.new
       t = 0
       json_data_hdr = "@json_template = { 'http' => { "
       json_data_ftr = " } }"
-      sql = "INSERT INTO #{reqtable} (recv_time,recv_date,guid,json_data) "
+      sql = "INSERT INTO #{@reqtable} (recv_time,recv_date,guid,json_data) "
       data.each_line {|n|
         unless n.strip == ""
           if t > 0 # Don't processes GET / POST Line
@@ -105,14 +136,25 @@ module Thm
             rkeyenc = filter_header?(lkey)
             if rkeyenc == false
               rkeyenc = rkey_decode(rkey)
+              if lkey == "useragent"
+                ua = Tools::ua_parser(rkeyenc)
+                sql_ua = "INSERT INTO #{@reqtableua} (family, "
+                sql_ua << "major, minor, " unless ua.version == nil
+                sql_ua << "os, guid) "
+                sql_ua << "VALUES ('#{ua.family}', "
+                sql_ua << "'#{ua.version.major}', '#{ua.version.minor}', " unless ua.version == nil
+                sql_ua << "'#{ua.os.to_s}', '#{guid}');"
+              end
             else 
               rkey = "ommited"
             end
-            if rkey.strip != "" or lkey.strip != ""
+            if rkey != "" or lkey != ""
               prerkeyins = rkey.gsub('"', '') # Strip Quotes
               prerkeyins = "blank" if prerkeyins.strip == "" # Seems JSON values can't be "accept":""
               puts "Found Blank Value!!!" if prerkeyins == "blank"
-              json_data_pieces << "'#{lkey}' => \"#{prerkeyins}\",\n"
+              if lkey != "useragent"
+                json_data_pieces << "'#{lkey}' => \"#{prerkeyins}\",\n"
+              end
             end
           end
           t += 1
@@ -129,7 +171,10 @@ module Thm
         remove_instance_variable("@json_template") # Hence remove instance variable here
         # Added GUID as i could extend TCP/IP capture suites in the future for HTTP traffic 
         sql = "#{sql}VALUES (NOW(), NOW(), '#{guid}', '#{json_data}');"
-        return sql
+        flt.watch('stop')
+        print "\e[4;36mFilter Time Taken:\e[0m\ "
+        flt.print_stats
+        return [sql, sql_ua]
       rescue => e
         pp e
       end
@@ -140,7 +185,7 @@ module Thm
               "Safari", "Mozilla", "Gecko", "AppleWebKit", "Windows", 
               "MSIE", "Win64", "Trident", "wispr", "PHPSESSID", "JSESSIONID",
               "AMD64", "Darwin", "Macintosh", "Mac OS X", "Dalvik", "text/html", "xml"]
-      cpicker = [2,3,4,1,7,5,6]
+      cpicker = [2,3,4,1,7,5,6] # Just a selection of colours
       keys.each {|n|
         text.gsub!("#{n}", "\e[4;3#{cpicker[rand(cpicker.size)]}m#{n}\e[0m\ \e[0;32m".strip)
       }

data/lib/thm/version.rb

@@ -3,9 +3,9 @@ module Thm #:nodoc:
   module VERSION #:nodoc:
 
     MAJOR = 0
-    MINOR = 3
-    TINY = 2
-    CODENAME = "Micel"
+    MINOR = 4
+    TINY = 5
+    CODENAME = "The Isnis"
 
     STRING = [MAJOR, MINOR, TINY].join('.')
 

data/lib/thm.rb

@@ -16,13 +16,9 @@ require 'guid'
 require 'yaml'
 require 'pcaplet'
 require 'pcaprub' # For Live capture / write
+require 'user_agent_parser'
 include Pcap
 
-# TODO
-#
-# Create def's for that packet SQL / Refactor to provent code duplication
-# Create def's for Hash table YAML same idea as above.
-
 class String
 
   def size_minus(min=1)
@@ -34,11 +30,20 @@ end
 module Tools
 
   class << self
-  
+    
+    # Guid.new isn't hard but this Module will expand
     def guid
       guid = Guid.new # Generate GUID
     end
-
+    
+    # User agent parsing magic for Trafviz via uap-ruby on Github
+    def ua_parser(agent)
+      # Load all user agent data / regexp / patterns once
+      @ua ||= UserAgentParser::Parser.new
+      @ua.parse(agent)
+    end
+    
+    # Thm system errors
     def log_errors(file, data)
       File.open("#{file}", 'a') {|n|
         n.puts("#{data}")
@@ -47,6 +52,7 @@ module Tools
     
   end
 
+  # User defined functions
   def use_const_defined_unless?(const)
     const_down = const.downcase
     if Kernel.const_defined?("#{const}")
@@ -68,7 +74,7 @@ end
 require File.expand_path(File.join(
         File.dirname(__FILE__),
         "../lib/thm/datalayerlight.rb"))
-          
+
 # Load Datasources / Services contains defaults
 require File.expand_path(File.join(
         File.dirname(__FILE__),

data/sql/geoipdata-monetdb.sql

@@ -7,6 +7,8 @@
 --           CACHE 2 CYCLE
 -- ) primary key,
 
+SET SCHEMA "threatmonitor";
+
 DROP TABLE "threatmonitor".geoipdata_ipv4blocks_city;
 CREATE TABLE "threatmonitor".geoipdata_ipv4blocks_city (
   network varchar(18) NOT NULL,

data/sql/threatmonitor-http.sql

@@ -1,3 +1,4 @@
+SET SCHEMA "threatmonitor";
 
 DROP TABLE "threatmonitor".http_traffic_json;
 CREATE TABLE "threatmonitor".http_traffic_json (
@@ -6,10 +7,65 @@ id INT GENERATED ALWAYS AS
            START WITH 0 INCREMENT BY 1
            NO MINVALUE NO MAXVALUE
            CACHE 2 CYCLE
-) primary key,
-  guid char(36),
-  recv_date date,
-  recv_time time,
+) PRIMARY KEY,
+  guid CHAR(36) NOT NULL,
+  recv_date DATE,
+  recv_time TIME,
   json_data JSON
 );
 
+CREATE INDEX index_traffic_json_id ON "threatmonitor".http_traffic_json(id);
+CREATE INDEX index_traffic_json_guid ON "threatmonitor".http_traffic_json(guid);
+
+DROP TABLE "threatmonitor".http_traffic_ua;
+CREATE TABLE "threatmonitor".http_traffic_ua (
+id INT GENERATED ALWAYS AS 
+        IDENTITY (
+           START WITH 0 INCREMENT BY 1
+           NO MINVALUE NO MAXVALUE
+           CACHE 2 CYCLE
+) PRIMARY KEY,
+  family VARCHAR(30),
+  major CHAR(3) default 'NaN',
+  minor CHAR(3) default 'NaN',
+  os CHAR(20) NOT NULL,
+  guid CHAR(36) NOT NULL
+);
+
+CREATE FUNCTION JSON_SQUASH(name string)
+RETURNS string
+BEGIN
+  RETURN REPLACE(REPLACE(REPLACE(name, '[\"', ''), '\"]', ''), '"', '');
+END;
+
+/*
+PLAN SELECT 
+JSON_SQUASH(host) AS host, 
+JSON_SQUASH(acceptlanguage) as acceptlanguage,
+JSON_SQUASH(acceptencoding) as acceptencoding,
+JSON_SQUASH(referer) as referer,
+family,
+major,
+minor,
+os
+FROM 
+(SELECT 
+json.filter(json_data, '$.http.host') AS host,
+json.filter(json_data, '$.http.acceptlanguage') AS acceptlanguage,
+json.filter(json_data, '$.http.acceptencoding') AS acceptencoding,
+json.filter(json_data, '$.http.referer') AS referer,
+b.family,
+b.major,
+b.minor,
+b.os
+FROM http_traffic_json a JOIN http_traffic_ua b 
+ON (a.guid = b.guid)) AS origin WHERE referer ILIKE '%http://%' LIMIT 30;
+*/
+
+/*
+SELECT MIN(json_data) FROM http_traffic_json
+*/
+
+/*
+Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0
+*/

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: thm
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.4.5
 platform: ruby
 authors:
 - puppetpies
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-07 00:00:00.000000000 Z
+date: 2015-09-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -170,14 +170,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.0'
+        version: 0.0.8
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.0'
+        version: 0.0.8
+- !ruby/object:Gem::Dependency
+  name: walltime
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.5
 description: Threatmonitor - Packet Capture / Analysis Suite
 email: brianh6854@googlemail.com
 executables:
@@ -194,6 +208,7 @@ extra_rdoc_files:
 files:
 - README.1ST
 - README.md
+- Rakefile
 - bin/thm-consumer
 - bin/thm-pcap
 - bin/thm-producer