thm 0.4.5 → 0.5.7

data/lib/thm/datalayerlight.rb

@@ -7,6 +7,7 @@
 # Database Connectivity Essential to the project
 #
 ########################################################################
+require 'keycounter'
 
 module DatalayerLight
 
@@ -14,7 +15,7 @@ module DatalayerLight
     
     require 'MonetDB'
     
-    attr_writer :hostname, :username, :password, :port, :debug, :autocommit
+    attr_writer :hostname, :username, :password, :port, :debug, :autocommit, :autocommitvalue
     attr_accessor :dbname
     
     def initialize
@@ -23,8 +24,10 @@ module DatalayerLight
       @password = "monetdb"
       @port = 50000
       @dbname = "demo"
-      @debug = 1
-      @autocommit = true
+      @debug = 0
+      @autocommit = false
+      @autocommitvalue = 100
+      @k = Keycounter.new
     end
 
     def connect
@@ -36,13 +39,17 @@ module DatalayerLight
         puts "Port:     #@port"
         puts "Dbname:   #@dbname"
       end
-      @db.connect(user = "#@username", 
-          passwd = "#@password", 
-          lang = "sql", 
-          host="#@hostname", 
-          port = @port, 
-          db_name = "#@dbname", 
-          auth_type = "SHA256")
+      begin
+        @db.connect(user = "#@username", 
+            passwd = "#@password", 
+            lang = "sql", 
+            host="#@hostname", 
+            port = @port, 
+            db_name = "#@dbname", 
+            auth_type = "SHA256")
+      rescue Errno::EHOSTUNREACH
+        puts "Please check if the server is running ?"
+      end
       @db.auto_commit(@autocommit)
     end
     
@@ -62,18 +69,47 @@ module DatalayerLight
       @db.auto_commit?
     end
     
+    def rollback
+      begin
+        @db.query("ROLLBACK;")
+        puts "Rollback sucess !!!"
+      rescue MonetDBQueryError # Worse case scenario
+        puts "Something went wrong"
+        @db.free
+        @db.close
+        puts "Exiting ..."
+        puts "Bye!"
+        exit
+      end
+    end
+    
     def commit
-      @db.query("COMMIT;")
+      begin
+        @db.query("COMMIT;")
+        @k.keycount_reset("COMMITCOUNTER")
+        puts "Syncing data to disk..."
+      rescue MonetDBQueryError
+        puts "Atempting Rollback..."
+        rollback
+      end
     end
     
     def query(sql)
-      @res = @db.query("#{sql};")
-      if @debug == 1; puts "#{sql}"; end
+      begin
+        @res = @db.query("#{sql};")
+        print "COMMIT COUNTER: " if @debug == 1
+        @k.keycount("COMMITCOUNTER")
+        pp @k.keycount_reader("COMMITCOUNTER") if @debug == 1
+        commit if @k.keycount_reader("COMMITCOUNTER") == @autocommitvalue
+        if @debug == 1; puts "#{sql}"; end
+      rescue MonetDBQueryError
+        rollback
+      end
       return @res
     end
     
     def free
-    @res.free
+      @res.free
     end
     
     def close
@@ -87,7 +123,7 @@ module DatalayerLight
     
     require 'mysql'
     
-    attr_writer :hostname, :username, :password, :port, :debug, :autocommit
+    attr_writer :hostname, :username, :password, :port, :debug, :autocommit, :autocommitvalue
     attr_accessor :dbname
     
     def initialize
@@ -96,8 +132,10 @@ module DatalayerLight
       @password = ""
       @port = 3306
       @dbname = "test"
-      @debug = 1
-      @autocommit = true
+      @debug = 0
+      @autocommit = false
+      @autocommitvalue = 100
+      @k = Keycounter.new
     end
 
     def connect
@@ -120,6 +158,10 @@ module DatalayerLight
     def query(sql)
       begin
         @res = @db.query("#{sql;}")
+        print "COMMIT COUNTER: " if @debug == 1
+        @k.keycount("COMMITCOUNTER")
+        pp @k.keycount_reader("COMMITCOUNTER") if @debug == 1
+        commit if @k.keycount_reader("COMMITCOUNTER") == @autocommitvalue
         if @debug == true; puts "#{sql}"; end
         return @res
       rescue Mysql::Error => e
@@ -133,6 +175,8 @@ module DatalayerLight
     
     def commit
       @db.commit
+      @k.keycount_reset("COMMITCOUNTER")
+      puts "Syncing data to disk..."
     end
     
     def close

data/lib/thm/dataservices.rb

@@ -71,10 +71,16 @@ module Thm
       res = @conn.query("#{sql}")
     end
 
+    def commit
+      @conn.commit
+    end
+    
   end
 
 end
 
+require_relative 'dataservices/external.rb'
+require_relative 'dataservices/safebrowsing_api.rb'
 require_relative 'dataservices/trafviz/trafviz.rb'
 require_relative 'dataservices/geolocation/geolocation.rb'
 

data/lib/thm/dataservices/external.rb

@@ -0,0 +1,68 @@
+########################################################################
+#
+# Author: Brian Hood
+# Email: <brianh6854@googlemail.com>
+# Description: Libraries - External
+#
+#   For external REST / API Services
+#
+########################################################################
+
+require 'net/http'
+require 'uri'
+require 'json'
+
+module Thm
+
+  class DataServices::External
+
+    attr_writer :debug
+    
+    def initialize
+      @debug = false
+    end
+    
+    def apiget(url)
+      uri = URI.parse("#{url}")
+      puts "Request URI: #{url}" unless @debug == false
+      http = Net::HTTP.new(uri.host, uri.port)
+      if url =~ %r=^https:=
+        http.use_ssl = true
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      end      
+      begin
+        response = http.request(Net::HTTP::Get.new(uri.request_uri))
+        puts response.body unless @debug == false
+        return response 
+      rescue
+        puts "Error retrieving data"
+      end
+    end
+    
+    def apipost(url, body="")
+      uri = URI.parse("#{url}")
+      puts "Request URI: #{url}" unless @debug == false
+      http = Net::HTTP.new(uri.host, uri.port)
+      request = Net::HTTP::Post.new(uri.request_uri)
+      request.set_content_type("application/json")
+      begin
+        request.body = body unless body.empty?
+        response = http.request(request)
+      rescue
+        puts "Error posting data"
+      end
+    end
+    
+    def apidelete(url)
+      uri = URI.parse("#{url}")
+      http = Net::HTTP.new(uri.host, uri.port)
+      begin
+        response = http.request(Net::HTTP::Delete.new(uri.request_uri))
+      rescue
+        puts "Error posting data"
+      end
+    end
+
+  end
+
+end

data/lib/thm/dataservices/geolocation/geolocation.rb

@@ -31,9 +31,9 @@ module Thm
     
     def self.define_component(name)
       name_func = name.to_sym
-      define_method(name_func) do |ip|
+      define_method(name_func) do |ip, geo|
         octets = formatinet(ip, 2)
-        geoquery = "SELECT count(*) as num FROM geoipdata_ipv4blocks_#{name_func} a "
+        geoquery = "SELECT COUNT(*) as num FROM geoipdata_ipv4blocks_#{name_func} a "
         geoquery << "JOIN geoipdata_locations_#{name_func} b ON (a.geoname_id = b.geoname_id) "
         geoquery << "WHERE network LIKE '#{octets}.%';"
         begin
@@ -45,17 +45,34 @@ module Thm
         end
         puts "Geo SELECT COUNT: #{geoquery}: Number: #{geocount}" if @geodebug == true
         if geocount > 0;
-          geoquery = "SELECT continent_name, #{name_func}_name FROM geoipdata_ipv4blocks_#{name_func} a "
+          geoquery = "SELECT "
+          if geo == false
+            geoquery << "continent_name, #{name_func}_name "
+          else
+            geoquery << "latitude, longitude "
+          end
+          geoquery << "FROM geoipdata_ipv4blocks_#{name_func} a "
           geoquery << "JOIN geoipdata_locations_#{name_func} b ON (a.geoname_id = b.geoname_id) "
-          geoquery << "WHERE network LIKE '#{octets}.%' GROUP BY b.continent_name, b.#{name_func}_name LIMIT 1;"
+          geoquery << "WHERE network LIKE '#{octets}.%' GROUP BY "
+          if geo == false
+            geoquery << "b.continent_name, b.#{name_func}_name "
+          else
+            geoquery << "a.latitude, a.longitude "
+          end
+          geoquery << "LIMIT 1;"
           puts "Geo SELECT: #{geoquery}" if @geodebug == true
           begin
             resgeo = @conn.query("#{geoquery}")
             while row = resgeo.fetch_hash do
-              populategeo = instance_variable_get("@#{name_func}_name")
-              populategeo << row["#{name_func}_name"].to_s
-              instance_variable_set("@#{name_func}_name", populategeo) # Only returns 1 row
-              @continent_name = row["continent_name"].to_s
+              if geo == false
+                populategeo = instance_variable_get("@#{name_func}_name")
+                populategeo << row["#{name_func}_name"].to_s
+                instance_variable_set("@#{name_func}_name", populategeo) # Only returns 1 row
+                @continent_name = row["continent_name"].to_s
+              else
+                instance_variable_set("@latitude", row["latitude"].to_s)
+                instance_variable_set("@longitude", row["longitude"].to_s)
+              end
             end
           rescue => e
             pp e
@@ -69,18 +86,24 @@ module Thm
     define_component :country
     define_component :city
 
-    def geoiplookup(ip)
-      t = country(ip)
-      city(ip)
+    # Geo set to false by default for normal operation
+    def geoiplookup(ip, geo=false)
+      t = country(ip, geo)
+      city(ip, geo)
       unless t == false
-        res = "(#{@continent_name}) - \n"
-        @country_name.each {|n|
-          res << "[ #{n} ]"
-        }
-        @city_name.each {|n|
-          res << "[ #{n} ] "
-        }
-        initialize
+        # Check if @longitude / @latitude exists for Reverse Geocoding options
+        if instance_variable_defined?("@latitude") and instance_variable_defined?("@longitude")
+          res = [@latitude, @longitude]
+        else
+          res = "(#{@continent_name}) - \n"
+          @country_name.each {|n|
+            res << "[ #{n} ]" unless n == ""
+          }
+          @city_name.each {|n|
+            res << "[ #{n} ] " unless n == ""
+          }
+          initialize
+        end
       else
         res = "Not Available"
       end
@@ -89,4 +112,8 @@ module Thm
 
   end
 
+  class DataServices::Geocoding < DataServices
+  
+  end
+  
 end

data/lib/thm/dataservices/safebrowsing_api.rb

@@ -0,0 +1,54 @@
+########################################################################
+#
+# Author: Brian Hood
+# Email: <brianh6854@googlemail.com>
+# Description: Libraries - External
+#
+#   For Google Safebrowsing API
+#
+########################################################################
+
+module Thm
+  
+  class DataServices::Safebrowsing < DataServices::External
+    
+    attr_writer :debug
+
+    def initialize
+      super
+    end
+    
+    def handle_response?(code)
+      if code == "200"
+        return code
+      elsif code == "204"
+        return code
+      elsif code == "400"
+        puts "Bad Request—The HTTP request was not correctly formed. The client did not provide all required CGI parameters."
+        return false
+      elsif code == "401"
+        puts "Not Authorized—The client id is invalid."
+        return false      
+      elsif code == "503"
+        puts "Service Unavailable"
+        return false
+      elsif code == "505"
+        puts "HTTP Version Not Supported—The server CANNOT handle the requested protocol major version."
+        return false
+      end    
+    end
+  
+    def lookup(url)
+      response = apiget(url)
+      print "Response: "
+      pp response
+      if handle_response?(response.code) =~ %r=^200|^204=
+        return [response.code, response.body]
+      else
+        return false
+      end
+    end
+
+  end
+
+end

data/lib/thm/dataservices/trafviz/trafviz.rb

@@ -31,29 +31,31 @@ module Thm
     
   class DataServices::Trafviz
     
-    attr_writer :reqtable, :reqtableua
-    
+    attr_writer :reqtable, :reqtableua, :debug
+    attr_reader :makeurl_last
+
     # For refinement of print_stats 
     using TimeWarp
     
     def initialize
       @debug = false
       @reqtable, @reqtableua = String.new, String.new
+      @makeurl_last = String.new
     end
     
     def makeurl(data)
       if !request_valid?(data)
         return false
-      end
-      hdrs = data
+      end 
       hostn, requestn = ""
-      hdrs.each_line {|n|
+      data.each_line {|n|
         if n.split(":")[0] == "Host"
           hostn = n.split(":")[1].strip
-        elsif n.split(" ")[0] == "GET"
+        elsif n.split(" ")[0] =~ /^GET|^HEAD/
           requestn = n.split(" ")[1]
         end
       }
+      @makeurl_last = "http://#{hostn}#{requestn}"
       puts "\e[1;37mURL: http://#{hostn}#{requestn} \e[0m\ "
     end
 
@@ -73,24 +75,25 @@ module Thm
     end
     
     # This is just an informal function when in debug mode
-    def hit_header(hdrs, comment="")
-      puts "Hit #{hdrs} header #{comment}"
+    def catch_header(hdrs, comment="")
+      print "Caught: #{hdrs} "
+      puts "Header comment: #{comment}" unless comment == ""
     end
     
-    
     # Cookie ommit as we don't want to steal cookie data and pointless to store.
     # Other useless headers / slight issues
+    # You can now add a comment to catch_header if you like
     def filter_header?(lkey)
       puts "MY LKEY: |#{lkey}|" if @debug == true
       case 
       when lkey == "cookie"
-        hit_header(lkey) if @debug == true
+        catch_header(lkey) if @debug == true
         return true
       when lkey == "range"
-        hit_header(lkey) if @debug == true
+        catch_header(lkey) if @debug == true
         return true
-      when lkey =~ /^get |^post /
-        hit_heaer(lkey, "Seen this unsure why it even occurs yet !") if @debug == true
+      when lkey =~ %r=^get |^post |^head =
+        catch_header(lkey, "Seen this unsure why it even occurs yet !") if @debug == true
         return true
       else
         return false
@@ -113,12 +116,12 @@ module Thm
     
     # Filter request data and build query
     def request_filter(data, keysamples=2000)
+      flt = Stopwatch.new
+      flt.watch('start')
       if !request_valid?(data)
         sql = "SELECT 1;"
         return sql
       end
-      flt = Stopwatch.new
-      flt.watch('start')
       guid = Tools::guid
       cols, vals = String.new, String.new
       lkey, rkey = String.new, String.new
@@ -152,14 +155,15 @@ module Thm
               prerkeyins = rkey.gsub('"', '') # Strip Quotes
               prerkeyins = "blank" if prerkeyins.strip == "" # Seems JSON values can't be "accept":""
               puts "Found Blank Value!!!" if prerkeyins == "blank"
-              if lkey != "useragent"
-                json_data_pieces << "'#{lkey}' => \"#{prerkeyins}\",\n"
-              end
+              json_data_pieces << "'#{lkey}' => \"#{prerkeyins}\",\n" if lkey != "useragent"
             end
           end
           t += 1
         end
       }
+      # Store the URL in the JSON unless its blank
+      # Build JSON Manually as i bet its faster than using some JSON encoder where it has to convert from Array etc.
+      json_data_pieces << "'url' => \"#{@makeurl_last}\",\n" unless @makeurl_last == ""
       # SQL for Datastore
       begin
         # Remove last , to fix hash table
@@ -180,18 +184,151 @@ module Thm
       end
     end
     
-    def text_highlighter(text)
-      keys = ["Linux", "Java", "Android", "iPhone", "Mobile", "Chrome", 
-              "Safari", "Mozilla", "Gecko", "AppleWebKit", "Windows", 
-              "MSIE", "Win64", "Trident", "wispr", "PHPSESSID", "JSESSIONID",
-              "AMD64", "Darwin", "Macintosh", "Mac OS X", "Dalvik", "text/html", "xml"]
-      cpicker = [2,3,4,1,7,5,6] # Just a selection of colours
-      keys.each {|n|
-        text.gsub!("#{n}", "\e[4;3#{cpicker[rand(cpicker.size)]}m#{n}\e[0m\ \e[0;32m".strip)
+    include TextProcessing
+
+  end
+
+end
+
+module Thm
+
+  class DataServices::Trafviz::FilterManager
+
+    attr_reader :bookmarks, :pcapsetfilter
+    
+    def initialize
+      @bookmarks = Array.new
+      @bkm = MyMenu.new
+      @bkm.settitle("Welcome to Trafviz")
+      @bkm.mymenuname = "Trafviz"
+      @bkm.prompt = "Trafviz"
+      @pcapsetfilter = String.new
+    end
+
+    def read(file)
+      b = 0
+      File.open("#{Dir.home}/.thm/#{file}", 'r') {|n|
+        n.each_line {|l|
+          puts "\e[1;36m#{b})\e[0m\ #{l}"
+          @bookmarks[b] = l
+          b += 1
+        }
+      }
+    end
+
+    def write(file)
+      @bkm.mymenuname = "Filters"
+      @bkm.prompt = "\e[1;33m\Set filter>\e[0m\ "
+      pcapfilter = @bkm.definemenuitem("selectfilter", true) do
+        # Just needs value returned via readline block into addfilter
+      end
+      fltvalid = validate_filter?("#{pcapfilter}")
+      if fltvalid == true
+        File.open("#{Dir.home}/.thm/#{file}", 'a') {|n| # Append to filter file
+          n.puts("#{addfilter}")
+        }
+      end
+    end
+    
+    def set_defaults(file)
+      # Add default example filters
+      File.open("#{Dir.home}/.thm/#{file}", 'w') {|n|
+        n.puts("webtraffic: tcp dst port 80")
+        n.puts("sourceportrange: tcp src portrange 1024-65535")
+      }
+    end
+
+    def validate_filter?(filter)
+      begin
+        Pcap::Filter.compile("#{filter}")
+        puts "Filter Compile #{filter}"
+        return true
+      rescue Pcap::PcapError => e
+        pp e
+        return false
+      end
+    end
+    
+    def build_filter_menu
+      @bkm.settitle("Welcome to Trafviz")
+      @bkm.mymenuname = "Trafviz"
+      @bkm.prompt = "Trafviz"
+      @bkm.debug = 3
+      pp @bookmarks
+      @bookmarks.each {|n|
+        func_name = n.split(":")[0]
+        pcap_filter = n.split(":")[1].lstrip
+        puts "#{pcap_filter}"
+        # Instance Eval probably nicer
+        fltvalid = validate_filter?("#{pcap_filter}") # Because validate_filter? won't exist inside instance_eval
+        @bkm.instance_eval do
+          pp fltvalid
+          if fltvalid == true
+            definemenuitem("#{func_name}") do
+              @pcapsetfilter = "#{pcap_filter}"
+              #thm = DataServices::Trafviz::Main.new
+            end
+            additemtolist("#{func_name}: #{pcap_filter}", "#{func_name};")
+          end
+        end
       }
-      return text
+      @bkm.instance_eval do
+        definemenuitem("showfilter") do
+          puts "Filter: #{@pcapsetfilter}"
+        end
+        additemtolist("Show Current Filter", "showfilter;")
+      end
+      @bkm.additemtolist("Display Menu", "showmenu;")
+      @bkm.additemtolist("Toggle Menu", "togglemenu;")
+      @bkm.additemtolist("Exit Trafviz", "exit;")
+      @bkm.menu!
+    end
+        
+    def load_filters(file)
+      if File.exists?("#{Dir.home}/.thm/#{file}")
+        read(file)
+      else
+        set_defaults(file)
+        read(file)
+      end
+      build_filter_menu
+    end
+    
+  end
+  
+end
+
+# Main class / Startup
+
+module Thm
+
+  class DataServices::Trafviz::Main
+
+    attr_accessor :startup
+    
+    def initialize
+      @filter_const = Array.new
+      @startup = String.new
+      @thm = Thm::DataServices::Trafviz::FilterManager.new
     end
 
+    def addfilter(const, filter)
+      if @thm.validate_filter?(filter) == true
+        filtercode = %Q{#{const} = Pcap::Filter.new('#{filter}', @trafviz.capture)}
+        @filter_const << "#{const})"
+        eval(filtercode)
+      end
+    end
+    
+    def commitfilters
+      flts = @filter_const.join(" | ") # Build string of CONST names
+      commitcode = %Q{@trafviz.add_filter(#{flts})}
+      eval(flts)
+    end
+    
+    def run!
+      @trafviz = Pcaplet.new(@startup)
+    end
   end
 
 end