thm 0.4.5 → 0.5.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -7,6 +7,7 @@
7
7
  # Database Connectivity Essential to the project
8
8
  #
9
9
  ########################################################################
10
+ require 'keycounter'
10
11
 
11
12
  module DatalayerLight
12
13
 
@@ -14,7 +15,7 @@ module DatalayerLight
14
15
 
15
16
  require 'MonetDB'
16
17
 
17
- attr_writer :hostname, :username, :password, :port, :debug, :autocommit
18
+ attr_writer :hostname, :username, :password, :port, :debug, :autocommit, :autocommitvalue
18
19
  attr_accessor :dbname
19
20
 
20
21
  def initialize
@@ -23,8 +24,10 @@ module DatalayerLight
23
24
  @password = "monetdb"
24
25
  @port = 50000
25
26
  @dbname = "demo"
26
- @debug = 1
27
- @autocommit = true
27
+ @debug = 0
28
+ @autocommit = false
29
+ @autocommitvalue = 100
30
+ @k = Keycounter.new
28
31
  end
29
32
 
30
33
  def connect
@@ -36,13 +39,17 @@ module DatalayerLight
36
39
  puts "Port: #@port"
37
40
  puts "Dbname: #@dbname"
38
41
  end
39
- @db.connect(user = "#@username",
40
- passwd = "#@password",
41
- lang = "sql",
42
- host="#@hostname",
43
- port = @port,
44
- db_name = "#@dbname",
45
- auth_type = "SHA256")
42
+ begin
43
+ @db.connect(user = "#@username",
44
+ passwd = "#@password",
45
+ lang = "sql",
46
+ host="#@hostname",
47
+ port = @port,
48
+ db_name = "#@dbname",
49
+ auth_type = "SHA256")
50
+ rescue Errno::EHOSTUNREACH
51
+ puts "Please check if the server is running ?"
52
+ end
46
53
  @db.auto_commit(@autocommit)
47
54
  end
48
55
 
@@ -62,18 +69,47 @@ module DatalayerLight
62
69
  @db.auto_commit?
63
70
  end
64
71
 
72
+ def rollback
73
+ begin
74
+ @db.query("ROLLBACK;")
75
+ puts "Rollback sucess !!!"
76
+ rescue MonetDBQueryError # Worse case scenario
77
+ puts "Something went wrong"
78
+ @db.free
79
+ @db.close
80
+ puts "Exiting ..."
81
+ puts "Bye!"
82
+ exit
83
+ end
84
+ end
85
+
65
86
  def commit
66
- @db.query("COMMIT;")
87
+ begin
88
+ @db.query("COMMIT;")
89
+ @k.keycount_reset("COMMITCOUNTER")
90
+ puts "Syncing data to disk..."
91
+ rescue MonetDBQueryError
92
+ puts "Atempting Rollback..."
93
+ rollback
94
+ end
67
95
  end
68
96
 
69
97
  def query(sql)
70
- @res = @db.query("#{sql};")
71
- if @debug == 1; puts "#{sql}"; end
98
+ begin
99
+ @res = @db.query("#{sql};")
100
+ print "COMMIT COUNTER: " if @debug == 1
101
+ @k.keycount("COMMITCOUNTER")
102
+ pp @k.keycount_reader("COMMITCOUNTER") if @debug == 1
103
+ commit if @k.keycount_reader("COMMITCOUNTER") == @autocommitvalue
104
+ if @debug == 1; puts "#{sql}"; end
105
+ rescue MonetDBQueryError
106
+ rollback
107
+ end
72
108
  return @res
73
109
  end
74
110
 
75
111
  def free
76
- @res.free
112
+ @res.free
77
113
  end
78
114
 
79
115
  def close
@@ -87,7 +123,7 @@ module DatalayerLight
87
123
 
88
124
  require 'mysql'
89
125
 
90
- attr_writer :hostname, :username, :password, :port, :debug, :autocommit
126
+ attr_writer :hostname, :username, :password, :port, :debug, :autocommit, :autocommitvalue
91
127
  attr_accessor :dbname
92
128
 
93
129
  def initialize
@@ -96,8 +132,10 @@ module DatalayerLight
96
132
  @password = ""
97
133
  @port = 3306
98
134
  @dbname = "test"
99
- @debug = 1
100
- @autocommit = true
135
+ @debug = 0
136
+ @autocommit = false
137
+ @autocommitvalue = 100
138
+ @k = Keycounter.new
101
139
  end
102
140
 
103
141
  def connect
@@ -120,6 +158,10 @@ module DatalayerLight
120
158
  def query(sql)
121
159
  begin
122
160
  @res = @db.query("#{sql;}")
161
+ print "COMMIT COUNTER: " if @debug == 1
162
+ @k.keycount("COMMITCOUNTER")
163
+ pp @k.keycount_reader("COMMITCOUNTER") if @debug == 1
164
+ commit if @k.keycount_reader("COMMITCOUNTER") == @autocommitvalue
123
165
  if @debug == true; puts "#{sql}"; end
124
166
  return @res
125
167
  rescue Mysql::Error => e
@@ -133,6 +175,8 @@ module DatalayerLight
133
175
 
134
176
  def commit
135
177
  @db.commit
178
+ @k.keycount_reset("COMMITCOUNTER")
179
+ puts "Syncing data to disk..."
136
180
  end
137
181
 
138
182
  def close
@@ -71,10 +71,16 @@ module Thm
71
71
  res = @conn.query("#{sql}")
72
72
  end
73
73
 
74
+ def commit
75
+ @conn.commit
76
+ end
77
+
74
78
  end
75
79
 
76
80
  end
77
81
 
82
+ require_relative 'dataservices/external.rb'
83
+ require_relative 'dataservices/safebrowsing_api.rb'
78
84
  require_relative 'dataservices/trafviz/trafviz.rb'
79
85
  require_relative 'dataservices/geolocation/geolocation.rb'
80
86
 
@@ -0,0 +1,68 @@
1
+ ########################################################################
2
+ #
3
+ # Author: Brian Hood
4
+ # Email: <brianh6854@googlemail.com>
5
+ # Description: Libraries - External
6
+ #
7
+ # For external REST / API Services
8
+ #
9
+ ########################################################################
10
+
11
+ require 'net/http'
12
+ require 'uri'
13
+ require 'json'
14
+
15
+ module Thm
16
+
17
+ class DataServices::External
18
+
19
+ attr_writer :debug
20
+
21
+ def initialize
22
+ @debug = false
23
+ end
24
+
25
+ def apiget(url)
26
+ uri = URI.parse("#{url}")
27
+ puts "Request URI: #{url}" unless @debug == false
28
+ http = Net::HTTP.new(uri.host, uri.port)
29
+ if url =~ %r=^https:=
30
+ http.use_ssl = true
31
+ http.verify_mode = OpenSSL::SSL::VERIFY_NONE
32
+ end
33
+ begin
34
+ response = http.request(Net::HTTP::Get.new(uri.request_uri))
35
+ puts response.body unless @debug == false
36
+ return response
37
+ rescue
38
+ puts "Error retrieving data"
39
+ end
40
+ end
41
+
42
+ def apipost(url, body="")
43
+ uri = URI.parse("#{url}")
44
+ puts "Request URI: #{url}" unless @debug == false
45
+ http = Net::HTTP.new(uri.host, uri.port)
46
+ request = Net::HTTP::Post.new(uri.request_uri)
47
+ request.set_content_type("application/json")
48
+ begin
49
+ request.body = body unless body.empty?
50
+ response = http.request(request)
51
+ rescue
52
+ puts "Error posting data"
53
+ end
54
+ end
55
+
56
+ def apidelete(url)
57
+ uri = URI.parse("#{url}")
58
+ http = Net::HTTP.new(uri.host, uri.port)
59
+ begin
60
+ response = http.request(Net::HTTP::Delete.new(uri.request_uri))
61
+ rescue
62
+ puts "Error posting data"
63
+ end
64
+ end
65
+
66
+ end
67
+
68
+ end
@@ -31,9 +31,9 @@ module Thm
31
31
 
32
32
  def self.define_component(name)
33
33
  name_func = name.to_sym
34
- define_method(name_func) do |ip|
34
+ define_method(name_func) do |ip, geo|
35
35
  octets = formatinet(ip, 2)
36
- geoquery = "SELECT count(*) as num FROM geoipdata_ipv4blocks_#{name_func} a "
36
+ geoquery = "SELECT COUNT(*) as num FROM geoipdata_ipv4blocks_#{name_func} a "
37
37
  geoquery << "JOIN geoipdata_locations_#{name_func} b ON (a.geoname_id = b.geoname_id) "
38
38
  geoquery << "WHERE network LIKE '#{octets}.%';"
39
39
  begin
@@ -45,17 +45,34 @@ module Thm
45
45
  end
46
46
  puts "Geo SELECT COUNT: #{geoquery}: Number: #{geocount}" if @geodebug == true
47
47
  if geocount > 0;
48
- geoquery = "SELECT continent_name, #{name_func}_name FROM geoipdata_ipv4blocks_#{name_func} a "
48
+ geoquery = "SELECT "
49
+ if geo == false
50
+ geoquery << "continent_name, #{name_func}_name "
51
+ else
52
+ geoquery << "latitude, longitude "
53
+ end
54
+ geoquery << "FROM geoipdata_ipv4blocks_#{name_func} a "
49
55
  geoquery << "JOIN geoipdata_locations_#{name_func} b ON (a.geoname_id = b.geoname_id) "
50
- geoquery << "WHERE network LIKE '#{octets}.%' GROUP BY b.continent_name, b.#{name_func}_name LIMIT 1;"
56
+ geoquery << "WHERE network LIKE '#{octets}.%' GROUP BY "
57
+ if geo == false
58
+ geoquery << "b.continent_name, b.#{name_func}_name "
59
+ else
60
+ geoquery << "a.latitude, a.longitude "
61
+ end
62
+ geoquery << "LIMIT 1;"
51
63
  puts "Geo SELECT: #{geoquery}" if @geodebug == true
52
64
  begin
53
65
  resgeo = @conn.query("#{geoquery}")
54
66
  while row = resgeo.fetch_hash do
55
- populategeo = instance_variable_get("@#{name_func}_name")
56
- populategeo << row["#{name_func}_name"].to_s
57
- instance_variable_set("@#{name_func}_name", populategeo) # Only returns 1 row
58
- @continent_name = row["continent_name"].to_s
67
+ if geo == false
68
+ populategeo = instance_variable_get("@#{name_func}_name")
69
+ populategeo << row["#{name_func}_name"].to_s
70
+ instance_variable_set("@#{name_func}_name", populategeo) # Only returns 1 row
71
+ @continent_name = row["continent_name"].to_s
72
+ else
73
+ instance_variable_set("@latitude", row["latitude"].to_s)
74
+ instance_variable_set("@longitude", row["longitude"].to_s)
75
+ end
59
76
  end
60
77
  rescue => e
61
78
  pp e
@@ -69,18 +86,24 @@ module Thm
69
86
  define_component :country
70
87
  define_component :city
71
88
 
72
- def geoiplookup(ip)
73
- t = country(ip)
74
- city(ip)
89
+ # Geo set to false by default for normal operation
90
+ def geoiplookup(ip, geo=false)
91
+ t = country(ip, geo)
92
+ city(ip, geo)
75
93
  unless t == false
76
- res = "(#{@continent_name}) - \n"
77
- @country_name.each {|n|
78
- res << "[ #{n} ]"
79
- }
80
- @city_name.each {|n|
81
- res << "[ #{n} ] "
82
- }
83
- initialize
94
+ # Check if @longitude / @latitude exists for Reverse Geocoding options
95
+ if instance_variable_defined?("@latitude") and instance_variable_defined?("@longitude")
96
+ res = [@latitude, @longitude]
97
+ else
98
+ res = "(#{@continent_name}) - \n"
99
+ @country_name.each {|n|
100
+ res << "[ #{n} ]" unless n == ""
101
+ }
102
+ @city_name.each {|n|
103
+ res << "[ #{n} ] " unless n == ""
104
+ }
105
+ initialize
106
+ end
84
107
  else
85
108
  res = "Not Available"
86
109
  end
@@ -89,4 +112,8 @@ module Thm
89
112
 
90
113
  end
91
114
 
115
+ class DataServices::Geocoding < DataServices
116
+
117
+ end
118
+
92
119
  end
@@ -0,0 +1,54 @@
1
+ ########################################################################
2
+ #
3
+ # Author: Brian Hood
4
+ # Email: <brianh6854@googlemail.com>
5
+ # Description: Libraries - External
6
+ #
7
+ # For Google Safebrowsing API
8
+ #
9
+ ########################################################################
10
+
11
+ module Thm
12
+
13
+ class DataServices::Safebrowsing < DataServices::External
14
+
15
+ attr_writer :debug
16
+
17
+ def initialize
18
+ super
19
+ end
20
+
21
+ def handle_response?(code)
22
+ if code == "200"
23
+ return code
24
+ elsif code == "204"
25
+ return code
26
+ elsif code == "400"
27
+ puts "Bad Request—The HTTP request was not correctly formed. The client did not provide all required CGI parameters."
28
+ return false
29
+ elsif code == "401"
30
+ puts "Not Authorized—The client id is invalid."
31
+ return false
32
+ elsif code == "503"
33
+ puts "Service Unavailable"
34
+ return false
35
+ elsif code == "505"
36
+ puts "HTTP Version Not Supported—The server CANNOT handle the requested protocol major version."
37
+ return false
38
+ end
39
+ end
40
+
41
+ def lookup(url)
42
+ response = apiget(url)
43
+ print "Response: "
44
+ pp response
45
+ if handle_response?(response.code) =~ %r=^200|^204=
46
+ return [response.code, response.body]
47
+ else
48
+ return false
49
+ end
50
+ end
51
+
52
+ end
53
+
54
+ end
@@ -31,29 +31,31 @@ module Thm
31
31
 
32
32
  class DataServices::Trafviz
33
33
 
34
- attr_writer :reqtable, :reqtableua
35
-
34
+ attr_writer :reqtable, :reqtableua, :debug
35
+ attr_reader :makeurl_last
36
+
36
37
  # For refinement of print_stats
37
38
  using TimeWarp
38
39
 
39
40
  def initialize
40
41
  @debug = false
41
42
  @reqtable, @reqtableua = String.new, String.new
43
+ @makeurl_last = String.new
42
44
  end
43
45
 
44
46
  def makeurl(data)
45
47
  if !request_valid?(data)
46
48
  return false
47
- end
48
- hdrs = data
49
+ end
49
50
  hostn, requestn = ""
50
- hdrs.each_line {|n|
51
+ data.each_line {|n|
51
52
  if n.split(":")[0] == "Host"
52
53
  hostn = n.split(":")[1].strip
53
- elsif n.split(" ")[0] == "GET"
54
+ elsif n.split(" ")[0] =~ /^GET|^HEAD/
54
55
  requestn = n.split(" ")[1]
55
56
  end
56
57
  }
58
+ @makeurl_last = "http://#{hostn}#{requestn}"
57
59
  puts "\e[1;37mURL: http://#{hostn}#{requestn} \e[0m\ "
58
60
  end
59
61
 
@@ -73,24 +75,25 @@ module Thm
73
75
  end
74
76
 
75
77
  # This is just an informal function when in debug mode
76
- def hit_header(hdrs, comment="")
77
- puts "Hit #{hdrs} header #{comment}"
78
+ def catch_header(hdrs, comment="")
79
+ print "Caught: #{hdrs} "
80
+ puts "Header comment: #{comment}" unless comment == ""
78
81
  end
79
82
 
80
-
81
83
  # Cookie ommit as we don't want to steal cookie data and pointless to store.
82
84
  # Other useless headers / slight issues
85
+ # You can now add a comment to catch_header if you like
83
86
  def filter_header?(lkey)
84
87
  puts "MY LKEY: |#{lkey}|" if @debug == true
85
88
  case
86
89
  when lkey == "cookie"
87
- hit_header(lkey) if @debug == true
90
+ catch_header(lkey) if @debug == true
88
91
  return true
89
92
  when lkey == "range"
90
- hit_header(lkey) if @debug == true
93
+ catch_header(lkey) if @debug == true
91
94
  return true
92
- when lkey =~ /^get |^post /
93
- hit_heaer(lkey, "Seen this unsure why it even occurs yet !") if @debug == true
95
+ when lkey =~ %r=^get |^post |^head =
96
+ catch_header(lkey, "Seen this unsure why it even occurs yet !") if @debug == true
94
97
  return true
95
98
  else
96
99
  return false
@@ -113,12 +116,12 @@ module Thm
113
116
 
114
117
  # Filter request data and build query
115
118
  def request_filter(data, keysamples=2000)
119
+ flt = Stopwatch.new
120
+ flt.watch('start')
116
121
  if !request_valid?(data)
117
122
  sql = "SELECT 1;"
118
123
  return sql
119
124
  end
120
- flt = Stopwatch.new
121
- flt.watch('start')
122
125
  guid = Tools::guid
123
126
  cols, vals = String.new, String.new
124
127
  lkey, rkey = String.new, String.new
@@ -152,14 +155,15 @@ module Thm
152
155
  prerkeyins = rkey.gsub('"', '') # Strip Quotes
153
156
  prerkeyins = "blank" if prerkeyins.strip == "" # Seems JSON values can't be "accept":""
154
157
  puts "Found Blank Value!!!" if prerkeyins == "blank"
155
- if lkey != "useragent"
156
- json_data_pieces << "'#{lkey}' => \"#{prerkeyins}\",\n"
157
- end
158
+ json_data_pieces << "'#{lkey}' => \"#{prerkeyins}\",\n" if lkey != "useragent"
158
159
  end
159
160
  end
160
161
  t += 1
161
162
  end
162
163
  }
164
+ # Store the URL in the JSON unless its blank
165
+ # Build JSON Manually as i bet its faster than using some JSON encoder where it has to convert from Array etc.
166
+ json_data_pieces << "'url' => \"#{@makeurl_last}\",\n" unless @makeurl_last == ""
163
167
  # SQL for Datastore
164
168
  begin
165
169
  # Remove last , to fix hash table
@@ -180,18 +184,151 @@ module Thm
180
184
  end
181
185
  end
182
186
 
183
- def text_highlighter(text)
184
- keys = ["Linux", "Java", "Android", "iPhone", "Mobile", "Chrome",
185
- "Safari", "Mozilla", "Gecko", "AppleWebKit", "Windows",
186
- "MSIE", "Win64", "Trident", "wispr", "PHPSESSID", "JSESSIONID",
187
- "AMD64", "Darwin", "Macintosh", "Mac OS X", "Dalvik", "text/html", "xml"]
188
- cpicker = [2,3,4,1,7,5,6] # Just a selection of colours
189
- keys.each {|n|
190
- text.gsub!("#{n}", "\e[4;3#{cpicker[rand(cpicker.size)]}m#{n}\e[0m\ \e[0;32m".strip)
187
+ include TextProcessing
188
+
189
+ end
190
+
191
+ end
192
+
193
+ module Thm
194
+
195
+ class DataServices::Trafviz::FilterManager
196
+
197
+ attr_reader :bookmarks, :pcapsetfilter
198
+
199
+ def initialize
200
+ @bookmarks = Array.new
201
+ @bkm = MyMenu.new
202
+ @bkm.settitle("Welcome to Trafviz")
203
+ @bkm.mymenuname = "Trafviz"
204
+ @bkm.prompt = "Trafviz"
205
+ @pcapsetfilter = String.new
206
+ end
207
+
208
+ def read(file)
209
+ b = 0
210
+ File.open("#{Dir.home}/.thm/#{file}", 'r') {|n|
211
+ n.each_line {|l|
212
+ puts "\e[1;36m#{b})\e[0m\ #{l}"
213
+ @bookmarks[b] = l
214
+ b += 1
215
+ }
216
+ }
217
+ end
218
+
219
+ def write(file)
220
+ @bkm.mymenuname = "Filters"
221
+ @bkm.prompt = "\e[1;33m\Set filter>\e[0m\ "
222
+ pcapfilter = @bkm.definemenuitem("selectfilter", true) do
223
+ # Just needs value returned via readline block into addfilter
224
+ end
225
+ fltvalid = validate_filter?("#{pcapfilter}")
226
+ if fltvalid == true
227
+ File.open("#{Dir.home}/.thm/#{file}", 'a') {|n| # Append to filter file
228
+ n.puts("#{addfilter}")
229
+ }
230
+ end
231
+ end
232
+
233
+ def set_defaults(file)
234
+ # Add default example filters
235
+ File.open("#{Dir.home}/.thm/#{file}", 'w') {|n|
236
+ n.puts("webtraffic: tcp dst port 80")
237
+ n.puts("sourceportrange: tcp src portrange 1024-65535")
238
+ }
239
+ end
240
+
241
+ def validate_filter?(filter)
242
+ begin
243
+ Pcap::Filter.compile("#{filter}")
244
+ puts "Filter Compile #{filter}"
245
+ return true
246
+ rescue Pcap::PcapError => e
247
+ pp e
248
+ return false
249
+ end
250
+ end
251
+
252
+ def build_filter_menu
253
+ @bkm.settitle("Welcome to Trafviz")
254
+ @bkm.mymenuname = "Trafviz"
255
+ @bkm.prompt = "Trafviz"
256
+ @bkm.debug = 3
257
+ pp @bookmarks
258
+ @bookmarks.each {|n|
259
+ func_name = n.split(":")[0]
260
+ pcap_filter = n.split(":")[1].lstrip
261
+ puts "#{pcap_filter}"
262
+ # Instance Eval probably nicer
263
+ fltvalid = validate_filter?("#{pcap_filter}") # Because validate_filter? won't exist inside instance_eval
264
+ @bkm.instance_eval do
265
+ pp fltvalid
266
+ if fltvalid == true
267
+ definemenuitem("#{func_name}") do
268
+ @pcapsetfilter = "#{pcap_filter}"
269
+ #thm = DataServices::Trafviz::Main.new
270
+ end
271
+ additemtolist("#{func_name}: #{pcap_filter}", "#{func_name};")
272
+ end
273
+ end
191
274
  }
192
- return text
275
+ @bkm.instance_eval do
276
+ definemenuitem("showfilter") do
277
+ puts "Filter: #{@pcapsetfilter}"
278
+ end
279
+ additemtolist("Show Current Filter", "showfilter;")
280
+ end
281
+ @bkm.additemtolist("Display Menu", "showmenu;")
282
+ @bkm.additemtolist("Toggle Menu", "togglemenu;")
283
+ @bkm.additemtolist("Exit Trafviz", "exit;")
284
+ @bkm.menu!
285
+ end
286
+
287
+ def load_filters(file)
288
+ if File.exists?("#{Dir.home}/.thm/#{file}")
289
+ read(file)
290
+ else
291
+ set_defaults(file)
292
+ read(file)
293
+ end
294
+ build_filter_menu
295
+ end
296
+
297
+ end
298
+
299
+ end
300
+
301
+ # Main class / Startup
302
+
303
+ module Thm
304
+
305
+ class DataServices::Trafviz::Main
306
+
307
+ attr_accessor :startup
308
+
309
+ def initialize
310
+ @filter_const = Array.new
311
+ @startup = String.new
312
+ @thm = Thm::DataServices::Trafviz::FilterManager.new
193
313
  end
194
314
 
315
+ def addfilter(const, filter)
316
+ if @thm.validate_filter?(filter) == true
317
+ filtercode = %Q{#{const} = Pcap::Filter.new('#{filter}', @trafviz.capture)}
318
+ @filter_const << "#{const})"
319
+ eval(filtercode)
320
+ end
321
+ end
322
+
323
+ def commitfilters
324
+ flts = @filter_const.join(" | ") # Build string of CONST names
325
+ commitcode = %Q{@trafviz.add_filter(#{flts})}
326
+ eval(flts)
327
+ end
328
+
329
+ def run!
330
+ @trafviz = Pcaplet.new(@startup)
331
+ end
195
332
  end
196
333
 
197
334
  end