thm 0.2.2 → 0.3.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.1ST +4 -4
- data/README.md +14 -2
- data/bin/thm-consumer +1 -1
- data/bin/thm-pcap +1 -1
- data/bin/thm-producer +1 -1
- data/bin/thm-session +12 -12
- data/bin/thm-trafviz +316 -0
- data/bin/thm-useradmin +14 -2
- data/config.rb +25 -15
- data/lib/thm/consumer.rb +9 -0
- data/lib/thm/datalayerlight.rb +7 -4
- data/lib/thm/dataservices/geolocation/geolocation.rb +92 -0
- data/lib/thm/dataservices/trafviz/trafviz.rb +152 -0
- data/lib/thm/dataservices.rb +9 -2
- data/lib/thm/fileservices.rb +15 -6
- data/lib/thm/localmachine.rb +9 -0
- data/lib/thm/producer.rb +10 -0
- data/lib/thm/version.rb +3 -3
- data/lib/thm.rb +46 -17
- data/sql/geoipdata-monetdb.sql +34 -34
- data/sql/threatmonitor-http.sql +15 -0
- data/thm-authentication.rb +3 -2
- data/thm-authorization.rb +2 -2
- data/thm-privileges.rb +1 -1
- metadata +21 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 68cf9e24de9c37c3ddfc874510040610213e6e5e
|
4
|
+
data.tar.gz: da51e597440bdb58e9360da1e9e20738919b98a0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 009f8996404e193f786e030e08c777c9f2278c61442789a2080f331724fa1bcffaccbd96e2e089fbc53ceb5eff39314ae46760c9d022ee7bd0a168ef46dce6fd
|
7
|
+
data.tar.gz: 020a8de2955f539ef44170037594de9232b8df897caeec8b2fa0f7203584bea5eac680d6f220d69287576c96a7ae2a7c20d6efc3f49e1529458958de7ef4d717
|
data/README.1ST
CHANGED
@@ -1,5 +1,5 @@
|
|
1
|
-
Threatmonitor Packet Analysis Suite HOWTO
|
2
|
-
|
1
|
+
Threatmonitor Packet Analysis Suite - Docker Image HOWTO
|
2
|
+
========================================================
|
3
3
|
|
4
4
|
To import example data.
|
5
5
|
|
@@ -11,7 +11,7 @@ Will also need to import the GeoIP Data.
|
|
11
11
|
|
12
12
|
The web interface admin login is admin - default password goblin
|
13
13
|
|
14
|
-
You will also need the GeoIP Data importing as well for the Web interface Dashboard
|
14
|
+
You will also need the GeoIP Data importing as well for the Web interface Dashboard to operate
|
15
15
|
|
16
16
|
mclient -u monetdb -d threatmonitor < geoip-import.sql
|
17
17
|
|
@@ -21,7 +21,7 @@ http://172.17.0.1:4567/dashboard
|
|
21
21
|
|
22
22
|
Your defaults file is the config.rb in the gem directory for now.
|
23
23
|
|
24
|
-
/
|
24
|
+
/home/<username>/config.rb
|
25
25
|
|
26
26
|
To Capture data:
|
27
27
|
|
data/README.md
CHANGED
@@ -1,5 +1,13 @@
|
|
1
1
|
|
2
|
-
Threatmonitor - Packet Analysis suite
|
2
|
+
Threatmonitor - Packet Analysis suite with MonetDB / MySQL - RabbitMQ & PCap integration
|
3
|
+
|
4
|
+
|
5
|
+
`,`,`,`
|
6
|
+
------
|
7
|
+
( ͡° ͜ʖ ͡°)
|
8
|
+
\ /
|
9
|
+
\--/
|
10
|
+
|
3
11
|
|
4
12
|
![GeoIP](https://raw.githubusercontent.com/puppetpies/threatmonitor/master/screenshot-3-geo.jpg)
|
5
13
|
|
@@ -60,6 +68,10 @@ Import the schema from the SQL provided now moved to sql/
|
|
60
68
|
|
61
69
|
I've now included MySQL Database support also however if your going to create big data sets i think i would use MonetDB
|
62
70
|
|
63
|
-
![Dashboard Other](https://raw.githubusercontent.com/puppetpies/threatmonitor/master/screenshot-
|
71
|
+
![Dashboard Other](https://raw.githubusercontent.com/puppetpies/threatmonitor/master/screenshot-4.jpg)
|
72
|
+
|
73
|
+
We are also working on a Traffic Visualizer
|
74
|
+
|
75
|
+
![Trafviz](https://raw.githubusercontent.com/puppetpies/threatmonitor/master/screenshot-trafviz-1.jpg)
|
64
76
|
|
65
77
|
Have fun !
|
data/bin/thm-consumer
CHANGED
data/bin/thm-pcap
CHANGED
data/bin/thm-producer
CHANGED
data/bin/thm-session
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
########################################################################
|
3
3
|
#
|
4
4
|
# Author: Brian Hood
|
5
|
-
#
|
5
|
+
# Email: <brianh6854@googlemail.com>
|
6
6
|
# Description: Threatmonitor User Administration / Dashboard
|
7
7
|
# Codename: Deedrah
|
8
8
|
#
|
@@ -18,12 +18,12 @@ require 'slim'
|
|
18
18
|
require 'colorize'
|
19
19
|
|
20
20
|
require File.expand_path(File.join(
|
21
|
-
|
22
|
-
|
21
|
+
File.dirname(__FILE__),
|
22
|
+
"../thm-authentication.rb"))
|
23
23
|
|
24
24
|
require File.expand_path(File.join(
|
25
|
-
|
26
|
-
|
25
|
+
File.dirname(__FILE__),
|
26
|
+
"../lib/thm/version.rb"))
|
27
27
|
|
28
28
|
|
29
29
|
class Sinatra::Base
|
@@ -110,10 +110,10 @@ module ThmUI extend self
|
|
110
110
|
:httponly => true,
|
111
111
|
:secure => production?,
|
112
112
|
:expire_after => 3600, # 1 hour
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
113
|
+
#:views => File.expand_path(File.expand_path('../../views/', __FILE__)),
|
114
|
+
:views => Proc.new { File.join(root, "views") },
|
115
|
+
#:layout_engine => :slim,
|
116
|
+
:public_folder => File.dirname(__FILE__) + '/bin'
|
117
117
|
|
118
118
|
enable :method_override
|
119
119
|
|
@@ -127,10 +127,10 @@ module ThmUI extend self
|
|
127
127
|
# NOTE: Monkey patch Sinatra initialize
|
128
128
|
# If you go to /dashboard without logging in @sessobj it won't have been created
|
129
129
|
# So if you create it objstart! in the Sinatra's initialize your sorted.
|
130
|
-
# Page requests will come in
|
130
|
+
# Page requests will come in and @sessobj will always exist and be the same one no headaches
|
131
131
|
def initialize(app = nil)
|
132
132
|
super()
|
133
|
-
objstart! # Little trick here patch so i can get
|
133
|
+
objstart! # Little trick here patch so i can get @sessobj out of the box!
|
134
134
|
@app = app
|
135
135
|
@template_cache = Tilt::Cache.new
|
136
136
|
yield self if block_given?
|
@@ -210,7 +210,7 @@ module ThmUI extend self
|
|
210
210
|
ip_dst = row["ip_dst"].to_s
|
211
211
|
location = geoiplookup(ip_dst)
|
212
212
|
locfix = location.to_s.gsub("(", "").gsub(")", "") # Yawn
|
213
|
-
if location != nil or location == ""
|
213
|
+
if location != nil or location == "" # You can't have a blank or nil instance_variable
|
214
214
|
@gcnt.geocount("#{locfix}")
|
215
215
|
end
|
216
216
|
rowusrcnt << ["#{ip_dst} #{location}", "#{num2}"]
|
data/bin/thm-trafviz
ADDED
@@ -0,0 +1,316 @@
|
|
1
|
+
#!/usr/bin/ruby
|
2
|
+
########################################################################
|
3
|
+
#
|
4
|
+
# Author: Brian Hood
|
5
|
+
# Email: <brianh6854@googlemail.com>
|
6
|
+
# Description: Threatmonitor HTTP Viz
|
7
|
+
#
|
8
|
+
# Analyze HTTP Traffic using Traffic Visualizer
|
9
|
+
#
|
10
|
+
########################################################################
|
11
|
+
|
12
|
+
require 'pcaplet'
|
13
|
+
require 'getoptlong'
|
14
|
+
require 'walltime'
|
15
|
+
require 'readline'
|
16
|
+
require 'mymenu'
|
17
|
+
require 'pp'
|
18
|
+
|
19
|
+
require File.expand_path(File.join(
|
20
|
+
File.dirname(__FILE__),
|
21
|
+
"../lib/thm.rb"))
|
22
|
+
|
23
|
+
conf = Thm::FileServices.new
|
24
|
+
conf.thmhome?
|
25
|
+
|
26
|
+
include Thm::Defaults
|
27
|
+
include Tools
|
28
|
+
|
29
|
+
class NilClass
|
30
|
+
|
31
|
+
def strip
|
32
|
+
# Slightly naughty as makeurl hits a bug.
|
33
|
+
=begin
|
34
|
+
exception when looping over each packet loop: #<NoMethodError: undefined method `strip' for nil:NilClass>
|
35
|
+
./thm-httpseries:120:in `block in makeurl': undefined method `strip' for nil:NilClass (NoMethodError)
|
36
|
+
from ./thm-httpseries:118:in `each_line'
|
37
|
+
from ./thm-httpseries:118:in `makeurl'
|
38
|
+
from ./thm-httpseries:189:in `block in <main>'
|
39
|
+
from /usr/lib/ruby/gems/2.1.0/gems/pcap-0.7.7/lib/pcaplet.rb:94:in `loop'
|
40
|
+
from /usr/lib/ruby/gems/2.1.0/gems/pcap-0.7.7/lib/pcaplet.rb:94:in `each_packet'
|
41
|
+
from ./thm-httpseries:178:in `<main>'
|
42
|
+
=end
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
46
|
+
|
47
|
+
ARGV[0] = "--help" if ARGV[0] == nil
|
48
|
+
@debug = false
|
49
|
+
banner = "\e[1;34mWelcome to Threatmonitor HTTP Traffic Visualizer \e[0m\ \n"
|
50
|
+
banner << "\e[1;34m=================================================\e[0m\ \n\n"
|
51
|
+
m = %Q{
|
52
|
+
/\
|
53
|
+
/\/\
|
54
|
+
\/\/
|
55
|
+
\/}
|
56
|
+
|
57
|
+
opts = GetoptLong.new(
|
58
|
+
[ '--help', '-h', GetoptLong::NO_ARGUMENT ],
|
59
|
+
[ '--interface', '-i', GetoptLong::OPTIONAL_ARGUMENT],
|
60
|
+
[ '--snaplength', '-s', GetoptLong::OPTIONAL_ARGUMENT],
|
61
|
+
[ '--debug', '-d', GetoptLong::NO_ARGUMENT ],
|
62
|
+
)
|
63
|
+
|
64
|
+
opts.each do |opt, arg|
|
65
|
+
case opt
|
66
|
+
when '--help'
|
67
|
+
helper = %q[
|
68
|
+
-h, --help:
|
69
|
+
show help
|
70
|
+
|
71
|
+
-i, --interface - Network Interface to collect measurements from [ OPTIONAL_ARGUMENT ]
|
72
|
+
|
73
|
+
-s, --snaplength - Snaplength [ OPTIONAL_ARGUMENT ]
|
74
|
+
|
75
|
+
-d --debug
|
76
|
+
|
77
|
+
]
|
78
|
+
puts banner
|
79
|
+
puts helper
|
80
|
+
exit
|
81
|
+
when '--debug'
|
82
|
+
@debug = true
|
83
|
+
when '--interface'
|
84
|
+
@interface = nil || arg
|
85
|
+
when '--snaplength'
|
86
|
+
@snaplength = nil || arg
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
puts banner
|
91
|
+
|
92
|
+
# Trafviz DataServices
|
93
|
+
tv = Thm::DataServices::Trafviz.new
|
94
|
+
# Connect to Datastore
|
95
|
+
gloc = Thm::DataServices::Geolocation.new
|
96
|
+
gloc.datastore = DATASTORE
|
97
|
+
gloc.debug = false
|
98
|
+
gloc.autocommit = true
|
99
|
+
gloc.dbhost = DBHOST
|
100
|
+
gloc.dbuser = DBUSER
|
101
|
+
gloc.dbpass = DBPASS
|
102
|
+
gloc.dbname = DBNAME
|
103
|
+
gloc.dbconnect
|
104
|
+
|
105
|
+
use_const_defined_unless?("INTERFACE")
|
106
|
+
use_const_defined_unless?("SNAPLENGTH")
|
107
|
+
|
108
|
+
startup = "-s #{@snaplength} -n -i #{@interface}"
|
109
|
+
puts "Trafviz - Startup Parameters: #{startup}"
|
110
|
+
|
111
|
+
module Thm
|
112
|
+
|
113
|
+
class DataServices::Trafviz::FilterManager
|
114
|
+
|
115
|
+
attr_reader :bookmarks, :pcapsetfilter
|
116
|
+
|
117
|
+
def initialize
|
118
|
+
@bookmarks = Array.new
|
119
|
+
@bkm = MyMenu.new
|
120
|
+
@bkm.settitle("Welcome to Trafviz")
|
121
|
+
@bkm.mymenuname = "Trafviz"
|
122
|
+
@bkm.prompt = "Trafviz"
|
123
|
+
@pcapsetfilter = String.new
|
124
|
+
end
|
125
|
+
|
126
|
+
def read(file)
|
127
|
+
b = 0
|
128
|
+
File.open("#{Dir.home}/.thm/#{file}", 'r') {|n|
|
129
|
+
n.each_line {|l|
|
130
|
+
puts "\e[1;36m#{b})\e[0m\ #{l}"
|
131
|
+
@bookmarks[b] = l
|
132
|
+
b += 1
|
133
|
+
}
|
134
|
+
}
|
135
|
+
end
|
136
|
+
|
137
|
+
def write(file)
|
138
|
+
@bkm.mymenuname = "Filters"
|
139
|
+
@bkm.prompt = "\e[1;33m\Set filter>\e[0m\ "
|
140
|
+
pcapfilter = @bkm.definemenuitem("selectfilter", true) do
|
141
|
+
# Just needs value returned via readline block into addfilter
|
142
|
+
end
|
143
|
+
fltvalid = validate_filter?("#{pcapfilter}")
|
144
|
+
if fltvalid == true
|
145
|
+
File.open("#{Dir.home}/.thm/#{file}", 'a') {|n| # Append to filter file
|
146
|
+
n.puts("#{addfilter}")
|
147
|
+
}
|
148
|
+
end
|
149
|
+
end
|
150
|
+
|
151
|
+
def set_defaults(file)
|
152
|
+
# Add default example filters
|
153
|
+
File.open("#{Dir.home}/.thm/#{file}", 'w') {|n|
|
154
|
+
n.puts("webtraffic: tcp dst port 80")
|
155
|
+
n.puts("sourceportrange: tcp src portrange 1024-65535")
|
156
|
+
}
|
157
|
+
end
|
158
|
+
|
159
|
+
def validate_filter?(filter)
|
160
|
+
begin
|
161
|
+
Pcap::Filter.compile("#{filter}")
|
162
|
+
puts "Filter Compile #{filter}"
|
163
|
+
return true
|
164
|
+
rescue Pcap::PcapError => e
|
165
|
+
pp e
|
166
|
+
return false
|
167
|
+
end
|
168
|
+
end
|
169
|
+
|
170
|
+
def build_filter_menu
|
171
|
+
@bkm.settitle("Welcome to Trafviz")
|
172
|
+
@bkm.mymenuname = "Trafviz"
|
173
|
+
@bkm.prompt = "Trafviz"
|
174
|
+
@bkm.debug = 3
|
175
|
+
pp @bookmarks
|
176
|
+
@bookmarks.each {|n|
|
177
|
+
func_name = n.split(":")[0]
|
178
|
+
pcap_filter = n.split(":")[1].lstrip
|
179
|
+
puts "#{pcap_filter}"
|
180
|
+
# Instance Eval probably nicer
|
181
|
+
fltvalid = validate_filter?("#{pcap_filter}") # Because validate_filter? won't exist inside instance_eval
|
182
|
+
@bkm.instance_eval do
|
183
|
+
pp fltvalid
|
184
|
+
if fltvalid == true
|
185
|
+
definemenuitem("#{func_name}") do
|
186
|
+
@pcapsetfilter = "#{pcap_filter}"
|
187
|
+
#thm = DataServices::Trafviz::Main.new
|
188
|
+
end
|
189
|
+
additemtolist("#{func_name}: #{pcap_filter}", "#{func_name};")
|
190
|
+
end
|
191
|
+
end
|
192
|
+
}
|
193
|
+
@bkm.instance_eval do
|
194
|
+
definemenuitem("showfilter") do
|
195
|
+
puts "Filter: #{@pcapsetfilter}"
|
196
|
+
end
|
197
|
+
additemtolist("Show Current Filter", "showfilter;")
|
198
|
+
end
|
199
|
+
@bkm.additemtolist("Display Menu", "showmenu;")
|
200
|
+
@bkm.additemtolist("Toggle Menu", "togglemenu;")
|
201
|
+
@bkm.additemtolist("Exit Trafviz", "exit;")
|
202
|
+
@bkm.menu!
|
203
|
+
end
|
204
|
+
|
205
|
+
def load_filters(file)
|
206
|
+
if File.exists?("#{Dir.home}/.thm/#{file}")
|
207
|
+
read(file)
|
208
|
+
else
|
209
|
+
set_defaults(file)
|
210
|
+
read(file)
|
211
|
+
end
|
212
|
+
build_filter_menu
|
213
|
+
end
|
214
|
+
|
215
|
+
end
|
216
|
+
|
217
|
+
end
|
218
|
+
|
219
|
+
# Main class / Startup
|
220
|
+
|
221
|
+
module Thm
|
222
|
+
|
223
|
+
class DataServices::Trafviz::Main
|
224
|
+
|
225
|
+
attr_accessor :startup
|
226
|
+
|
227
|
+
def initialize
|
228
|
+
@filter_const = Array.new
|
229
|
+
@startup = String.new
|
230
|
+
@thm = Thm::DataServices::Trafviz::FilterManager.new
|
231
|
+
end
|
232
|
+
|
233
|
+
def addfilter(const, filter)
|
234
|
+
if @thm.validate_filter?(filter) == true
|
235
|
+
filtercode = %Q{#{const} = Pcap::Filter.new('#{filter}', @trafviz.capture)}
|
236
|
+
@filter_const << "#{const})"
|
237
|
+
eval(filtercode)
|
238
|
+
end
|
239
|
+
end
|
240
|
+
|
241
|
+
def commitfilters
|
242
|
+
flts = @filter_const.join(" | ") # Build string of CONST names
|
243
|
+
commitcode = %Q{@trafviz.add_filter(#{flts})}
|
244
|
+
eval(flts)
|
245
|
+
end
|
246
|
+
|
247
|
+
def run!
|
248
|
+
@trafviz = Pcaplet.new(@startup)
|
249
|
+
end
|
250
|
+
end
|
251
|
+
|
252
|
+
end
|
253
|
+
=begin
|
254
|
+
FILTERLIST = 'filters.lst'
|
255
|
+
a = Thm::DataServices::Trafviz::FilterManager.new
|
256
|
+
a.load_filters("#{FILTERLIST}")
|
257
|
+
a.instance_eval do
|
258
|
+
definemenuitem("addfilter") do
|
259
|
+
puts "Filter: #{@pcapsetfilter}"
|
260
|
+
end
|
261
|
+
additemtolist("Add New Filter", "write('#{FILTERLIST}');")
|
262
|
+
end
|
263
|
+
a.menu!
|
264
|
+
=end
|
265
|
+
|
266
|
+
@trafviz = Pcaplet.new(startup)
|
267
|
+
HTTP_REQUEST = Pcap::Filter.new('tcp dst port 80', @trafviz.capture)
|
268
|
+
HTTP_RESPONSE = Pcap::Filter.new('tcp src portrange 1024-65535', @trafviz.capture)
|
269
|
+
|
270
|
+
@trafviz.add_filter(HTTP_REQUEST | HTTP_RESPONSE)
|
271
|
+
@trafviz.each_packet {|pkt|
|
272
|
+
data = pkt.tcp_data.to_s
|
273
|
+
data_orig = data.clone
|
274
|
+
data_highlight = tv.text_highlighter(data_orig)
|
275
|
+
case pkt
|
276
|
+
when HTTP_REQUEST
|
277
|
+
if data =~ HTTP_METHODS_REGEXP
|
278
|
+
stwt = Stopwatch.new
|
279
|
+
stwt.watch('start')
|
280
|
+
path = $1
|
281
|
+
host = pkt.dst.to_s
|
282
|
+
host << ":\e[1;33m#{pkt.dport}\e[0m\ "
|
283
|
+
s = "\e[1;33m#{pkt.src}:\e[1;31m#{pkt.sport}\e[0m\ > GET \e[1;33mhttp://#{host}\e[1;32mHTTP/1.1\e[0m "
|
284
|
+
geo = gloc.geoiplookup(host.split(":")[0])
|
285
|
+
puts "\e[4;36mGeo Location:\e[0m\ \n\e[0;35m#{geo} \e[0m\ "
|
286
|
+
puts "\e[4;36mRequest Data:\e[0m\ \n\e[0;32m#{data_highlight} \e[0m\ "
|
287
|
+
tv.makeurl(data_orig)
|
288
|
+
# Process data and prepare then send elsewhere
|
289
|
+
query_return_sql = tv.request_filter(HTTP_REQUEST_TABLE, data)
|
290
|
+
# Store data into InfluxDB API Capture if @mtable exists else Datastore
|
291
|
+
begin
|
292
|
+
ires = gloc.query("#{query_return_sql}")
|
293
|
+
if @debug == true
|
294
|
+
puts "\e[4;36mStructured Query:\e[0m\ #{query_return_sql} \e[4;36mResult:\e[0m\ #{ires}"
|
295
|
+
end
|
296
|
+
rescue
|
297
|
+
Tools::log_errors("/tmp/thm-sql-errors.log", "SQL Error - #{Time.now} - #{query_return_sql}") # Catch them all
|
298
|
+
end
|
299
|
+
stwt.watch('stop')
|
300
|
+
stwt.print_stats
|
301
|
+
elsif data =~ %r=^POST $=
|
302
|
+
puts data_highlight
|
303
|
+
end
|
304
|
+
when HTTP_RESPONSE
|
305
|
+
if data =~ HTTP_METHODS_REGEXP_RESPONSE
|
306
|
+
stwt = Stopwatch.new
|
307
|
+
stwt.watch('start')
|
308
|
+
status = $1
|
309
|
+
s = "#{pkt.dst}:#{pkt.dport} < #{status}"
|
310
|
+
puts "\e[1;31mResponse Data: #{data_highlight} \e[0m\ "
|
311
|
+
stwt.watch('stop')
|
312
|
+
stwt.print_stats
|
313
|
+
end
|
314
|
+
end
|
315
|
+
puts s.gsub("GET", "\e[1;36mGET\e[0m").gsub("POST", "\e[1;36mPOST\e[0m") if s
|
316
|
+
}
|
data/bin/thm-useradmin
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
########################################################################
|
3
3
|
#
|
4
4
|
# Author: Brian Hood
|
5
|
-
#
|
5
|
+
# Email: <brianh6854@googlemail.com>
|
6
6
|
# Description: Threatmonitor User Administration
|
7
7
|
#
|
8
8
|
# Extends the functionality of the Thm module adding Authorization
|
@@ -23,9 +23,21 @@ trap("INT") {
|
|
23
23
|
exit
|
24
24
|
}
|
25
25
|
|
26
|
+
conf = Thm::FileServices.new
|
27
|
+
conf.thmhome?
|
28
|
+
|
29
|
+
include Thm::Defaults
|
30
|
+
|
26
31
|
obj = Thm::Authorization::Privileges.new
|
27
|
-
obj.datastore =
|
32
|
+
obj.datastore = DATASTORE
|
33
|
+
obj.debug = false
|
34
|
+
obj.autocommit = true
|
35
|
+
obj.dbhost = DBHOST
|
36
|
+
obj.dbuser = DBUSER
|
37
|
+
obj.dbpass = DBPASS
|
38
|
+
obj.dbname = DBNAME
|
28
39
|
obj.dbconnect
|
40
|
+
|
29
41
|
if obj.user_exists?("admin") == true
|
30
42
|
# Menu
|
31
43
|
while buf = Readline.readline("\e[1;32m\Threatmonitor>\e[0m\ ", true)
|
data/config.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
########################################################################
|
2
2
|
#
|
3
3
|
# Author: Brian Hood
|
4
|
-
#
|
4
|
+
# Email: <brianh6854@googlemail.com>
|
5
5
|
# Description: Threatmonitor User configuration
|
6
6
|
#
|
7
7
|
# Lets keep it simple and use constants
|
@@ -10,21 +10,31 @@
|
|
10
10
|
module Thm
|
11
11
|
|
12
12
|
module Defaults
|
13
|
+
|
14
|
+
DATASTORE = "monetdb"
|
15
|
+
MQHOST = "127.0.0.1"
|
16
|
+
MQUSER = "traffic"
|
17
|
+
MQPASS = "dk3rbi9l"
|
18
|
+
MQVHOST = "/"
|
19
|
+
DBHOST = "127.0.0.1"
|
20
|
+
DBUSER = "threatmonitor"
|
21
|
+
DBPASS = "dk3rbi9l"
|
22
|
+
DBNAME = "threatmonitor"
|
23
|
+
QUEUEPREFIX = "wifi"
|
24
|
+
TBLNAME_IPPACKET = "#{QUEUEPREFIX}_ippacket"
|
25
|
+
TBLNAME_TCPPACKET = "#{QUEUEPREFIX}_tcppacket"
|
26
|
+
TBLNAME_UDPPACKET = "#{QUEUEPREFIX}_udppacket"
|
27
|
+
|
28
|
+
# Regexp strings first currently excludes POST Data
|
29
|
+
HTTP_METHODS_REGEXP = %r=^GET |^HEAD |^PUT |^TRACE |^CONNECT |^OPTIONS |^DELETE |^PROPFIND |^PROPPATCH |^MKCOL |^COPY |^MOVE |^LOCK |^UNLOCK =
|
30
|
+
HTTP_METHODS_REGEXP_RESPONSE = %r=^(HTTP\/.*)$=
|
31
|
+
HTTP_REQUEST_TABLE = "http_traffic_json"
|
32
|
+
HTTP_RESPONSE_TABLE = "http_traffic_json"
|
13
33
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
MQVHOST = "/"
|
19
|
-
DBHOST = "127.0.0.1"
|
20
|
-
DBUSER = "threatmonitor"
|
21
|
-
DBPASS = "dk3rbi9l"
|
22
|
-
DBNAME = "threatmonitor"
|
23
|
-
QUEUEPREFIX = "wifi"
|
24
|
-
TBLNAME_IPPACKET = "#{QUEUEPREFIX}_ippacket"
|
25
|
-
TBLNAME_TCPPACKET = "#{QUEUEPREFIX}_tcppacket"
|
26
|
-
TBLNAME_UDPPACKET = "#{QUEUEPREFIX}_udppacket"
|
27
|
-
|
34
|
+
# Misc
|
35
|
+
SNAPLENGTH = 65536
|
36
|
+
INTERFACE = "eth0"
|
37
|
+
|
28
38
|
end
|
29
39
|
|
30
40
|
end
|
data/lib/thm/consumer.rb
CHANGED
@@ -1,3 +1,12 @@
|
|
1
|
+
########################################################################
|
2
|
+
#
|
3
|
+
# Author: Brian Hood
|
4
|
+
# Email: <brianh6854@googlemail.com>
|
5
|
+
# Description: Libraries
|
6
|
+
#
|
7
|
+
# Pull data from required sources
|
8
|
+
#
|
9
|
+
########################################################################
|
1
10
|
|
2
11
|
trap("INT") {
|
3
12
|
|
data/lib/thm/datalayerlight.rb
CHANGED
@@ -149,7 +149,7 @@ module DatalayerLight
|
|
149
149
|
require "json"
|
150
150
|
require "pp"
|
151
151
|
|
152
|
-
attr_accessor :dbhost, :dbuser, :dbpass, :dbport, :dburl
|
152
|
+
attr_accessor :dbhost, :dbuser, :dbpass, :dbport, :dburl, :dbname
|
153
153
|
|
154
154
|
def initialize
|
155
155
|
@dbhost = "127.0.0.1"
|
@@ -189,13 +189,16 @@ module DatalayerLight
|
|
189
189
|
begin
|
190
190
|
request.body = data unless data.empty?
|
191
191
|
response = http.request(request)
|
192
|
-
if response.code == 204 # Good response
|
193
|
-
|
194
|
-
|
192
|
+
if response.code == "204" # Good response
|
193
|
+
# Be quiet
|
194
|
+
return response.code
|
195
|
+
elsif response.code =~ %r=[200,400,500]= # 200 can be an error in some cases !!
|
195
196
|
puts "Error code #{response.code}"
|
197
|
+
return response.code
|
196
198
|
end
|
197
199
|
rescue
|
198
200
|
puts "Error posting data"
|
201
|
+
return "404"
|
199
202
|
end
|
200
203
|
end
|
201
204
|
|
@@ -0,0 +1,92 @@
|
|
1
|
+
########################################################################
|
2
|
+
#
|
3
|
+
# Author: Brian Hood
|
4
|
+
# Email: <brianh6854@googlemail.com>
|
5
|
+
# Description: Libraries - geolocation
|
6
|
+
#
|
7
|
+
# Geo Data quick and simple
|
8
|
+
#
|
9
|
+
########################################################################
|
10
|
+
|
11
|
+
require 'pp'
|
12
|
+
|
13
|
+
module Thm
|
14
|
+
|
15
|
+
class DataServices::Geolocation < DataServices
|
16
|
+
|
17
|
+
attr_writer :geodebug
|
18
|
+
|
19
|
+
def initialize
|
20
|
+
@geodebug = false
|
21
|
+
@continent_name, @country_name, @city_name = Array.new, Array.new, Array.new
|
22
|
+
end
|
23
|
+
|
24
|
+
def formatinet(ip, octets=2)
|
25
|
+
if octets == 2
|
26
|
+
"#{ip.split(".")[0]}.#{ip.split(".")[1]}"
|
27
|
+
elsif octets == 3
|
28
|
+
"#{ip.split(".")[0]}.#{ip.split(".")[1]}.#{ip.split(".")[2]}"
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def self.define_component(name)
|
33
|
+
name_func = name.to_sym
|
34
|
+
define_method(name_func) do |ip|
|
35
|
+
octets = formatinet(ip, 2)
|
36
|
+
geoquery = "SELECT count(*) as num FROM geoipdata_ipv4blocks_#{name_func} a "
|
37
|
+
geoquery << "JOIN geoipdata_locations_#{name_func} b ON (a.geoname_id = b.geoname_id) "
|
38
|
+
geoquery << "WHERE network LIKE '#{octets}.%';"
|
39
|
+
begin
|
40
|
+
res = @conn.query("#{geoquery}")
|
41
|
+
rowgeocount = res.fetch_hash
|
42
|
+
geocount = rowgeocount["num"].to_i
|
43
|
+
rescue => e
|
44
|
+
pp e
|
45
|
+
end
|
46
|
+
puts "Geo SELECT COUNT: #{geoquery}: Number: #{geocount}" if @geodebug == true
|
47
|
+
if geocount > 0;
|
48
|
+
geoquery = "SELECT continent_name, #{name_func}_name FROM geoipdata_ipv4blocks_#{name_func} a "
|
49
|
+
geoquery << "JOIN geoipdata_locations_#{name_func} b ON (a.geoname_id = b.geoname_id) "
|
50
|
+
geoquery << "WHERE network LIKE '#{octets}.%' GROUP BY b.continent_name, b.#{name_func}_name LIMIT 1;"
|
51
|
+
puts "Geo SELECT: #{geoquery}" if @geodebug == true
|
52
|
+
begin
|
53
|
+
resgeo = @conn.query("#{geoquery}")
|
54
|
+
while row = resgeo.fetch_hash do
|
55
|
+
populategeo = instance_variable_get("@#{name_func}_name")
|
56
|
+
populategeo << row["#{name_func}_name"].to_s
|
57
|
+
instance_variable_set("@#{name_func}_name", populategeo)
|
58
|
+
@continent_name = row["continent_name"].to_s
|
59
|
+
end
|
60
|
+
rescue => e
|
61
|
+
pp e
|
62
|
+
end
|
63
|
+
else
|
64
|
+
return false
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
define_component :country
|
70
|
+
define_component :city
|
71
|
+
|
72
|
+
def geoiplookup(ip)
|
73
|
+
t = country(ip)
|
74
|
+
city(ip)
|
75
|
+
unless t == false
|
76
|
+
res = "(#{@continent_name}) - \n"
|
77
|
+
@country_name.each {|n|
|
78
|
+
res << "[ #{n} ]"
|
79
|
+
}
|
80
|
+
@city_name.each {|n|
|
81
|
+
res << "[ #{n} ] "
|
82
|
+
}
|
83
|
+
initialize
|
84
|
+
else
|
85
|
+
res = "Not Available"
|
86
|
+
end
|
87
|
+
return res
|
88
|
+
end
|
89
|
+
|
90
|
+
end
|
91
|
+
|
92
|
+
end
|
@@ -0,0 +1,152 @@
|
|
1
|
+
########################################################################
|
2
|
+
#
|
3
|
+
# Author: Brian Hood
|
4
|
+
# Email: <brianh6854@googlemail.com>
|
5
|
+
# Description: Libraries - Trafviz
|
6
|
+
#
|
7
|
+
# Data parsing functionality
|
8
|
+
#
|
9
|
+
########################################################################
|
10
|
+
|
11
|
+
require 'json'
|
12
|
+
|
13
|
+
module Thm
|
14
|
+
|
15
|
+
class DataServices::Trafviz
|
16
|
+
|
17
|
+
def initialize
|
18
|
+
@debug = true
|
19
|
+
end
|
20
|
+
|
21
|
+
def makeurl(data)
|
22
|
+
if !request_valid?(data)
|
23
|
+
return false
|
24
|
+
end
|
25
|
+
hdrs = data
|
26
|
+
hostn, requestn = ""
|
27
|
+
hdrs.each_line {|n|
|
28
|
+
if n.split(":")[0] == "Host"
|
29
|
+
hostn = n.split(":")[1].strip
|
30
|
+
elsif n.split(" ")[0] == "GET"
|
31
|
+
requestn = n.split(" ")[1]
|
32
|
+
end
|
33
|
+
}
|
34
|
+
puts "\e[1;37mURL: http://#{hostn}#{requestn} \e[0m\ "
|
35
|
+
end
|
36
|
+
|
37
|
+
# Check if a request isn't just a GET line without headers / single line
|
38
|
+
# Not sure if this is valid HTTP
|
39
|
+
def request_valid?(data)
|
40
|
+
ln = 0
|
41
|
+
data.each_line {|l|
|
42
|
+
ln += 1
|
43
|
+
}
|
44
|
+
if ln > 1
|
45
|
+
return true
|
46
|
+
else
|
47
|
+
puts "\e[1;31mCatch GET's without header information / Other \e[0m\ "
|
48
|
+
return false # Due to single GET Requests to no headers
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
# This is just an informal function when in debug mode
|
53
|
+
def hit_header(hdrs)
|
54
|
+
puts "Hit #{hdrs} header"
|
55
|
+
end
|
56
|
+
|
57
|
+
# Cookie ommit as we don't want to steal cookie data and pointless to store.
|
58
|
+
def filter_header?(lkey)
|
59
|
+
puts "MY LKEY: |#{lkey}|" if @debug == true
|
60
|
+
case lkey.strip
|
61
|
+
when "cookie"
|
62
|
+
hit_header(lkey) if @debug == true
|
63
|
+
return true
|
64
|
+
when "range"
|
65
|
+
hit_header(lkey) if @debug == true
|
66
|
+
return true
|
67
|
+
else
|
68
|
+
return false
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
# Right Cell eval
|
73
|
+
def rkey_decode(rkey)
|
74
|
+
rkeyenc = URI.decode(rkey)
|
75
|
+
end
|
76
|
+
|
77
|
+
# Filter lkey = header, rkey = requestdata
|
78
|
+
def lkey_strip(hdrs)
|
79
|
+
hdrs.split(": ")[0].downcase.gsub("-", "").to_s.strip
|
80
|
+
end
|
81
|
+
|
82
|
+
def rkey_strip(data)
|
83
|
+
data.split(": ")[1].to_s.strip #to_s.gsub(",", "").gsub(";", "").gsub("=", "").strip
|
84
|
+
end
|
85
|
+
|
86
|
+
# Filter request data and build query
|
87
|
+
def request_filter(reqtable, data, keysamples=2000)
|
88
|
+
if !request_valid?(data)
|
89
|
+
sql = "SELECT 1;"
|
90
|
+
return sql
|
91
|
+
end
|
92
|
+
guid = Tools::guid
|
93
|
+
cols, vals = String.new, String.new
|
94
|
+
lkey, rkey = String.new, String.new
|
95
|
+
json_data_pieces = String.new
|
96
|
+
t = 0
|
97
|
+
json_data_hdr = "@json_template = { 'http' => { "
|
98
|
+
json_data_ftr = " } }"
|
99
|
+
sql = "INSERT INTO #{reqtable} (recv_time,recv_date,guid,json_data) "
|
100
|
+
data.each_line {|n|
|
101
|
+
unless n.strip == ""
|
102
|
+
if t > 0 # Don't processes GET / POST Line
|
103
|
+
lkey, rkey = lkey_strip(n), rkey_strip(n)
|
104
|
+
puts "LKEY: #{lkey} RKEY: #{rkey}" if @debug == true
|
105
|
+
rkeyenc = filter_header?(lkey)
|
106
|
+
if rkeyenc == false
|
107
|
+
rkeyenc = rkey_decode(rkey)
|
108
|
+
else
|
109
|
+
rkey = "ommited"
|
110
|
+
end
|
111
|
+
if rkey.strip != "" or lkey.strip != ""
|
112
|
+
prerkeyins = rkey.gsub('"', '') # Strip Quotes
|
113
|
+
prerkeyins = "blank" if prerkeyins.strip == "" # Seems JSON values can't be "accept":""
|
114
|
+
puts "Found Blank Value!!!" if prerkeyins == "blank"
|
115
|
+
json_data_pieces << "'#{lkey}' => \"#{prerkeyins}\",\n"
|
116
|
+
end
|
117
|
+
end
|
118
|
+
t += 1
|
119
|
+
end
|
120
|
+
}
|
121
|
+
# SQL for Datastore
|
122
|
+
begin
|
123
|
+
# Remove last , to fix hash table
|
124
|
+
json_data_pieces.sub!(%r{,\n$}, "")
|
125
|
+
json_eval = %Q{#{json_data_hdr}#{json_data_pieces}#{json_data_ftr}}
|
126
|
+
puts "\e[4;36mJSON Data:\e[0m\ \n#{json_eval}"
|
127
|
+
eval(json_eval) # Unsure why a local variable works for this in IRB
|
128
|
+
json_data = @json_template.to_json
|
129
|
+
remove_instance_variable("@json_template") # Hence remove instance variable here
|
130
|
+
# Added GUID as i could extend TCP/IP capture suites in the future for HTTP traffic
|
131
|
+
sql = "#{sql}VALUES (NOW(), NOW(), '#{guid}', '#{json_data}');"
|
132
|
+
return sql
|
133
|
+
rescue => e
|
134
|
+
pp e
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
def text_highlighter(text)
|
139
|
+
keys = ["Linux", "Java", "Android", "iPhone", "Mobile", "Chrome",
|
140
|
+
"Safari", "Mozilla", "Gecko", "AppleWebKit", "Windows",
|
141
|
+
"MSIE", "Win64", "Trident", "wispr", "PHPSESSID", "JSESSIONID",
|
142
|
+
"AMD64", "Darwin", "Macintosh", "Mac OS X", "Dalvik", "text/html", "xml"]
|
143
|
+
cpicker = [2,3,4,1,7,5,6]
|
144
|
+
keys.each {|n|
|
145
|
+
text.gsub!("#{n}", "\e[4;3#{cpicker[rand(cpicker.size)]}m#{n}\e[0m\ \e[0;32m".strip)
|
146
|
+
}
|
147
|
+
return text
|
148
|
+
end
|
149
|
+
|
150
|
+
end
|
151
|
+
|
152
|
+
end
|
data/lib/thm/dataservices.rb
CHANGED
@@ -13,10 +13,12 @@ module Thm
|
|
13
13
|
# obj.mqconnect
|
14
14
|
# obj.dbconnect
|
15
15
|
|
16
|
-
attr_accessor :datastore, :mqhost, :mquser, :mqpass, :mqvhost, :dbhost, :dbuser, :dbpass, :dbname, :queueprefix, :tblname_ippacket, :tblname_tcppacket, :tblname_udppacket
|
16
|
+
attr_accessor :autocommit, :datastore, :debug, :mqhost, :mquser, :mqpass, :mqvhost, :dbhost, :dbuser, :dbpass, :dbname, :queueprefix, :tblname_ippacket, :tblname_tcppacket, :tblname_udppacket
|
17
17
|
|
18
18
|
def initialize
|
19
|
+
@autocommit = false
|
19
20
|
@datastore = "monetdb"
|
21
|
+
@debug = false
|
20
22
|
@mqhost = "127.0.0.1"
|
21
23
|
@mquser = "traffic"
|
22
24
|
@mqpass = "dk3rbi9l"
|
@@ -54,7 +56,8 @@ module Thm
|
|
54
56
|
@conn.username = @dbuser
|
55
57
|
@conn.password = @dbpass
|
56
58
|
@conn.dbname = @dbname
|
57
|
-
@conn.
|
59
|
+
@conn.debug = @debug
|
60
|
+
@conn.autocommit = @autocommit
|
58
61
|
begin
|
59
62
|
@conn.connect
|
60
63
|
rescue Errno::ECONNREFUSED
|
@@ -71,3 +74,7 @@ module Thm
|
|
71
74
|
end
|
72
75
|
|
73
76
|
end
|
77
|
+
|
78
|
+
require_relative 'dataservices/trafviz/trafviz.rb'
|
79
|
+
require_relative 'dataservices/geolocation/geolocation.rb'
|
80
|
+
|
data/lib/thm/fileservices.rb
CHANGED
@@ -1,3 +1,12 @@
|
|
1
|
+
########################################################################
|
2
|
+
#
|
3
|
+
# Author: Brian Hood
|
4
|
+
# Email: <brianh6854@googlemail.com>
|
5
|
+
# Description: Libraries
|
6
|
+
#
|
7
|
+
# File services for local / global settings
|
8
|
+
#
|
9
|
+
########################################################################
|
1
10
|
|
2
11
|
module Thm
|
3
12
|
|
@@ -15,10 +24,10 @@ module Thm
|
|
15
24
|
}
|
16
25
|
end
|
17
26
|
begin
|
18
|
-
if loadswitch == true
|
19
|
-
require Dir.home
|
27
|
+
if loadswitch == true # So original backup config doesn't change your settings
|
28
|
+
require "#{Dir.home}/.thm/#{file}"
|
20
29
|
end
|
21
|
-
|
30
|
+
rescue
|
22
31
|
puts "Failed to load something went wrong check permissions !"
|
23
32
|
end
|
24
33
|
end
|
@@ -28,7 +37,7 @@ module Thm
|
|
28
37
|
if Dir.exists?("#{Dir.home}/.thm") == false
|
29
38
|
Dir.mkdir("#{Dir.home}/.thm")
|
30
39
|
puts "Creating .thm home subfolder copying config.rb"
|
31
|
-
|
40
|
+
#puts "#{File.getwd}"
|
32
41
|
File.open(File.expand_path(File.join(File.dirname(__FILE__), "../../#{file}")), 'r') {|n|
|
33
42
|
n.each_line {|l|
|
34
43
|
@fdata << l
|
@@ -36,8 +45,8 @@ module Thm
|
|
36
45
|
}
|
37
46
|
end
|
38
47
|
begin
|
39
|
-
|
40
|
-
|
48
|
+
#puts "FDATA: #{@fdata}"
|
49
|
+
#puts "Begin"
|
41
50
|
conf_loader("#{file}")
|
42
51
|
conf_loader("config-original.rb", false)
|
43
52
|
rescue
|
data/lib/thm/localmachine.rb
CHANGED
@@ -1,3 +1,12 @@
|
|
1
|
+
########################################################################
|
2
|
+
#
|
3
|
+
# Author: Brian Hood
|
4
|
+
# Email: <brianh6854@googlemail.com>
|
5
|
+
# Description: Libraries
|
6
|
+
#
|
7
|
+
# Remote to Local Pcap operations
|
8
|
+
#
|
9
|
+
########################################################################
|
1
10
|
|
2
11
|
module Thm
|
3
12
|
|
data/lib/thm/producer.rb
CHANGED
@@ -1,3 +1,13 @@
|
|
1
|
+
########################################################################
|
2
|
+
#
|
3
|
+
# Author: Brian Hood
|
4
|
+
# Email: <brianh6854@googlemail.com>
|
5
|
+
# Description: Libraries
|
6
|
+
#
|
7
|
+
# Message queue data producer / data transport
|
8
|
+
#
|
9
|
+
########################################################################
|
10
|
+
|
1
11
|
module Thm
|
2
12
|
# Send data else were
|
3
13
|
|
data/lib/thm/version.rb
CHANGED
data/lib/thm.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
########################################################################
|
2
2
|
#
|
3
3
|
# Author: Brian Hood
|
4
|
-
#
|
4
|
+
# Email: <brianh6854@googlemail.com>
|
5
5
|
# Description: Threatmonitor Producer
|
6
6
|
#
|
7
7
|
# Producer / Consumer controller module
|
@@ -22,7 +22,15 @@ include Pcap
|
|
22
22
|
#
|
23
23
|
# Create def's for that packet SQL / Refactor to provent code duplication
|
24
24
|
# Create def's for Hash table YAML same idea as above.
|
25
|
-
|
25
|
+
|
26
|
+
class String
|
27
|
+
|
28
|
+
def size_minus(min=1)
|
29
|
+
size - min
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
33
|
+
|
26
34
|
module Tools
|
27
35
|
|
28
36
|
class << self
|
@@ -30,38 +38,59 @@ module Tools
|
|
30
38
|
def guid
|
31
39
|
guid = Guid.new # Generate GUID
|
32
40
|
end
|
33
|
-
|
41
|
+
|
42
|
+
def log_errors(file, data)
|
43
|
+
File.open("#{file}", 'a') {|n|
|
44
|
+
n.puts("#{data}")
|
45
|
+
}
|
46
|
+
end
|
47
|
+
|
48
|
+
end
|
49
|
+
|
50
|
+
def use_const_defined_unless?(const)
|
51
|
+
const_down = const.downcase
|
52
|
+
if Kernel.const_defined?("#{const}")
|
53
|
+
if instance_variable_get("@#{const_down}") == nil
|
54
|
+
instance_variable_set("@#{const_down}", Kernel.const_get("#{const}"))
|
55
|
+
puts "Config Constant #{const}: #{Kernel.const_get("#{const}")}"
|
56
|
+
puts "Instance Variable @#{const_down}: #{instance_variable_get("@#{const_down}")}"
|
57
|
+
else
|
58
|
+
puts "Param via Getoptlong: Instance Variable #{@const_down}: #{instance_variable_get("@#{const_down}")}"
|
59
|
+
end
|
60
|
+
else
|
61
|
+
raise "No Config option set add #{const} to your config.rb"
|
62
|
+
end
|
34
63
|
end
|
35
64
|
|
36
65
|
end
|
37
66
|
|
38
67
|
# Load Database drivers
|
39
68
|
require File.expand_path(File.join(
|
40
|
-
|
41
|
-
|
69
|
+
File.dirname(__FILE__),
|
70
|
+
"../lib/thm/datalayerlight.rb"))
|
42
71
|
|
43
72
|
# Load Datasources / Services contains defaults
|
44
73
|
require File.expand_path(File.join(
|
45
|
-
|
46
|
-
|
74
|
+
File.dirname(__FILE__),
|
75
|
+
"../lib/thm/dataservices.rb"))
|
47
76
|
|
48
77
|
require File.expand_path(File.join(
|
49
|
-
|
50
|
-
|
78
|
+
File.dirname(__FILE__),
|
79
|
+
"../lib/thm/producer.rb"))
|
51
80
|
|
52
81
|
require File.expand_path(File.join(
|
53
|
-
|
54
|
-
|
82
|
+
File.dirname(__FILE__),
|
83
|
+
"../lib/thm/consumer.rb"))
|
55
84
|
|
56
85
|
require File.expand_path(File.join(
|
57
|
-
|
58
|
-
|
86
|
+
File.dirname(__FILE__),
|
87
|
+
"../lib/thm/localmachine.rb"))
|
59
88
|
|
60
89
|
require File.expand_path(File.join(
|
61
|
-
|
62
|
-
|
90
|
+
File.dirname(__FILE__),
|
91
|
+
"../lib/thm/fileservices.rb"))
|
63
92
|
|
64
93
|
# Versioning information
|
65
94
|
require File.expand_path(File.join(
|
66
|
-
|
67
|
-
|
95
|
+
File.dirname(__FILE__),
|
96
|
+
"../lib/thm/version.rb"))
|
data/sql/geoipdata-monetdb.sql
CHANGED
@@ -9,15 +9,15 @@
|
|
9
9
|
|
10
10
|
DROP TABLE "threatmonitor".geoipdata_ipv4blocks_city;
|
11
11
|
CREATE TABLE "threatmonitor".geoipdata_ipv4blocks_city (
|
12
|
-
network varchar(18),
|
13
|
-
geoname_id char(10),
|
14
|
-
registered_country_geoname_id char(30),
|
15
|
-
represented_country_geoname_id char(30),
|
16
|
-
is_anonymous_proxy char(30),
|
17
|
-
is_satellite_provider char(30),
|
18
|
-
postal_code char(30),
|
19
|
-
latitude char(10),
|
20
|
-
longitude char(10)
|
12
|
+
network varchar(18) NOT NULL,
|
13
|
+
geoname_id char(10) NOT NULL,
|
14
|
+
registered_country_geoname_id char(30) NOT NULL,
|
15
|
+
represented_country_geoname_id char(30) NOT NULL,
|
16
|
+
is_anonymous_proxy char(30) NOT NULL,
|
17
|
+
is_satellite_provider char(30) NOT NULL,
|
18
|
+
postal_code char(30) NOT NULL,
|
19
|
+
latitude char(10) NOT NULL,
|
20
|
+
longitude char(10) NOT NULL
|
21
21
|
);
|
22
22
|
|
23
23
|
CREATE INDEX cindex_ipv4_network ON "threatmonitor".geoipdata_ipv4blocks_city(network);
|
@@ -27,19 +27,19 @@ COPY 2519918 OFFSET 2 RECORDS INTO "threatmonitor".geoipdata_ipv4blocks_city FRO
|
|
27
27
|
|
28
28
|
DROP TABLE "threatmonitor".geoipdata_locations_city;
|
29
29
|
CREATE TABLE "threatmonitor".geoipdata_locations_city (
|
30
|
-
geoname_id char(10),
|
31
|
-
locale_code char(2),
|
32
|
-
continent_code char(2),
|
33
|
-
continent_name char(15),
|
34
|
-
country_iso_code char(2),
|
35
|
-
country_name char(50),
|
36
|
-
subdivision_1_iso_code char(70),
|
37
|
-
subdivision_1_name char(50),
|
38
|
-
subdivision_2_iso_code char(70),
|
39
|
-
subdivision_2_name char(50),
|
40
|
-
city_name char(70),
|
41
|
-
metro_code char(30),
|
42
|
-
time_zone char(30)
|
30
|
+
geoname_id char(10) NOT NULL,
|
31
|
+
locale_code char(2) NOT NULL,
|
32
|
+
continent_code char(2) NOT NULL,
|
33
|
+
continent_name char(15) NOT NULL,
|
34
|
+
country_iso_code char(2) NOT NULL,
|
35
|
+
country_name char(50) NOT NULL,
|
36
|
+
subdivision_1_iso_code char(70) NOT NULL,
|
37
|
+
subdivision_1_name char(50) NOT NULL,
|
38
|
+
subdivision_2_iso_code char(70) NOT NULL,
|
39
|
+
subdivision_2_name char(50) NOT NULL,
|
40
|
+
city_name char(70) NOT NULL,
|
41
|
+
metro_code char(30) NOT NULL,
|
42
|
+
time_zone char(30) NOT NULL
|
43
43
|
);
|
44
44
|
|
45
45
|
CREATE INDEX cindex_country_geoname_id ON "threatmonitor".geoipdata_locations_city(geoname_id);
|
@@ -48,12 +48,12 @@ COPY 80006 OFFSET 2 RECORDS INTO "threatmonitor".geoipdata_locations_city FROM '
|
|
48
48
|
|
49
49
|
DROP TABLE "threatmonitor".geoipdata_ipv4blocks_country;
|
50
50
|
CREATE TABLE "threatmonitor".geoipdata_ipv4blocks_country (
|
51
|
-
network varchar(18),
|
52
|
-
geoname_id char(10),
|
53
|
-
registered_country_geoname_id char(30),
|
54
|
-
represented_country_geoname_id char(30),
|
55
|
-
is_anonymous_proxy char(30),
|
56
|
-
is_satellite_provider char(30)
|
51
|
+
network varchar(18) NOT NULL,
|
52
|
+
geoname_id char(10) NOT NULL,
|
53
|
+
registered_country_geoname_id char(30) NOT NULL,
|
54
|
+
represented_country_geoname_id char(30) NOT NULL,
|
55
|
+
is_anonymous_proxy char(30) NOT NULL,
|
56
|
+
is_satellite_provider char(30) NOT NULL
|
57
57
|
);
|
58
58
|
|
59
59
|
CREATE INDEX index_ipv4_network ON "threatmonitor".geoipdata_ipv4blocks_country(network);
|
@@ -62,12 +62,12 @@ COPY 169357 OFFSET 2 RECORDS INTO "threatmonitor".geoipdata_ipv4blocks_country F
|
|
62
62
|
|
63
63
|
DROP TABLE "threatmonitor".geoipdata_locations_country;
|
64
64
|
CREATE TABLE "threatmonitor".geoipdata_locations_country (
|
65
|
-
geoname_id char(10),
|
66
|
-
locale_code char(2),
|
67
|
-
continent_code char(2),
|
68
|
-
continent_name char(15),
|
69
|
-
country_iso_code char(2),
|
70
|
-
country_name char(50)
|
65
|
+
geoname_id char(10) NOT NULL,
|
66
|
+
locale_code char(2) NOT NULL,
|
67
|
+
continent_code char(2) NOT NULL,
|
68
|
+
continent_name char(15) NOT NULL,
|
69
|
+
country_iso_code char(2) NOT NULL,
|
70
|
+
country_name char(50) NOT NULL
|
71
71
|
-- FOREIGN KEY (geoname_id) REFERENCES "geoipdata".geoipdata_ipv4blocks_country (index_geoname_id)
|
72
72
|
);
|
73
73
|
|
@@ -0,0 +1,15 @@
|
|
1
|
+
|
2
|
+
DROP TABLE "threatmonitor".http_traffic_json;
|
3
|
+
CREATE TABLE "threatmonitor".http_traffic_json (
|
4
|
+
id INT GENERATED ALWAYS AS
|
5
|
+
IDENTITY (
|
6
|
+
START WITH 0 INCREMENT BY 1
|
7
|
+
NO MINVALUE NO MAXVALUE
|
8
|
+
CACHE 2 CYCLE
|
9
|
+
) primary key,
|
10
|
+
guid char(36),
|
11
|
+
recv_date date,
|
12
|
+
recv_time time,
|
13
|
+
json_data JSON
|
14
|
+
);
|
15
|
+
|
data/thm-authentication.rb
CHANGED
@@ -2,8 +2,9 @@
|
|
2
2
|
#
|
3
3
|
# Author: Brian Hood
|
4
4
|
#
|
5
|
+
# Email: <brianh6854@googlemail.com>
|
5
6
|
# Description: Threatmonitor User Administration
|
6
|
-
#
|
7
|
+
#
|
7
8
|
# Extends the functionality of the Thm module adding Authorization
|
8
9
|
# Adding Authentication to the Privileges model
|
9
10
|
#
|
@@ -24,7 +25,7 @@ module Thm::Authorization
|
|
24
25
|
|
25
26
|
def initialize
|
26
27
|
super
|
27
|
-
@debug =
|
28
|
+
@debug = true
|
28
29
|
end
|
29
30
|
|
30
31
|
def login(username, password)
|
data/thm-authorization.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
########################################################################
|
2
2
|
#
|
3
3
|
# Author: Brian Hood
|
4
|
-
#
|
4
|
+
# Email: <brianh6854@googlemail.com>
|
5
5
|
# Description: Threatmonitor Authorization
|
6
6
|
#
|
7
7
|
# Extends the functionality of the Thm module adding Authorization
|
@@ -21,7 +21,7 @@ module Thm::Authorization
|
|
21
21
|
|
22
22
|
def initialize
|
23
23
|
super
|
24
|
-
@debug =
|
24
|
+
@debug = true
|
25
25
|
end
|
26
26
|
|
27
27
|
def setup_privileges(name, obj)
|
data/thm-privileges.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
########################################################################
|
2
2
|
#
|
3
3
|
# Author: Brian Hood
|
4
|
-
#
|
4
|
+
# Email: <brianh6854@googlemail.com>
|
5
5
|
# Description: Threatmonitor User Administration
|
6
6
|
#
|
7
7
|
# Extends the functionality of the Thm module adding Authorization
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: thm
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- puppetpies
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-07
|
11
|
+
date: 2015-09-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -164,6 +164,20 @@ dependencies:
|
|
164
164
|
- - "~>"
|
165
165
|
- !ruby/object:Gem::Version
|
166
166
|
version: '3.0'
|
167
|
+
- !ruby/object:Gem::Dependency
|
168
|
+
name: keycounter
|
169
|
+
requirement: !ruby/object:Gem::Requirement
|
170
|
+
requirements:
|
171
|
+
- - "~>"
|
172
|
+
- !ruby/object:Gem::Version
|
173
|
+
version: '0.0'
|
174
|
+
type: :runtime
|
175
|
+
prerelease: false
|
176
|
+
version_requirements: !ruby/object:Gem::Requirement
|
177
|
+
requirements:
|
178
|
+
- - "~>"
|
179
|
+
- !ruby/object:Gem::Version
|
180
|
+
version: '0.0'
|
167
181
|
description: Threatmonitor - Packet Capture / Analysis Suite
|
168
182
|
email: brianh6854@googlemail.com
|
169
183
|
executables:
|
@@ -171,6 +185,7 @@ executables:
|
|
171
185
|
- thm-pcap
|
172
186
|
- thm-producer
|
173
187
|
- thm-session
|
188
|
+
- thm-trafviz
|
174
189
|
- thm-useradmin
|
175
190
|
extensions: []
|
176
191
|
extra_rdoc_files:
|
@@ -183,6 +198,7 @@ files:
|
|
183
198
|
- bin/thm-pcap
|
184
199
|
- bin/thm-producer
|
185
200
|
- bin/thm-session
|
201
|
+
- bin/thm-trafviz
|
186
202
|
- bin/thm-useradmin
|
187
203
|
- config.rb
|
188
204
|
- js/JSXTransformer.js
|
@@ -196,12 +212,15 @@ files:
|
|
196
212
|
- lib/thm/consumer.rb
|
197
213
|
- lib/thm/datalayerlight.rb
|
198
214
|
- lib/thm/dataservices.rb
|
215
|
+
- lib/thm/dataservices/geolocation/geolocation.rb
|
216
|
+
- lib/thm/dataservices/trafviz/trafviz.rb
|
199
217
|
- lib/thm/fileservices.rb
|
200
218
|
- lib/thm/localmachine.rb
|
201
219
|
- lib/thm/producer.rb
|
202
220
|
- lib/thm/version.rb
|
203
221
|
- service_definitions.csv
|
204
222
|
- sql/geoipdata-monetdb.sql
|
223
|
+
- sql/threatmonitor-http.sql
|
205
224
|
- sql/threatmonitor-monetdb.sql
|
206
225
|
- sql/threatmonitor-mysql.sql
|
207
226
|
- stylesheets/screen.css
|