thm 0.4.5 → 0.5.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/Rakefile +2 -0
- data/bin/thm-session +4 -39
- data/bin/thm-trafviz +26 -145
- data/bin/thm-useradmin +2 -3
- data/config.rb +14 -1
- data/lib/thm.rb +140 -2
- data/lib/thm/datalayerlight.rb +61 -17
- data/lib/thm/dataservices.rb +6 -0
- data/lib/thm/dataservices/external.rb +68 -0
- data/lib/thm/dataservices/geolocation/geolocation.rb +46 -19
- data/lib/thm/dataservices/safebrowsing_api.rb +54 -0
- data/lib/thm/dataservices/trafviz/trafviz.rb +164 -27
- data/lib/thm/version.rb +3 -3
- data/sql/geoipdata-monetdb.sql +4 -4
- data/sql/threatmonitor-http.sql +108 -11
- metadata +4 -2
data/lib/thm/version.rb
CHANGED
data/sql/geoipdata-monetdb.sql
CHANGED
@@ -24,7 +24,7 @@ CREATE TABLE "threatmonitor".geoipdata_ipv4blocks_city (
|
|
24
24
|
|
25
25
|
CREATE INDEX cindex_ipv4_network ON "threatmonitor".geoipdata_ipv4blocks_city(network);
|
26
26
|
CREATE INDEX cindex_ipv4_geoname_id ON "threatmonitor".geoipdata_ipv4blocks_city(geoname_id);
|
27
|
-
COPY
|
27
|
+
COPY 3030997 OFFSET 2 RECORDS INTO "threatmonitor".geoipdata_ipv4blocks_city FROM '/home/brian/Downloads/GeoLite2-City-CSV_20151006/GeoLite2-City-Blocks-IPv4.csv' USING DELIMITERS ',', '\n', '';
|
28
28
|
|
29
29
|
|
30
30
|
DROP TABLE "threatmonitor".geoipdata_locations_city;
|
@@ -45,7 +45,7 @@ CREATE TABLE "threatmonitor".geoipdata_locations_city (
|
|
45
45
|
);
|
46
46
|
|
47
47
|
CREATE INDEX cindex_country_geoname_id ON "threatmonitor".geoipdata_locations_city(geoname_id);
|
48
|
-
COPY
|
48
|
+
COPY 91509 OFFSET 2 RECORDS INTO "threatmonitor".geoipdata_locations_city FROM '/home/brian/Downloads/GeoLite2-City-CSV_20151006/GeoLite2-City-Locations-en.csv' USING DELIMITERS ',', '\n', '';
|
49
49
|
|
50
50
|
|
51
51
|
DROP TABLE "threatmonitor".geoipdata_ipv4blocks_country;
|
@@ -60,7 +60,7 @@ CREATE TABLE "threatmonitor".geoipdata_ipv4blocks_country (
|
|
60
60
|
|
61
61
|
CREATE INDEX index_ipv4_network ON "threatmonitor".geoipdata_ipv4blocks_country(network);
|
62
62
|
CREATE INDEX index_ipv4_geoname_id ON "threatmonitor".geoipdata_ipv4blocks_country(geoname_id);
|
63
|
-
COPY
|
63
|
+
COPY 178589 OFFSET 2 RECORDS INTO "threatmonitor".geoipdata_ipv4blocks_country FROM '/home/brian/Downloads/GeoLite2-Country-CSV_20151006/GeoLite2-Country-Blocks-IPv4.csv' USING DELIMITERS ',', '\n', '';
|
64
64
|
|
65
65
|
DROP TABLE "threatmonitor".geoipdata_locations_country;
|
66
66
|
CREATE TABLE "threatmonitor".geoipdata_locations_country (
|
@@ -74,7 +74,7 @@ CREATE TABLE "threatmonitor".geoipdata_locations_country (
|
|
74
74
|
);
|
75
75
|
|
76
76
|
CREATE INDEX index_country_geoname_id ON "threatmonitor".geoipdata_locations_country(geoname_id);
|
77
|
-
COPY 250 OFFSET 2 RECORDS INTO "threatmonitor".geoipdata_locations_country FROM '/
|
77
|
+
COPY 250 OFFSET 2 RECORDS INTO "threatmonitor".geoipdata_locations_country FROM '/home/brian/Downloads/GeoLite2-Country-CSV_20151006/GeoLite2-Country-Locations-en.csv' USING DELIMITERS ',', '\n', '';
|
78
78
|
|
79
79
|
plan SELECT continent_name, country_name
|
80
80
|
FROM "threatmonitor".geoipdata_ipv4blocks_country a
|
data/sql/threatmonitor-http.sql
CHANGED
@@ -32,25 +32,39 @@ id INT GENERATED ALWAYS AS
|
|
32
32
|
guid CHAR(36) NOT NULL
|
33
33
|
);
|
34
34
|
|
35
|
+
CREATE INDEX index_traffic_ua_id ON "threatmonitor".http_traffic_ua(id);
|
36
|
+
CREATE INDEX index_traffic_ua_guid ON "threatmonitor".http_traffic_ua(guid);
|
37
|
+
|
38
|
+
DROP FUNCTION JSON_SQUASH;
|
35
39
|
CREATE FUNCTION JSON_SQUASH(name string)
|
36
40
|
RETURNS string
|
37
41
|
BEGIN
|
38
|
-
|
42
|
+
DECLARE res STRING;
|
43
|
+
SET res = REPLACE(REPLACE(REPLACE(name, '[\"', ''), '\"]', ''), '"', '');
|
44
|
+
IF (res = '[]') THEN
|
45
|
+
SET res = REPLACE(res, '[]', '<no data>');
|
46
|
+
END IF;
|
47
|
+
RETURN res;
|
39
48
|
END;
|
40
49
|
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
JSON_SQUASH(
|
46
|
-
JSON_SQUASH(
|
50
|
+
DROP VIEW traffic_view_5mins;
|
51
|
+
CREATE VIEW traffic_view_5mins AS (SELECT
|
52
|
+
recv_date,
|
53
|
+
recv_time,
|
54
|
+
JSON_SQUASH(hostname) AS hostname,
|
55
|
+
JSON_SQUASH(url) AS url,
|
56
|
+
JSON_SQUASH(acceptlanguage) AS acceptlanguage,
|
57
|
+
JSON_SQUASH(referer) AS referer,
|
47
58
|
family,
|
48
59
|
major,
|
49
60
|
minor,
|
50
61
|
os
|
51
62
|
FROM
|
52
|
-
(SELECT
|
53
|
-
|
63
|
+
(SELECT
|
64
|
+
a.recv_date AS recv_date,
|
65
|
+
a.recv_time AS recv_time,
|
66
|
+
json.filter(json_data, '$.http.host') AS hostname,
|
67
|
+
json.filter(json_data, '$.http.url') AS url,
|
54
68
|
json.filter(json_data, '$.http.acceptlanguage') AS acceptlanguage,
|
55
69
|
json.filter(json_data, '$.http.acceptencoding') AS acceptencoding,
|
56
70
|
json.filter(json_data, '$.http.referer') AS referer,
|
@@ -59,8 +73,91 @@ b.major,
|
|
59
73
|
b.minor,
|
60
74
|
b.os
|
61
75
|
FROM http_traffic_json a JOIN http_traffic_ua b
|
62
|
-
ON (a.guid = b.guid)) AS origin WHERE
|
63
|
-
|
76
|
+
ON (a.guid = b.guid)) AS origin WHERE recv_time BETWEEN CURTIME() - 300 AND CURTIME());
|
77
|
+
|
78
|
+
DROP VIEW traffic_view_15mins;
|
79
|
+
CREATE VIEW traffic_view_15mins AS (SELECT
|
80
|
+
recv_date,
|
81
|
+
recv_time,
|
82
|
+
JSON_SQUASH(hostname) AS hostname,
|
83
|
+
JSON_SQUASH(url) AS url,
|
84
|
+
JSON_SQUASH(acceptlanguage) AS acceptlanguage,
|
85
|
+
JSON_SQUASH(referer) AS referer,
|
86
|
+
family,
|
87
|
+
major,
|
88
|
+
minor,
|
89
|
+
os
|
90
|
+
FROM
|
91
|
+
(SELECT
|
92
|
+
a.recv_date AS recv_date,
|
93
|
+
a.recv_time AS recv_time,
|
94
|
+
json.filter(json_data, '$.http.host') AS hostname,
|
95
|
+
json.filter(json_data, '$.http.url') AS url,
|
96
|
+
json.filter(json_data, '$.http.acceptlanguage') AS acceptlanguage,
|
97
|
+
json.filter(json_data, '$.http.acceptencoding') AS acceptencoding,
|
98
|
+
json.filter(json_data, '$.http.referer') AS referer,
|
99
|
+
b.family,
|
100
|
+
b.major,
|
101
|
+
b.minor,
|
102
|
+
b.os
|
103
|
+
FROM http_traffic_json a JOIN http_traffic_ua b
|
104
|
+
ON (a.guid = b.guid)) AS origin WHERE recv_time BETWEEN CURTIME() - 900 AND CURTIME());
|
105
|
+
|
106
|
+
DROP VIEW traffic_view_30mins;
|
107
|
+
CREATE VIEW traffic_view_30mins AS (SELECT
|
108
|
+
recv_date,
|
109
|
+
recv_time,
|
110
|
+
JSON_SQUASH(hostname) AS hostname,
|
111
|
+
JSON_SQUASH(url) AS url,
|
112
|
+
JSON_SQUASH(acceptlanguage) AS acceptlanguage,
|
113
|
+
JSON_SQUASH(referer) AS referer,
|
114
|
+
family,
|
115
|
+
major,
|
116
|
+
minor,
|
117
|
+
os
|
118
|
+
FROM
|
119
|
+
(SELECT
|
120
|
+
a.recv_date AS recv_date,
|
121
|
+
a.recv_time AS recv_time,
|
122
|
+
json.filter(json_data, '$.http.host') AS hostname,
|
123
|
+
json.filter(json_data, '$.http.url') AS url,
|
124
|
+
json.filter(json_data, '$.http.acceptlanguage') AS acceptlanguage,
|
125
|
+
json.filter(json_data, '$.http.acceptencoding') AS acceptencoding,
|
126
|
+
json.filter(json_data, '$.http.referer') AS referer,
|
127
|
+
b.family,
|
128
|
+
b.major,
|
129
|
+
b.minor,
|
130
|
+
b.os
|
131
|
+
FROM http_traffic_json a JOIN http_traffic_ua b
|
132
|
+
ON (a.guid = b.guid)) AS origin WHERE recv_time BETWEEN CURTIME() - 1800 AND CURTIME());
|
133
|
+
|
134
|
+
DROP VIEW traffic_view_24hrs;
|
135
|
+
CREATE VIEW traffic_view_24hrs AS (SELECT
|
136
|
+
recv_date,
|
137
|
+
recv_time,
|
138
|
+
JSON_SQUASH(hostname) AS hostname,
|
139
|
+
JSON_SQUASH(url) AS url,
|
140
|
+
JSON_SQUASH(acceptlanguage) AS acceptlanguage,
|
141
|
+
JSON_SQUASH(referer) AS referer,
|
142
|
+
family,
|
143
|
+
major,
|
144
|
+
minor,
|
145
|
+
os
|
146
|
+
FROM
|
147
|
+
(SELECT
|
148
|
+
a.recv_date AS recv_date,
|
149
|
+
a.recv_time AS recv_time,
|
150
|
+
json.filter(json_data, '$.http.host') AS hostname,
|
151
|
+
json.filter(json_data, '$.http.url') AS url,
|
152
|
+
json.filter(json_data, '$.http.acceptlanguage') AS acceptlanguage,
|
153
|
+
json.filter(json_data, '$.http.acceptencoding') AS acceptencoding,
|
154
|
+
json.filter(json_data, '$.http.referer') AS referer,
|
155
|
+
b.family,
|
156
|
+
b.major,
|
157
|
+
b.minor,
|
158
|
+
b.os
|
159
|
+
FROM http_traffic_json a JOIN http_traffic_ua b
|
160
|
+
ON (a.guid = b.guid)) AS origin WHERE recv_time BETWEEN CURTIME() - 86400 AND CURTIME());
|
64
161
|
|
65
162
|
/*
|
66
163
|
SELECT MIN(json_data) FROM http_traffic_json
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: thm
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- puppetpies
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-10-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -227,7 +227,9 @@ files:
|
|
227
227
|
- lib/thm/consumer.rb
|
228
228
|
- lib/thm/datalayerlight.rb
|
229
229
|
- lib/thm/dataservices.rb
|
230
|
+
- lib/thm/dataservices/external.rb
|
230
231
|
- lib/thm/dataservices/geolocation/geolocation.rb
|
232
|
+
- lib/thm/dataservices/safebrowsing_api.rb
|
231
233
|
- lib/thm/dataservices/trafviz/trafviz.rb
|
232
234
|
- lib/thm/fileservices.rb
|
233
235
|
- lib/thm/localmachine.rb
|