thm 0.4.5 → 0.5.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,9 +3,9 @@ module Thm #:nodoc:
3
3
  module VERSION #:nodoc:
4
4
 
5
5
  MAJOR = 0
6
- MINOR = 4
7
- TINY = 5
8
- CODENAME = "The Isnis"
6
+ MINOR = 5
7
+ TINY = 7
8
+ CODENAME = "World on fire !"
9
9
 
10
10
  STRING = [MAJOR, MINOR, TINY].join('.')
11
11
 
@@ -24,7 +24,7 @@ CREATE TABLE "threatmonitor".geoipdata_ipv4blocks_city (
24
24
 
25
25
  CREATE INDEX cindex_ipv4_network ON "threatmonitor".geoipdata_ipv4blocks_city(network);
26
26
  CREATE INDEX cindex_ipv4_geoname_id ON "threatmonitor".geoipdata_ipv4blocks_city(geoname_id);
27
- COPY 2519918 OFFSET 2 RECORDS INTO "threatmonitor".geoipdata_ipv4blocks_city FROM '/data2/MaxMind/GeoLite2-City-CSV_20150602/GeoLite2-City-Blocks-IPv4.csv' USING DELIMITERS ',', '\n', '';
27
+ COPY 3030997 OFFSET 2 RECORDS INTO "threatmonitor".geoipdata_ipv4blocks_city FROM '/home/brian/Downloads/GeoLite2-City-CSV_20151006/GeoLite2-City-Blocks-IPv4.csv' USING DELIMITERS ',', '\n', '';
28
28
 
29
29
 
30
30
  DROP TABLE "threatmonitor".geoipdata_locations_city;
@@ -45,7 +45,7 @@ CREATE TABLE "threatmonitor".geoipdata_locations_city (
45
45
  );
46
46
 
47
47
  CREATE INDEX cindex_country_geoname_id ON "threatmonitor".geoipdata_locations_city(geoname_id);
48
- COPY 80006 OFFSET 2 RECORDS INTO "threatmonitor".geoipdata_locations_city FROM '/data2/MaxMind/GeoLite2-City-CSV_20150602/GeoLite2-City-Locations-en.csv' USING DELIMITERS ',', '\n', '';
48
+ COPY 91509 OFFSET 2 RECORDS INTO "threatmonitor".geoipdata_locations_city FROM '/home/brian/Downloads/GeoLite2-City-CSV_20151006/GeoLite2-City-Locations-en.csv' USING DELIMITERS ',', '\n', '';
49
49
 
50
50
 
51
51
  DROP TABLE "threatmonitor".geoipdata_ipv4blocks_country;
@@ -60,7 +60,7 @@ CREATE TABLE "threatmonitor".geoipdata_ipv4blocks_country (
60
60
 
61
61
  CREATE INDEX index_ipv4_network ON "threatmonitor".geoipdata_ipv4blocks_country(network);
62
62
  CREATE INDEX index_ipv4_geoname_id ON "threatmonitor".geoipdata_ipv4blocks_country(geoname_id);
63
- COPY 169357 OFFSET 2 RECORDS INTO "threatmonitor".geoipdata_ipv4blocks_country FROM '/data2/MaxMind/GeoLite2-Country-CSV_20150602/GeoLite2-Country-Blocks-IPv4.csv' USING DELIMITERS ',', '\n', '';
63
+ COPY 178589 OFFSET 2 RECORDS INTO "threatmonitor".geoipdata_ipv4blocks_country FROM '/home/brian/Downloads/GeoLite2-Country-CSV_20151006/GeoLite2-Country-Blocks-IPv4.csv' USING DELIMITERS ',', '\n', '';
64
64
 
65
65
  DROP TABLE "threatmonitor".geoipdata_locations_country;
66
66
  CREATE TABLE "threatmonitor".geoipdata_locations_country (
@@ -74,7 +74,7 @@ CREATE TABLE "threatmonitor".geoipdata_locations_country (
74
74
  );
75
75
 
76
76
  CREATE INDEX index_country_geoname_id ON "threatmonitor".geoipdata_locations_country(geoname_id);
77
- COPY 250 OFFSET 2 RECORDS INTO "threatmonitor".geoipdata_locations_country FROM '/data2/MaxMind/GeoLite2-Country-CSV_20150602/GeoLite2-Country-Locations-en.csv' USING DELIMITERS ',', '\n', '';
77
+ COPY 250 OFFSET 2 RECORDS INTO "threatmonitor".geoipdata_locations_country FROM '/home/brian/Downloads/GeoLite2-Country-CSV_20151006/GeoLite2-Country-Locations-en.csv' USING DELIMITERS ',', '\n', '';
78
78
 
79
79
  plan SELECT continent_name, country_name
80
80
  FROM "threatmonitor".geoipdata_ipv4blocks_country a
@@ -32,25 +32,39 @@ id INT GENERATED ALWAYS AS
32
32
  guid CHAR(36) NOT NULL
33
33
  );
34
34
 
35
+ CREATE INDEX index_traffic_ua_id ON "threatmonitor".http_traffic_ua(id);
36
+ CREATE INDEX index_traffic_ua_guid ON "threatmonitor".http_traffic_ua(guid);
37
+
38
+ DROP FUNCTION JSON_SQUASH;
35
39
  CREATE FUNCTION JSON_SQUASH(name string)
36
40
  RETURNS string
37
41
  BEGIN
38
- RETURN REPLACE(REPLACE(REPLACE(name, '[\"', ''), '\"]', ''), '"', '');
42
+ DECLARE res STRING;
43
+ SET res = REPLACE(REPLACE(REPLACE(name, '[\"', ''), '\"]', ''), '"', '');
44
+ IF (res = '[]') THEN
45
+ SET res = REPLACE(res, '[]', '<no data>');
46
+ END IF;
47
+ RETURN res;
39
48
  END;
40
49
 
41
- /*
42
- PLAN SELECT
43
- JSON_SQUASH(host) AS host,
44
- JSON_SQUASH(acceptlanguage) as acceptlanguage,
45
- JSON_SQUASH(acceptencoding) as acceptencoding,
46
- JSON_SQUASH(referer) as referer,
50
+ DROP VIEW traffic_view_5mins;
51
+ CREATE VIEW traffic_view_5mins AS (SELECT
52
+ recv_date,
53
+ recv_time,
54
+ JSON_SQUASH(hostname) AS hostname,
55
+ JSON_SQUASH(url) AS url,
56
+ JSON_SQUASH(acceptlanguage) AS acceptlanguage,
57
+ JSON_SQUASH(referer) AS referer,
47
58
  family,
48
59
  major,
49
60
  minor,
50
61
  os
51
62
  FROM
52
- (SELECT
53
- json.filter(json_data, '$.http.host') AS host,
63
+ (SELECT
64
+ a.recv_date AS recv_date,
65
+ a.recv_time AS recv_time,
66
+ json.filter(json_data, '$.http.host') AS hostname,
67
+ json.filter(json_data, '$.http.url') AS url,
54
68
  json.filter(json_data, '$.http.acceptlanguage') AS acceptlanguage,
55
69
  json.filter(json_data, '$.http.acceptencoding') AS acceptencoding,
56
70
  json.filter(json_data, '$.http.referer') AS referer,
@@ -59,8 +73,91 @@ b.major,
59
73
  b.minor,
60
74
  b.os
61
75
  FROM http_traffic_json a JOIN http_traffic_ua b
62
- ON (a.guid = b.guid)) AS origin WHERE referer ILIKE '%http://%' LIMIT 30;
63
- */
76
+ ON (a.guid = b.guid)) AS origin WHERE recv_time BETWEEN CURTIME() - 300 AND CURTIME());
77
+
78
+ DROP VIEW traffic_view_15mins;
79
+ CREATE VIEW traffic_view_15mins AS (SELECT
80
+ recv_date,
81
+ recv_time,
82
+ JSON_SQUASH(hostname) AS hostname,
83
+ JSON_SQUASH(url) AS url,
84
+ JSON_SQUASH(acceptlanguage) AS acceptlanguage,
85
+ JSON_SQUASH(referer) AS referer,
86
+ family,
87
+ major,
88
+ minor,
89
+ os
90
+ FROM
91
+ (SELECT
92
+ a.recv_date AS recv_date,
93
+ a.recv_time AS recv_time,
94
+ json.filter(json_data, '$.http.host') AS hostname,
95
+ json.filter(json_data, '$.http.url') AS url,
96
+ json.filter(json_data, '$.http.acceptlanguage') AS acceptlanguage,
97
+ json.filter(json_data, '$.http.acceptencoding') AS acceptencoding,
98
+ json.filter(json_data, '$.http.referer') AS referer,
99
+ b.family,
100
+ b.major,
101
+ b.minor,
102
+ b.os
103
+ FROM http_traffic_json a JOIN http_traffic_ua b
104
+ ON (a.guid = b.guid)) AS origin WHERE recv_time BETWEEN CURTIME() - 900 AND CURTIME());
105
+
106
+ DROP VIEW traffic_view_30mins;
107
+ CREATE VIEW traffic_view_30mins AS (SELECT
108
+ recv_date,
109
+ recv_time,
110
+ JSON_SQUASH(hostname) AS hostname,
111
+ JSON_SQUASH(url) AS url,
112
+ JSON_SQUASH(acceptlanguage) AS acceptlanguage,
113
+ JSON_SQUASH(referer) AS referer,
114
+ family,
115
+ major,
116
+ minor,
117
+ os
118
+ FROM
119
+ (SELECT
120
+ a.recv_date AS recv_date,
121
+ a.recv_time AS recv_time,
122
+ json.filter(json_data, '$.http.host') AS hostname,
123
+ json.filter(json_data, '$.http.url') AS url,
124
+ json.filter(json_data, '$.http.acceptlanguage') AS acceptlanguage,
125
+ json.filter(json_data, '$.http.acceptencoding') AS acceptencoding,
126
+ json.filter(json_data, '$.http.referer') AS referer,
127
+ b.family,
128
+ b.major,
129
+ b.minor,
130
+ b.os
131
+ FROM http_traffic_json a JOIN http_traffic_ua b
132
+ ON (a.guid = b.guid)) AS origin WHERE recv_time BETWEEN CURTIME() - 1800 AND CURTIME());
133
+
134
+ DROP VIEW traffic_view_24hrs;
135
+ CREATE VIEW traffic_view_24hrs AS (SELECT
136
+ recv_date,
137
+ recv_time,
138
+ JSON_SQUASH(hostname) AS hostname,
139
+ JSON_SQUASH(url) AS url,
140
+ JSON_SQUASH(acceptlanguage) AS acceptlanguage,
141
+ JSON_SQUASH(referer) AS referer,
142
+ family,
143
+ major,
144
+ minor,
145
+ os
146
+ FROM
147
+ (SELECT
148
+ a.recv_date AS recv_date,
149
+ a.recv_time AS recv_time,
150
+ json.filter(json_data, '$.http.host') AS hostname,
151
+ json.filter(json_data, '$.http.url') AS url,
152
+ json.filter(json_data, '$.http.acceptlanguage') AS acceptlanguage,
153
+ json.filter(json_data, '$.http.acceptencoding') AS acceptencoding,
154
+ json.filter(json_data, '$.http.referer') AS referer,
155
+ b.family,
156
+ b.major,
157
+ b.minor,
158
+ b.os
159
+ FROM http_traffic_json a JOIN http_traffic_ua b
160
+ ON (a.guid = b.guid)) AS origin WHERE recv_time BETWEEN CURTIME() - 86400 AND CURTIME());
64
161
 
65
162
  /*
66
163
  SELECT MIN(json_data) FROM http_traffic_json
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: thm
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.5
4
+ version: 0.5.7
5
5
  platform: ruby
6
6
  authors:
7
7
  - puppetpies
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-09-14 00:00:00.000000000 Z
11
+ date: 2015-10-21 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -227,7 +227,9 @@ files:
227
227
  - lib/thm/consumer.rb
228
228
  - lib/thm/datalayerlight.rb
229
229
  - lib/thm/dataservices.rb
230
+ - lib/thm/dataservices/external.rb
230
231
  - lib/thm/dataservices/geolocation/geolocation.rb
232
+ - lib/thm/dataservices/safebrowsing_api.rb
231
233
  - lib/thm/dataservices/trafviz/trafviz.rb
232
234
  - lib/thm/fileservices.rb
233
235
  - lib/thm/localmachine.rb