thm 0.4.5 → 0.5.7

data/lib/thm/version.rb

@@ -3,9 +3,9 @@ module Thm #:nodoc:
   module VERSION #:nodoc:
 
     MAJOR = 0
-    MINOR = 4
-    TINY = 5
-    CODENAME = "The Isnis"
+    MINOR = 5
+    TINY = 7
+    CODENAME = "World on fire !"
 
     STRING = [MAJOR, MINOR, TINY].join('.')
 

data/sql/geoipdata-monetdb.sql

@@ -24,7 +24,7 @@ CREATE TABLE "threatmonitor".geoipdata_ipv4blocks_city (
 
 CREATE INDEX cindex_ipv4_network ON "threatmonitor".geoipdata_ipv4blocks_city(network);
 CREATE INDEX cindex_ipv4_geoname_id ON "threatmonitor".geoipdata_ipv4blocks_city(geoname_id);
-COPY 2519918 OFFSET 2 RECORDS INTO "threatmonitor".geoipdata_ipv4blocks_city FROM '/data2/MaxMind/GeoLite2-City-CSV_20150602/GeoLite2-City-Blocks-IPv4.csv' USING DELIMITERS ',', '\n', '';
+COPY 3030997 OFFSET 2 RECORDS INTO "threatmonitor".geoipdata_ipv4blocks_city FROM '/home/brian/Downloads/GeoLite2-City-CSV_20151006/GeoLite2-City-Blocks-IPv4.csv' USING DELIMITERS ',', '\n', '';
 
 
 DROP TABLE "threatmonitor".geoipdata_locations_city;
@@ -45,7 +45,7 @@ CREATE TABLE "threatmonitor".geoipdata_locations_city (
 );
 
 CREATE INDEX cindex_country_geoname_id ON "threatmonitor".geoipdata_locations_city(geoname_id);
-COPY 80006 OFFSET 2 RECORDS INTO "threatmonitor".geoipdata_locations_city FROM '/data2/MaxMind/GeoLite2-City-CSV_20150602/GeoLite2-City-Locations-en.csv' USING DELIMITERS ',', '\n', '';
+COPY 91509 OFFSET 2 RECORDS INTO "threatmonitor".geoipdata_locations_city FROM '/home/brian/Downloads/GeoLite2-City-CSV_20151006/GeoLite2-City-Locations-en.csv' USING DELIMITERS ',', '\n', '';
 
 
 DROP TABLE "threatmonitor".geoipdata_ipv4blocks_country;
@@ -60,7 +60,7 @@ CREATE TABLE "threatmonitor".geoipdata_ipv4blocks_country (
 
 CREATE INDEX index_ipv4_network ON "threatmonitor".geoipdata_ipv4blocks_country(network);
 CREATE INDEX index_ipv4_geoname_id ON "threatmonitor".geoipdata_ipv4blocks_country(geoname_id);
-COPY 169357 OFFSET 2 RECORDS INTO "threatmonitor".geoipdata_ipv4blocks_country FROM '/data2/MaxMind/GeoLite2-Country-CSV_20150602/GeoLite2-Country-Blocks-IPv4.csv' USING DELIMITERS ',', '\n', '';
+COPY 178589 OFFSET 2 RECORDS INTO "threatmonitor".geoipdata_ipv4blocks_country FROM '/home/brian/Downloads/GeoLite2-Country-CSV_20151006/GeoLite2-Country-Blocks-IPv4.csv' USING DELIMITERS ',', '\n', '';
 
 DROP TABLE "threatmonitor".geoipdata_locations_country;
 CREATE TABLE "threatmonitor".geoipdata_locations_country (
@@ -74,7 +74,7 @@ CREATE TABLE "threatmonitor".geoipdata_locations_country (
 );
 
 CREATE INDEX index_country_geoname_id ON "threatmonitor".geoipdata_locations_country(geoname_id);
-COPY 250 OFFSET 2 RECORDS INTO "threatmonitor".geoipdata_locations_country FROM '/data2/MaxMind/GeoLite2-Country-CSV_20150602/GeoLite2-Country-Locations-en.csv' USING DELIMITERS ',', '\n', '';
+COPY 250 OFFSET 2 RECORDS INTO "threatmonitor".geoipdata_locations_country FROM '/home/brian/Downloads/GeoLite2-Country-CSV_20151006/GeoLite2-Country-Locations-en.csv' USING DELIMITERS ',', '\n', '';
 
 plan SELECT continent_name, country_name 
 FROM "threatmonitor".geoipdata_ipv4blocks_country a 

data/sql/threatmonitor-http.sql

@@ -32,25 +32,39 @@ id INT GENERATED ALWAYS AS
   guid CHAR(36) NOT NULL
 );
 
+CREATE INDEX index_traffic_ua_id ON "threatmonitor".http_traffic_ua(id);
+CREATE INDEX index_traffic_ua_guid ON "threatmonitor".http_traffic_ua(guid);
+
+DROP FUNCTION JSON_SQUASH;
 CREATE FUNCTION JSON_SQUASH(name string)
 RETURNS string
 BEGIN
-  RETURN REPLACE(REPLACE(REPLACE(name, '[\"', ''), '\"]', ''), '"', '');
+  DECLARE res STRING;
+  SET res = REPLACE(REPLACE(REPLACE(name, '[\"', ''), '\"]', ''), '"', '');
+  IF (res = '[]') THEN
+    SET res = REPLACE(res, '[]', '<no data>');
+  END IF;
+  RETURN res;
 END;
 
-/*
-PLAN SELECT 
-JSON_SQUASH(host) AS host, 
-JSON_SQUASH(acceptlanguage) as acceptlanguage,
-JSON_SQUASH(acceptencoding) as acceptencoding,
-JSON_SQUASH(referer) as referer,
+DROP VIEW traffic_view_5mins;
+CREATE VIEW traffic_view_5mins AS (SELECT
+recv_date,
+recv_time,
+JSON_SQUASH(hostname) AS hostname,
+JSON_SQUASH(url) AS url,
+JSON_SQUASH(acceptlanguage) AS acceptlanguage,
+JSON_SQUASH(referer) AS referer,
 family,
 major,
 minor,
 os
 FROM 
-(SELECT 
-json.filter(json_data, '$.http.host') AS host,
+(SELECT
+a.recv_date AS recv_date,
+a.recv_time AS recv_time,
+json.filter(json_data, '$.http.host') AS hostname,
+json.filter(json_data, '$.http.url') AS url,
 json.filter(json_data, '$.http.acceptlanguage') AS acceptlanguage,
 json.filter(json_data, '$.http.acceptencoding') AS acceptencoding,
 json.filter(json_data, '$.http.referer') AS referer,
@@ -59,8 +73,91 @@ b.major,
 b.minor,
 b.os
 FROM http_traffic_json a JOIN http_traffic_ua b 
-ON (a.guid = b.guid)) AS origin WHERE referer ILIKE '%http://%' LIMIT 30;
-*/
+ON (a.guid = b.guid)) AS origin WHERE recv_time BETWEEN CURTIME() - 300 AND CURTIME());
+
+DROP VIEW traffic_view_15mins;
+CREATE VIEW traffic_view_15mins AS (SELECT
+recv_date,
+recv_time,
+JSON_SQUASH(hostname) AS hostname,
+JSON_SQUASH(url) AS url,
+JSON_SQUASH(acceptlanguage) AS acceptlanguage,
+JSON_SQUASH(referer) AS referer,
+family,
+major,
+minor,
+os
+FROM 
+(SELECT
+a.recv_date AS recv_date,
+a.recv_time AS recv_time,
+json.filter(json_data, '$.http.host') AS hostname,
+json.filter(json_data, '$.http.url') AS url,
+json.filter(json_data, '$.http.acceptlanguage') AS acceptlanguage,
+json.filter(json_data, '$.http.acceptencoding') AS acceptencoding,
+json.filter(json_data, '$.http.referer') AS referer,
+b.family,
+b.major,
+b.minor,
+b.os
+FROM http_traffic_json a JOIN http_traffic_ua b 
+ON (a.guid = b.guid)) AS origin WHERE recv_time BETWEEN CURTIME() - 900 AND CURTIME());
+
+DROP VIEW traffic_view_30mins;
+CREATE VIEW traffic_view_30mins AS (SELECT
+recv_date,
+recv_time,
+JSON_SQUASH(hostname) AS hostname,
+JSON_SQUASH(url) AS url,
+JSON_SQUASH(acceptlanguage) AS acceptlanguage,
+JSON_SQUASH(referer) AS referer,
+family,
+major,
+minor,
+os
+FROM 
+(SELECT
+a.recv_date AS recv_date,
+a.recv_time AS recv_time,
+json.filter(json_data, '$.http.host') AS hostname,
+json.filter(json_data, '$.http.url') AS url,
+json.filter(json_data, '$.http.acceptlanguage') AS acceptlanguage,
+json.filter(json_data, '$.http.acceptencoding') AS acceptencoding,
+json.filter(json_data, '$.http.referer') AS referer,
+b.family,
+b.major,
+b.minor,
+b.os
+FROM http_traffic_json a JOIN http_traffic_ua b 
+ON (a.guid = b.guid)) AS origin WHERE recv_time BETWEEN CURTIME() - 1800 AND CURTIME());
+
+DROP VIEW traffic_view_24hrs;
+CREATE VIEW traffic_view_24hrs AS (SELECT
+recv_date,
+recv_time,
+JSON_SQUASH(hostname) AS hostname,
+JSON_SQUASH(url) AS url,
+JSON_SQUASH(acceptlanguage) AS acceptlanguage,
+JSON_SQUASH(referer) AS referer,
+family,
+major,
+minor,
+os
+FROM 
+(SELECT
+a.recv_date AS recv_date,
+a.recv_time AS recv_time,
+json.filter(json_data, '$.http.host') AS hostname,
+json.filter(json_data, '$.http.url') AS url,
+json.filter(json_data, '$.http.acceptlanguage') AS acceptlanguage,
+json.filter(json_data, '$.http.acceptencoding') AS acceptencoding,
+json.filter(json_data, '$.http.referer') AS referer,
+b.family,
+b.major,
+b.minor,
+b.os
+FROM http_traffic_json a JOIN http_traffic_ua b 
+ON (a.guid = b.guid)) AS origin WHERE recv_time BETWEEN CURTIME() - 86400 AND CURTIME());
 
 /*
 SELECT MIN(json_data) FROM http_traffic_json

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: thm
 version: !ruby/object:Gem::Version
-  version: 0.4.5
+  version: 0.5.7
 platform: ruby
 authors:
 - puppetpies
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-14 00:00:00.000000000 Z
+date: 2015-10-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -227,7 +227,9 @@ files:
 - lib/thm/consumer.rb
 - lib/thm/datalayerlight.rb
 - lib/thm/dataservices.rb
+- lib/thm/dataservices/external.rb
 - lib/thm/dataservices/geolocation/geolocation.rb
+- lib/thm/dataservices/safebrowsing_api.rb
 - lib/thm/dataservices/trafviz/trafviz.rb
 - lib/thm/fileservices.rb
 - lib/thm/localmachine.rb