sms-logparser 0.12.4 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ad817588ffc290e40e11563db5d856da73a1c058
-  data.tar.gz: 6a68fbf74e90aed8127c356cb37d2a6e48762172
+  metadata.gz: e157cd8f8e09ea804bd41f02d9a6023338bd1f4a
+  data.tar.gz: dd7afb96c6374b04b3e8bacb1d2051f8ab37e9ba
 SHA512:
-  metadata.gz: af70780b477df0155f6fe10d06ea02327f7879c4cca38f413778345f3318f531a60a84de91b27124c874ee6911f402520953b9dded8c96de31fd775a1e195d3d
-  data.tar.gz: 099385b75fb60fa67bc62d002c84d35007870a727af9b4a534cd3332a5eab6a3b30a95e515c6dc42b3c31b633fe3d87e2b4557e63b967879e34b4d93a3af058e
+  metadata.gz: 1ea303b8e1abf537c7df2b3ebbbaa508bd50ee3e4ea1d5ff2286184067f40f002bbdcb64154247fac046a9580ffc7cbdd4ba1c2e704d022c563077a802c5c77f
+  data.tar.gz: 46b47d8a357cc26ea9362f06b320cbba237b2ed0decc110a37fbf06ac3c5f0c7747a95ac7484b836a2b4bed7f4bc63a010ababd0d98bc3fb6fc16c5214a86d5a

data/lib/sms-logparser/cli.rb

@@ -134,6 +134,28 @@ module SmsLogparser
       say(e.backtrace.join("\n"), :yellow) if options[:debug]
     end
 
+    desc "clean-up [mode]", "Clean up the database - delete unused records"
+    def clean_up(mode = "all")
+      mysql = Mysql.new(options)
+      case mode
+      when "all"
+        mysql.clean_up_parser_table
+        mysql.clean_up_events_table
+        logger.info("All tables have been cleaned up.")
+      when "events"
+        mysql.clean_up_events_table
+        logger.info("Events table has been cleaned up.")
+      when "parser"
+        mysql.clean_up_events_table
+        logger.info("Parser table has been cleaned up.")
+      else
+        logger.warn("Mode '#{mode}' not allowed. Allowed modes are 'events', 'parser' or 'all'.")
+      end
+      SmsLogparser::Loggster.instance.close
+    rescue => e
+      logger.fatal e
+    end
+
     desc "setup", "Create the parser table to track the last logs parsed"
     option :force, type: :boolean, default: false, aliases: %w(-f),
       desc: "Drop an existing table if it exists"

data/lib/sms-logparser/mysql.rb

@@ -3,9 +3,10 @@ module SmsLogparser
 
     def initialize(options)
       @options = options
-      @query_limit = options[:query_limit] || 1000
+      @query_limit = options[:query_limit] || 5000
       @client = client
       @logger = SmsLogparser::Loggster.instance
+      @max_retries = 5
     end
 
     def client
@@ -67,14 +68,29 @@ module SmsLogparser
     end
 
     def write_parse_result(options)
-      client.query(
-        "UPDATE sms_logparser_runs SET\
-          last_event_id = #{options[:last_event_id]},\
-          match_count = #{options[:match_count]},\
-          status = #{options[:status]},\
-          run_time = #{options[:run_time]}\
-        WHERE id = #{options[:id]}"
-      )
+      query = %Q{
+        UPDATE sms_logparser_runs
+        SET last_event_id = #{options[:last_event_id]},
+          match_count = #{options[:match_count]},
+          status = #{options[:status]},
+          run_time = #{options[:run_time]}
+        WHERE id = #{options[:id]};
+      }.gsub(/\s+/, " ").strip
+      retries = 0
+      begin
+        client.query(query)
+      rescue Mysql2::Error => exception
+        @logger.error exception
+        if (retries += 1) > @max_retries
+          @logger.fatal "Can't write parser result to database after #{@max_retries} retries. (#{@query})"
+          raise
+        end
+        sleeptime = @max_retries * retries
+        @logger.info "Retry writing parser results in #{sleeptime} seconds."
+        sleep sleeptime
+        retry 
+      end
+      true
     end
 
     def get_entries(options={})
@@ -90,11 +106,27 @@ module SmsLogparser
 
     def get_last_parse_id
       last_parse = client.query(
-        "SELECT last_event_id FROM sms_logparser_runs ORDER BY id DESC LIMIT 1"
+        "SELECT last_event_id FROM sms_logparser_runs ORDER BY id DESC LIMIT 1;"
       )
       last_parse.first ? last_parse.first['last_event_id'] : 0
     end
 
+    def clean_up_events_table
+      last_parse = client.query(
+        "SELECT last_event_id FROM sms_logparser_runs WHERE status = 0 ORDER BY id DESC LIMIT 1;"
+      )
+      if last_parse.first
+        id = last_parse.first['last_event_id']
+        client.query("DELETE FROM sms_logparser_runs WHERE id < #{id};")
+      end
+      true
+    end
+
+    def clean_up_parser_table(keep_days = 7)
+      time = (Time.now - 3600 * 24 * keep_days.to_i).strftime("%Y-%m-%d %H:%M:%S")
+      client.query("DELETE FROM sms_logparser_runs WHERE run_at < '#{time}';")
+    end
+
     private
 
     def get_max_id(last_id, user_limit = nil)
@@ -118,20 +150,20 @@ module SmsLogparser
 
     def get_last_event_id
       last_event = client.query(
-        "SELECT MAX(ID) as max_id FROM SystemEvents"
+        "SELECT MAX(ID) as max_id FROM SystemEvents;"
       )
       last_event.first ? last_event.first['max_id'] : 0
     end
 
     def parser_table_exists?
       client.query(
-        "SHOW TABLES LIKE 'sms_logparser_runs'"
+        "SHOW TABLES LIKE 'sms_logparser_runs';"
       ).size > 0
     end
 
     def drop_parser_table
       return false unless parser_table_exists?
-      client.query("DROP TABLE sms_logparser_runs")
+      client.query("DROP TABLE sms_logparser_runs;")
       true
     end
 

data/lib/sms-logparser/parser.rb

@@ -40,7 +40,7 @@ module SmsLogparser
     # for mobile browser detection
     def self.get_traffic_type(user_agent)
       case user_agent
-      when /.*(iTunes).*/
+      when /.*(iTunes).*/i
         'TRAFFIC_PODCAST'
       when /.*(Mobi|IEMobile|Mobile Safari|iPhone|iPod|iPad|Android|BlackBerry|Opera Mini).*/
         'TRAFFIC_MOBILE'

data/lib/sms-logparser/version.rb

@@ -1,3 +1,3 @@
 module SmsLogparser
-  VERSION = "0.12.4"
+  VERSION = "0.13.0"
 end

data/spec/syslog_events.sql

@@ -96,5 +96,5 @@ INSERT INTO Syslog.SystemEvents (FromHost, Message) VALUES ("pcache", '- - [25/F
 INSERT INTO Syslog.SystemEvents (FromHost, Message) VALUES ("pcache", '- - [25/Feb/2014:17:28:57 +0100] \"GET /content/2/719/54986/simvid_1.f4v HTTP/1.1\" 200 6741309 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:27.0) Gecko/20100101 Firefox/27.0\"');
 INSERT INTO Syslog.SystemEvents (FromHost, Message) VALUES ("pcache", '- - [25/Feb/2014:17:28:57 +0100] \"GET /content/2/719/54986/simvid_1.f4v HTTP/1.1\" 200 6741309 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:27.0) Gecko/20100101 Firefox/27.0\"');
 INSERT INTO Syslog.SystemEvents (FromHost, Message) VALUES ("pcache", '- - [25/Feb/2014:17:28:57 +0100] \"GET /content/2/719/54986/simvid_1.f4v HTTP/1.1\" 200 6741309 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:27.0) Gecko/20100101 Firefox/27.0\"');
-INSERT INTO Syslog.SystemEvents (FromHost, Message) VALUES ("pcache", '- - [25/Feb/2014:17:28:57 +0100] \"GET /content/2/719/54986/simvid_1.f4v HTTP/1.1\" 200 6741309 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:27.0) Gecko/20100101 Firefox/27.0\"');
-INSERT INTO Syslog.SystemEvents (FromHost, Message) VALUES ("pcache", '- - [25/Feb/2014:17:28:57 +0100] \"GET /content/2/719/54986/simvid_1.f4v HTTP/1.1\" 200 6741309 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:27.0) Gecko/20100101 Firefox/27.0\"');
+INSERT INTO Syslog.SystemEvents (FromHost, Message) VALUES ("pcache", '- - [25/Feb/2014:17:28:57 +0100] \"GET /content/2/719/54986/dadad.mp3 HTTP/1.1\" 200 6741309 \"-\" \"iTunes/9.0.3 (Macintosh; U; Intel Mac OS X 10_6_2; en-ca)\"');
+INSERT INTO Syslog.SystemEvents (FromHost, Message) VALUES ("pcache", '- - [25/Feb/2014:17:28:57 +0100] \"GET /content/2/719/54986/bla.mp4 HTTP/1.1\" 200 6741309 \"-\" \"itunes/9.0.2 (Macintosh; Intel Mac OS X 10.4.11) AppleWebKit/531.21.8\"');

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sms-logparser
 version: !ruby/object:Gem::Version
-  version: 0.12.4
+  version: 0.13.0
 platform: ruby
 authors:
 - niwo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-16 00:00:00.000000000 Z
+date: 2014-04-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler