sms-logparser 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9947d072a5d20d349783fd951fdd84a0e7ae399d
-  data.tar.gz: df012488b0a89171d4266a01c05a98e20f7e6e35
+  metadata.gz: ac7a2c7bd9b90b797a3ee9a0200117c47139f54b
+  data.tar.gz: 88b67958f0fd4b641414a028b329be5bbd50e0da
 SHA512:
-  metadata.gz: fa94858261fc273a87d3ebba39120795631c477979d0fef1b770d1092b17c50c95387a662b8b7743c841f34a06e687bf3e0ce0181c630e07a05549145ebd5685
-  data.tar.gz: f97dd70501d1b23919b52e79159b179eb1d91da5bcc192738ee0d0419a581543307c599b627bcdfc6f48f4464e2079add11e7492be4d3e5a77f954388d739822
+  metadata.gz: 8e2fca495d375a1edf97a182552bdb38b15001929dacc3ae777c70a8a33ec377ced9c481d6b636acdb75aab90e6fe3384ee981b92841835fc2c78ad9b369b665
+  data.tar.gz: 40f73533a55aa2deb3e2f76a1fda30037e375e179cee86401df26b7551c0027b12c289dae30043419b691e117b507eaab8464dd44f33061b601b1e0be1721522

data/lib/sms-logparser/api.rb

@@ -8,20 +8,18 @@ module SmsLogparser
     end
 
     def connection
-      @connection ||= new_connection
-    end
-
-    def new_connection
-      conn = Faraday.new(url: @base_url) do |faraday|
-        faraday.request :url_encoded
-        faraday.response :logger if @options[:debug]
-        faraday.adapter  Faraday.default_adapter
-      end
-      conn.headers[:user_agent] = "sms-logparser v#{SmsLogparser::VERSION}"
-      if @options[:api_key]
-        conn.headers['X-simplex-api-key'] = @options[:api_key]
+      unless @connection
+        @connection = Faraday.new(url: @base_url) do |faraday|
+          faraday.request :url_encoded
+          faraday.response :logger if @options[:debug]
+          faraday.adapter :net_http_persistent
+        end
+        @connection.headers[:user_agent] = "sms-logparser v#{SmsLogparser::VERSION}"
+        if @options[:api_key]
+          @connection.headers['X-simplex-api-key'] = @options[:api_key]
+        end
       end
-      conn
+      @connection
     end
 
     def send(data)

data/lib/sms-logparser/cli.rb

@@ -55,47 +55,52 @@ module SmsLogparser
       type: :boolean,
       default: false
     def parse
+      say "Starting the parser...", :green
       start_time = Time.now
       count = 0
       begin
         mysql = Mysql.new(options)
-        entries = mysql.get_entries(limit: options[:limit])
         api = Api.new(options)
         last_id = mysql.get_last_parse_id
         status = STATUS[:ok]
-        entries.each do |entry|
-          if Parser.match?(entry['Message'])
-            data = Parser.extract_data_from_msg(entry['Message'])
-            begin
-              uris = api.send(data)
-            rescue => e
-              say "Error: #{e.message}", :red
-              say "Aborting parser run...", :red
-              status = STATUS[:api_error]
-              break
-            end
-            last_id = entry['ID']
-            count += 1
-            if options[:verbose]
-              verbose_parser_output(data, uris, entry)
+        begin
+          mysql.get_entries(last_id: last_id, limit: options[:limit]) do |entries|
+            entries.each do |entry| 
+              if data = Parser.extract_data_from_msg(entry['Message'])
+                uris = api.send(data)
+                last_id = entry['ID']
+                count += 1
+                if options[:verbose]
+                  verbose_parser_output(data, uris, entry)
+                end
+              end
             end
           end
-        end
-        unless options[:simulate]
-          mysql.write_parse_result(last_id, count, status)
+        rescue => e
+          say "Error: #{e.message}", :red
+          say "Aborting parser run...", :red
+          status = STATUS[:api_error]
+        ensure
+          mysql.write_parse_result(
+            last_event_id: last_id, 
+            match_count: count,
+            status: status,
+            run_at: start_time,
+            run_time: (Time.now - start_time).round(2)
+          ) unless options[:simulate]
         end
         say "Started:\t", :cyan
         say start_time.strftime('%d.%d.%Y %T')
         say "Runtime:\t", :cyan
         say "#{(Time.now - start_time).round(2)}s"
-        if options[:simulate]
-          say("Events found:\t", :cyan)
-        else
-          say("Events sent:\t", :cyan)
-        end
+        say "Status:\t\t", :cyan
+        say STATUS.key(status).upcase
+        action = options[:simulate] ? "found" : "sent"
+        say("Events #{action}:\t", :cyan)
         say count
       rescue => e
         say "Error: #{e.message}", :red
+        say e.backtrace; :yellow
       end
     end
 
@@ -109,13 +114,14 @@ module SmsLogparser
       begin
         runs = Mysql.new(options).last_runs(options[:results])
         if runs.size > 0
-          table = [%w(RunAt #Events LastEventID Status)]
+          table = [%w(run_at run_time match_count last_event_id status)]
           runs.to_a.reverse.each do |run|
             table << [
-              run['RunAt'],
-              run['EventsFound'],
-              run['LastEventID'],
-              STATUS.key(run['Status']).upcase
+              run['run_at'],
+              "#{run['run_time']}s",
+              run['match_count'],
+              run['last_event_id'],
+              STATUS.key(run['status']).upcase
             ]
           end
           print_table table

data/lib/sms-logparser/mysql.rb

@@ -3,6 +3,9 @@ module SmsLogparser
 
     def initialize(options)
       @options = options
+      @host_filter = options[:host_filter] || 'pcache%'
+      @query_limit = options[:query_limit] || 1000
+      @client = client
     end
 
     def client
@@ -15,23 +18,9 @@ module SmsLogparser
     end
 
     def last_runs(results = 10)
-      begin 
-        runs = client.query(
-          "SELECT * FROM SmsParserRuns ORDER BY ID DESC LIMIT #{results}"
-        )
-      rescue Mysql2::Error => e
-        raise e
-      end
-    end
-
-    def parser_table_exists?
-      begin
-        return client.query(
-          "SHOW TABLES LIKE 'SmsParserRuns'"
-        ).size > 0
-      rescue Mysql2::Error => e
-        raise e
-      end
+      runs = client.query(
+        "SELECT * FROM sms_logparser_runs ORDER BY id DESC LIMIT #{results}"
+      )
     end
 
     def create_parser_table(force = false)
@@ -40,69 +29,80 @@ module SmsLogparser
       elsif parser_table_exists?
         return 1
       end
-      begin
-        client.query(
-          "CREATE TABLE SmsParserRuns(\
-            ID SERIAL PRIMARY KEY AUTO_INCREMENT,\
-            RunAt datetime DEFAULT NULL,\
-            LastEventID BIGINT(20) UNSIGNED DEFAULT 0,\
-            EventsFound INT DEFAULT 0,\
-            Status TINYINT UNSIGNED DEFAULT 0,\
-            INDEX `LastEventID_I1` (`LastEventID`)
-          )"
-        )
-      rescue Mysql2::Error => e
-        raise e
-      end
+      client.query(
+        "CREATE TABLE sms_logparser_runs(\
+          id SERIAL PRIMARY KEY AUTO_INCREMENT,\
+          run_at datetime DEFAULT NULL,\
+          last_event_id BIGINT(20) UNSIGNED DEFAULT 0,\
+          match_count INT UNSIGNED DEFAULT 0,\
+          status TINYINT UNSIGNED DEFAULT 0,\
+          run_time DOUBLE(10,2) NOT NULL DEFAULT '0.00',\
+          INDEX `last_event_id_I1` (`last_event_id`)
+        )"
+      )
       return 0
     end
 
-    def drop_parser_table
-      return nil unless parser_table_exists?
-      begin
-        return client.query(
-          "DROP TABLE SmsParserRuns"
-        )
-      rescue Mysql2::Error => e
-        raise e
-      end
+    def write_parse_result(options={})
+      client.query(
+        "INSERT INTO sms_logparser_runs(run_at, last_event_id, match_count, status, run_time)\
+        VALUES(\
+          '#{options[:run_at].strftime("%Y-%m-%d %H:%M:%S")}',\
+          #{options[:last_event_id]},\
+          #{options[:match_count]},\
+          #{options[:status]},\
+          #{options[:run_time]}
+        )"
+      )
     end
 
     def get_entries(options={})
       last_id = options[:last_id] || get_last_parse_id
-      begin
-        sql = "SELECT * FROM SystemEvents\
-          WHERE `FromHost` like 'pcache%'\
-          AND ID > #{last_id} ORDER BY ID ASC"
-        sql += "  LIMIT #{options[:limit].to_i}" if options[:limit]
-        return client.query(sql)
-      rescue Mysql2::Error => e
-        raise e
+      if options[:limit]
+        max_id = last_id + options[:limit] + 1
+      else 
+        max_id = get_last_event_id
+      end
+      while last_id < max_id
+        entries = select_entries(last_id)
+        last_id = entries.to_a[-1]['ID']
+        yield entries
       end
     end
 
-    def write_parse_result(id, count, status)
-      client.query("INSERT INTO SmsParserRuns(RunAt, LastEventID, EventsFound, Status)\
-        VALUES(\
-          '#{Time.now.strftime("%Y-%m-%d %H:%M:%S")}',\
-          #{id},\
-          #{count},\
-          #{status}
-        )"
+    def get_last_parse_id
+      last_parse = client.query(
+        "SELECT last_event_id FROM sms_logparser_runs ORDER BY id DESC LIMIT 1"
       )
+      last_parse.first ? last_parse.first['last_event_id'] : 0
     end
 
-    def get_last_parse_id
-      id = 0
-      begin
-        last_parse = client.query(
-          "SELECT LastEventID FROM SmsParserRuns ORDER BY ID DESC LIMIT 1"
-        )
-        id = last_parse.first ? last_parse.first['LastEventID'] : 0
-      rescue Mysql2::Error => e
-        raise e
-      end
-      id
+    private
+
+    def select_entries(last_id)
+      sql = "SELECT * FROM SystemEvents\
+        WHERE `FromHost` like '#{@host_filter}'\
+        ORDER BY ID ASC\
+        LIMIT #{last_id},#{@query_limit};"
+      client.query(sql)
+    end
+
+    def get_last_event_id
+      last_event = client.query(
+        "SELECT ID FROM SystemEvents ORDER BY ID DESC LIMIT 1"
+      )
+      last_event.first ? last_event.first['ID'] : 0
+    end
+
+    def parser_table_exists?
+      client.query(
+        "SHOW TABLES LIKE 'sms_logparser_runs'"
+      ).size > 0
+    end
+
+    def drop_parser_table
+      return nil unless parser_table_exists?
+      client.query("DROP TABLE sms_logparser_runs")
     end
 
   end # class

data/lib/sms-logparser/parser.rb

@@ -2,20 +2,23 @@ module SmsLogparser
   class Parser
 
     def self.extract_data_from_msg(message)
-      m = message.match /\/content\/(\d+)\/(\d+)\/(\d+)\/(\w+\.\w+)\s.*\"\s\d+\s(\d+).+"(.*)"$/
-      raise "No match found." unless m
-      traffic_type = Parser.get_traffic_type(m[6])
-      visitor_type = Parser.get_visitor_type(traffic_type, m[4])
-      data = {
-        :customer_id => m[1],
-        :author_id => m[2],
-        :project_id => m[3],
-        :file =>  m[4],
-        :bytes => m[5],
-        :user_agent => m[6],
-        :traffic_type => traffic_type,
-        :visitor_type => visitor_type
-      }
+      if self.match?(message)
+        m = message.match /\/content\/(\d+)\/(\d+)\/(\d+)\/(\w+\.\w+)\s.*\"\s\d+\s(\d+).+"(.*)"$/
+        raise "No match found." unless m
+        traffic_type = Parser.get_traffic_type(m[6])
+        visitor_type = Parser.get_visitor_type(traffic_type, m[4])
+        return {
+          :customer_id => m[1],
+          :author_id => m[2],
+          :project_id => m[3],
+          :file =>  m[4],
+          :bytes => m[5],
+          :user_agent => m[6],
+          :traffic_type => traffic_type,
+          :visitor_type => visitor_type
+        }
+      end
+      nil
     end
 
     def self.match?(message)

data/lib/sms-logparser/version.rb

@@ -1,3 +1,3 @@
 module SmsLogparser
-  VERSION = "0.5.0"
+  VERSION = "0.6.0"
 end

data/sms-logparser.gemspec

@@ -24,6 +24,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rake'
 
   spec.add_dependency 'thor', '~> 0.19.1'
-  spec.add_dependency 'faraday', '~> 0.9.0'
   spec.add_dependency 'mysql2'
+  spec.add_dependency 'faraday', '~> 0.9.0'
+  spec.add_dependency 'net-http-persistent', '>= 2.9.4'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sms-logparser
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - niwo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-03 00:00:00.000000000 Z
+date: 2014-04-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,6 +52,20 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.19.1
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: faraday
   requirement: !ruby/object:Gem::Requirement
@@ -67,19 +81,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.9.0
 - !ruby/object:Gem::Dependency
-  name: mysql2
+  name: net-http-persistent
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.9.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.9.4
 description: Reads access logs stored in a MySQL database (coming from the SWISS TXT
   CDN) and sends them to the SMS API.
 email: