sms-logparser 0.0.4 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 51e05056b98cf58379c37b19df329c948e2b8656
-  data.tar.gz: bc696cef467afb68d9ee9b18e281ea8aa6b497b5
+  metadata.gz: 2e9efb4712d1d9d54c9904f57993ced13742bd51
+  data.tar.gz: 2fa877596b733aac7a13a27cd76ac22902978564
 SHA512:
-  metadata.gz: 86d201529f5ac57d50082fb992d815e1969ae2a8dac652c11a017b27b0f8410a107b694829d55b9d5a3118d6ee5dac82797ee9d64132d87bc7067006f51aa0ae
-  data.tar.gz: 808350b006292eae603d3a12a2274ff080640c65a65052e9b22f9797ad97ce222b548e6d0f6558f12dc707485ac9f148e6ee5be627b75290ede18ef67b6ee3b9
+  metadata.gz: ecb5588a86ae77f592f67117172b4b1d259fec82d7bd6e80305c70de60032db646c8751734eed245d5d90867a1e875bf08e366465a1db23dade2400df1744b61
+  data.tar.gz: 1202cb4ff8e9d3820682e5cb6bbc0b032c0a5168dfc32c963915b5055f5bfe7a344e64870eb10a1d972e3d0f413e236b5e243f1fcd7b06638b94ea7ab6e5f098

data/lib/sms-logparser/api.rb

@@ -0,0 +1,26 @@
+module SmsLogparser
+  class Api
+
+    def initialize(options)
+      @options = options
+    end
+
+    def send(data)
+      url = "#{@options[:api_base_path]}/"
+      url += "#{data[:customer_id]}/"
+      url += "#{data[:author_id]}/"
+      url += "#{data[:project_id]}/"
+      url += "#{data[:traffic_type]}/"
+      url += "#{data[:bytes]}"
+      unless @options[:simulate]
+        begin
+          RestClient.get(url)
+        rescue
+          raise "Can't send log to #{url}"
+        end
+      end
+      url
+    end
+
+  end # class
+end # module

data/lib/sms-logparser/cli.rb

@@ -1,14 +1,6 @@
-module SmsLogparser 
+module SmsLogparser
   class Cli < Thor
 
-    EXTENSION_MAP = {
-      'default' => 'TRAFFIC_PODCAST',
-      'mp4' => 'TRAFFIC_PODCAST',
-      'f4v' => 'TRAFFIC_PODCAST',
-      'flv' => 'TRAFFIC_PODCAST',
-      'ts' => 'TRAFFIC_MOBILE'
-    }
-
     class_option :mysql_host, :default => 'localhost'
     class_option :mysql_user, :default => 'root'
     class_option :mysql_db, :default => 'Syslog'
@@ -25,130 +17,73 @@ module SmsLogparser
     def parse
       start_time = Time.now
       count = 0
-      last_id = get_last_parse_id 
-      begin 
-        results = client.query(
-          "SELECT * FROM SystemEvents\
-          WHERE `FromHost` like 'pcache%'\
-          AND ID > #{last_id} ORDER BY ID ASC"
-        )
-      rescue Mysql2::Error
-        say "parser_table not found please create it with 'create_parser_table'", :red
-        exit 1
-      end
-      results.each do |result|
-        if result['Message'] =~ /\/content\/.*\.(f4v|flv|mp4|ts) .*/
-          data = extract_data_from_msg(result['Message'])
-          url = "#{options[:api_base_path]}/"
-          url += "#{data[:customer_id]}/"
-          url += "#{data[:author_id]}/"
-          url += "#{data[:project_id]}/"
-          url += "#{data[:traffic_type]}/"
-          url += "#{data[:bytes]}"
-          if options[:simulate]
-            puts "Message ID: #{result['ID']}"
-            puts "URL: #{url}"
-            puts "Data: #{data}"
-            puts "-----------------------"
-          else
-            begin
-              RestClient.get(url)
-            rescue
-              say "Error: Can't send log to #{url}", :red
-              say "Aborting.", :red
-              break
-            end
+      begin
+        mysql = Mysql.new(options)
+        entries = mysql.get_entries
+        api = Api.new(options)
+        last_id = mysql.get_last_parse_id
+        entries.each do |entry|
+          if Parser.match(entry)
+            data = Parser.extract_data_from_msg(entry['Message'])
+            url = api.send(data)
+            last_id = entry['ID']
+            count += 1
+            debug_parser_output(data, url, entry) if options[:simulate]
           end
-          count += 1
         end
-        last_id = result['ID']
+        mysql.write_parse_result(last_id, count) unless options[:simulate]
+        puts "Started: #{start_time.strftime('%d.%d.%Y %T')}"
+        puts "Runtime: #{(Time.now - start_time).round(2)}s"
+        puts "Matches: #{count}"
+      rescue => e
+        say "Error: #{e.message}", :red
       end
-      write_parse_result(last_id, count) unless options[:simulate]
-      puts "Started: #{start_time.strftime('%d.%d.%Y %T')}"
-      puts "Runtime: #{(Time.now - start_time).round(2)}s"
-      puts "Matches: #{count}"
     end
 
     desc "last_runs", "List the last paser runs"
     def last_runs
-      begin 
-        runs = client.query(
-          "SELECT * FROM SmsParserRuns ORDER BY ID ASC LIMIT 10"
-        )
-      rescue Mysql2::Error
-        say "parser_table not found please create it with 'create_parser_table'", :red
-        exit 1
-      end
-      if runs.size > 0
-        table = [%w(RunAt #Events LastEventID)]
-        runs.each do |run|
-          table << [
-            run['RunAt'],
-            run['EventsFound'],
-            run['LastEventID']
-          ]
+      begin
+        runs = Mysql.new(options).last_runs
+        if runs.size > 0
+          table = [%w(RunAt #Events LastEventID)]
+          runs.each do |run|
+            table << [
+              run['RunAt'],
+              run['EventsFound'],
+              run['LastEventID']
+            ]
+          end
+          print_table table
+        else
+          say "No parser runs found in the database."
         end
-        print_table table
-      else
-        say "No parser runs found in the database."
+      rescue => e
+        say "Error: #{e.message}", :red
       end
     end
 
-    desc "create_parser_table", "Create the parser table to track the last logs parsed"
-    def create_parser_table
-      client.query(
-        "CREATE TABLE IF NOT EXISTS\
-          SmsParserRuns(\
-            ID INT PRIMARY KEY AUTO_INCREMENT,\
-            RunAt datetime DEFAULT NULL,\
-            LastEventID INT DEFAULT NULL,\
-            EventsFound INT DEFAULT 0,\
-            INDEX `LastEventID_I1` (`LastEventID`)
-          )"
-      )
-      say "OK", :green
+    desc "setup", "Create the parser table to track the last logs parsed"
+    def setup
+      begin
+        Mysql.new(options).create_parser_table
+        say "OK", :green
+      rescue => e
+        say "Error: #{e.message}", :red
+      end
     end
 
     no_commands do
-      def get_last_parse_id
-        last_parse = client.query(
-          "SELECT LastEventID FROM SmsParserRuns ORDER BY ID DESC LIMIT 1"
-        )
-        last_parse.first ? last_parse.first['LastEventID'] : 0
-      rescue Mysql2::Error
-        say "parser_table not found please create it with 'create_parser_table'", :red
-        exit 1
-      end
-
-      def write_parse_result(id, count)
-        client.query("INSERT INTO SmsParserRuns(RunAt, LastEventID, EventsFound)\
-          VALUES(\
-            '#{Time.now.strftime("%Y-%m-%d %H:%M:%S")}',\
-            #{id},\
-            #{count}
-          )"
-        )
-      end
-
-      def extract_data_from_msg(msg)
-        m = msg.match /\/content\/(\d+)\/(\d+)\/(\d+)\/.+\.(\S+)\s.*\"\s\d+\s(\d+)/
-        data = {
-          :customer_id => m[1],
-          :author_id => m[2],
-          :project_id => m[3],
-          :ext => m[4],
-          :traffic_type => (EXTENSION_MAP[m[4]] || EXTENSION_MAP['default']),
-          :bytes => m[5]
-        }
-      end
-
-      def client
-        @client ||= Mysql2::Client.new(
-          :host => options[:mysql_host],
-          :username => options[:mysql_user],
-          :database => options[:mysql_db]
-        )
+      def debug_parser_output(data, url, entry)
+        puts
+        say "Message ID: ", :green
+        say entry['ID']
+        say "URL: ", :green
+        say url
+        say "Data: ", :green
+        say data
+        puts
       end
     end
+
   end
 end

data/lib/sms-logparser/mysql.rb

@@ -0,0 +1,79 @@
+module SmsLogparser
+  class Mysql
+
+    def initialize(options)
+      @options = options
+    end
+
+    def client
+      @client ||= Mysql2::Client.new(
+        :host => @options[:mysql_host],
+        :username => @options[:mysql_user],
+        :database => @options[:mysql_db]
+      )
+    end
+
+    def last_runs
+      begin 
+        runs = client.query(
+          "SELECT * FROM SmsParserRuns ORDER BY ID ASC LIMIT 10"
+        )
+      rescue Mysql2::Error => e
+        raise e
+      end
+    end
+
+    def create_parser_table
+      begin
+        return client.query(
+          "CREATE TABLE IF NOT EXISTS\
+            SmsParserRuns(\
+              ID INT PRIMARY KEY AUTO_INCREMENT,\
+              RunAt datetime DEFAULT NULL,\
+              LastEventID INT DEFAULT NULL,\
+              EventsFound INT DEFAULT 0,\
+              INDEX `LastEventID_I1` (`LastEventID`)
+            )"
+        )
+      rescue Mysql2::Error => e
+        raise e
+      end
+    end
+
+    def get_entries(last_id = get_last_parse_id)
+      begin 
+        return client.query(
+          "SELECT * FROM SystemEvents\
+          WHERE `FromHost` like 'pcache%'\
+          AND ID > #{last_id} ORDER BY ID ASC"
+        )
+      rescue Mysql2::Error => e
+        raise e
+      end
+    end
+
+    def write_parse_result(id, count)
+      client.query("INSERT INTO SmsParserRuns(RunAt, LastEventID, EventsFound)\
+        VALUES(\
+          '#{Time.now.strftime("%Y-%m-%d %H:%M:%S")}',\
+          #{id},\
+          #{count}
+        )"
+      )
+    end
+
+    def get_last_parse_id
+      id = 0
+      begin
+        last_parse = client.query(
+          "SELECT LastEventID FROM SmsParserRuns ORDER BY ID DESC LIMIT 1"
+        )
+        id = last_parse.first ? last_parse.first['LastEventID'] : 0
+      rescue Mysql2::Error => e
+        raise e
+      end
+      id
+    end
+
+  end # class
+end # module

data/lib/sms-logparser/parser.rb

@@ -0,0 +1,34 @@
+module SmsLogparser
+  class Parser
+
+    def self.extract_data_from_msg(msg)
+      m = msg.match /\/content\/(\d+)\/(\d+)\/(\d+)\/.+\.(\S+)\s.*\"\s\d+\s(\d+).+"(.*)"$/
+      raise "No match found." unless m
+      data = {
+        :customer_id => m[1],
+        :author_id => m[2],
+        :project_id => m[3],
+        :ext => m[4],
+        :traffic_type => Parser.get_traffic_type(m[4]),
+        :bytes => m[5],
+        :user_agent => m[6]
+      }
+    end
+
+    def self.match(entry)
+      entry['Message'] =~ /\/content\/.*\.(f4v|flv|mp4|mp3|ts) .*/
+    end
+
+    def self.get_traffic_type(user_agent)
+      case user_agent
+      when /.*(iTunes).*/
+        "TRAFFIC_PODCAST"
+      when /.*(IEMobile|Mobile Safari|iPhone|iPod|iPad).*/
+        "TRAFFIC_MOBILE"
+      else
+        "TRAFFIC_WEBCAST"
+      end
+    end
+
+  end # class
+end # module

data/lib/sms-logparser/version.rb

@@ -1,3 +1,3 @@
 module SmsLogparser
-  VERSION = "0.0.4"
+  VERSION = "0.1.0"
 end

data/lib/sms-logparser.rb

@@ -4,4 +4,7 @@ require 'mysql2'
 require 'rest_client'
 
 require "sms-logparser/version"
+require "sms-logparser/mysql"
+require "sms-logparser/parser"
+require "sms-logparser/api"
 require "sms-logparser/cli"

data/spec/{logparser_spec.rb → cli_spec.rb}

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe SmsLogparser do
+describe SmsLogparser::Cli do
   before do
     TestHelper.create_test_db
     TestHelper.create_sylog_db_table
@@ -12,7 +12,7 @@ describe SmsLogparser do
 
   it "can create the parser_runs database table" do
     out, err = capture_io do
-      TestHelper.sms_logparser.create_parser_table
+      TestHelper.sms_logparser.setup
     end
     out.must_match /OK/
   end
@@ -22,7 +22,7 @@ describe SmsLogparser do
     parser = TestHelper.sms_logparser
     parser.options[:api_base_path] = 'http://devnull-as-a-service.com/dev/null/'
     out, err = capture_io do
-      TestHelper.sms_logparser.create_parser_table
+      TestHelper.sms_logparser.setup
       parser.parse  
     end
     out.must_match /.*Matches: 10$/
@@ -33,22 +33,22 @@ describe SmsLogparser do
     parser = TestHelper.sms_logparser
     parser.options[:api_base_path] = 'http://devnull-as-a-service.com/dev/null/'
     out, err = capture_io do
-      TestHelper.sms_logparser.create_parser_table
+      TestHelper.sms_logparser.setup
       parser.parse
       parser.parse  
     end
     out.must_match /.*Matches: 0$/
   end
 
-  it "list parser runs" do
+  it "lists parser runs" do
     TestHelper.seed_db(1)
     parser = TestHelper.sms_logparser
     parser.options[:api_base_path] = 'http://devnull-as-a-service.com/dev/null/'
     out, err = capture_io do
-      TestHelper.sms_logparser.create_parser_table
+      TestHelper.sms_logparser.setup
       parser.parse
       parser.last_runs 
     end
     assert_equal(err, "")
   end
-end
+end

data/spec/spec_helper.rb

@@ -41,13 +41,13 @@ module TestHelper
   end
 
   def self.insert_logs(host = "pcache", message = "", number_of_inserts = 1)
-    values = ''
+    values = []
     number_of_inserts.times do
-      values += "('#{host}', '#{message}'), "
+      values << "('#{host}', '#{message}')"
     end
     self.client.query(
       "INSERT INTO #{@@mysql_db}.SystemEvents(FromHost, Message)\
-        VALUES #{values.chomp(', ')}"
+        VALUES #{values.join(', ')}"
     )
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sms-logparser
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.1.0
 platform: ruby
 authors:
 - niwo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-05 00:00:00.000000000 Z
+date: 2014-03-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -95,10 +95,13 @@ files:
 - Rakefile
 - bin/sms-logparser
 - lib/sms-logparser.rb
+- lib/sms-logparser/api.rb
 - lib/sms-logparser/cli.rb
+- lib/sms-logparser/mysql.rb
+- lib/sms-logparser/parser.rb
 - lib/sms-logparser/version.rb
 - sms-logparser.gemspec
-- spec/logparser_spec.rb
+- spec/cli_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/swisstxt/sms-logparser
 licenses:
@@ -125,5 +128,5 @@ signing_key:
 specification_version: 4
 summary: SMS Logparser
 test_files:
-- spec/logparser_spec.rb
+- spec/cli_spec.rb
 - spec/spec_helper.rb