sms-logparser 0.8.4 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ee62704f6c086eca49b8be2a23d8d553bb2ecdc5
-  data.tar.gz: fe95f14c8bc3733ba2046d8501ff15fbf900013f
+  metadata.gz: 9f876436c72457987cb31430a76668f0613cde56
+  data.tar.gz: cb5a9b5a5b6d4d2a09bded36130cbe557492dc10
 SHA512:
-  metadata.gz: 0da2eeb8730a5c9efea1379c02feeacc9e29c145ca29ba8c9e6e390503042c7b3a56833c792f68e9d505bcd296c7ad08c2fc8a2af0b490d68fdc97dc409a7048
-  data.tar.gz: b564ad5f22a5797ed5982d7ba9855526986d5e495fe8f1be7b327e8e2f501f5ca2b99f7c8f3934c1a998779563ecef4be65b6e49acb9d3084f91fd5d52988d31
+  metadata.gz: 1c258366166a28499e44ab7867d7a6b4e6cbdfca5a837bd6910322cd8ef8c022127518539954fea02303cc7d22d01708b3a1a2951dc69811e8db37a053ca1825
+  data.tar.gz: b5b2c2199aa31028db755f46cdcaf2a408232a8e6cdd0359b9084ff399db4645f31e3bbbeb97419cda1a7f6445322b96d0b15f43e3060150826f7041a100d59c

data/lib/sms-logparser.rb

@@ -1,10 +1,11 @@
 require "rubygems" # ruby1.9 doesn't "require" it though
 require "thor"
-require 'mysql2'
-require 'faraday'
+require "mysql2"
+require "faraday"
 
 require "sms-logparser/version"
 require "sms-logparser/mysql"
 require "sms-logparser/parser"
 require "sms-logparser/api"
+require "sms-logparser/app_logger"
 require "sms-logparser/cli"

data/lib/sms-logparser/api.rb

@@ -4,25 +4,26 @@ module SmsLogparser
 
     def initialize(options)
       @options = options
-      @base_url = URI(@options[:api_base_url] || 'http://localhost:8080/creator/rest/')
+      @base_url = URI(@options[:api_base_url] || 'http://localhost:8080/creator/rest/') 
       @url = @base_url.to_s.chomp(@base_url.path)
+      @base_path = (@base_url.path << "/").squeeze('/')
       @accepted_responses =  parse_status_codes(options[:accepted_api_responses]) || [200]
       @connection = connection
     end
 
     def send(data)
       requests = []
-      base_path = @base_url.path + [data[:customer_id], data[:author_id], data[:project_id]].join('/')
+      path = @base_path << [data[:customer_id], data[:author_id], data[:project_id]].join('/')
       unless data[:file] =~ /.*\.m3u8$/
         requests << {
           url: @url, 
-          uri: [base_path, data[:traffic_type], data[:bytes]].join('/'),
+          uri: [path, data[:traffic_type], data[:bytes]].join('/'),
         }
       end
       if data[:visitor_type]
         requests << {
           url: @url,
-          uri: [base_path, data[:visitor_type], 1].join('/')
+          uri: [path, data[:visitor_type], 1].join('/')
         }
       end
       unless @options[:simulate]
@@ -45,9 +46,11 @@ module SmsLogparser
     private
 
     def connection
-      connection = Faraday.new(url: @url) do |faraday|
+      connection = Faraday.new(url: @url, request: {timeout: 5}) do |faraday|
         faraday.request :url_encoded
-        faraday.response :logger if @options[:debug]
+        if @options[:debug]
+          faraday.use Faraday::Response::Logger, SmsLogparser::AppLogger.instance 
+        end
         faraday.adapter :net_http_persistent
       end
       connection.headers[:user_agent] = "sms-logparser v#{SmsLogparser::VERSION}"

data/lib/sms-logparser/app_logger.rb

@@ -0,0 +1,24 @@
+module SmsLogparser
+  require 'logger'
+  require 'singleton'
+
+  class AppLogger < Logger
+    include Singleton
+    
+    def initialize
+      @level = INFO
+      @logdev = STDOUT
+      @progname ||= 'sms-logparser'
+      @formatter = proc do |severity, datetime, progname, msg|
+        "#{datetime} #{severity} [sms-logparser v#{SmsLogparser::VERSION}] #{msg}\n"
+      end
+    end
+
+    def set_log_device(log_file_path = nil)
+      device = log_file_path ? File.open(log_file_path, "a") : STDOUT
+      @logdev = Logger::LogDevice.new(device)
+      self
+    end
+
+  end
+end

data/lib/sms-logparser/cli.rb

@@ -9,21 +9,12 @@ module SmsLogparser
       aliases: %w(-c),
       desc: "Configuration file for default options"
 
-    class_option :mysql_host, 
-      aliases: %w(-h),
-      desc: "MySQL host"
-    
-    class_option :mysql_user,
-      aliases: %w(-u),
-      desc: "MySQL user (Default: root)"
-    
-    class_option :mysql_password,
-      aliases: %w(-p),
-      desc: "MySQL password"
-    
-    class_option :mysql_db,
-      aliases: %w(-d),
-      desc: "MySQL database (Default: Syslog)"
+    class_option :debug, type: :boolean, desc: "Show debug output"
+    class_option :logfile, desc: "Path to the logfile (Default: STDOUT)"
+    class_option :mysql_host, aliases: %w(-h), desc: "MySQL host"
+    class_option :mysql_user, aliases: %w(-u), desc: "MySQL user (Default: root)"
+    class_option :mysql_password, aliases: %w(-p), desc: "MySQL password"
+    class_option :mysql_db, aliases: %w(-d), desc: "MySQL database (Default: Syslog)"
 
     desc "version", "Print sms-logparser version number"
     def version
@@ -32,45 +23,29 @@ module SmsLogparser
     map %w(-v --version) => :version
 
     desc "parse", "Check the database for pcache logs and send them to the SMS-API"
-    option :api_base_url,
-      aliases: %w(-a),
+    option :api_base_url, aliases: %w(-a),
       desc: "Base path of the SMS API (Default: http://localhost:8080/creator/rest/)"
-    option :api_key,
-      aliases: %w(-k),
-      desc: "SMS API Key"
-    option :simulate,
-      type: :boolean,
-      default: false,
-      aliases: %w(-s),
+    option :api_key, aliases: %w(-k), desc: "SMS API Key"
+    option :simulate, type: :boolean, aliases: %w(-s),
       desc: "Dry run without submitting any data"
-    option :verbose,
-      type: :boolean,
-      default: false,
-      aliases: %w(-v),
-      desc: "Verbose output"
-    option :limit,
-      type: :numeric,
-      aliases: %w(-l),
-      desc: "Limit the number of entries to query"
-    option :debug,
-      type: :boolean,
-      default: false,
-      desc: "Show debug output"
-    option :accepted_api_responses,
-      type: :array,
+    option :verbose, type: :boolean, aliases: %w(-v), desc: "Verbose output"
+    option :limit, type: :numeric, aliases: %w(-l), desc: "Limit the number of entries to query"
+    option :accepted_api_responses, type: :array, aliases: %w(-r),
       desc: "API HTTP responses which are accepted (Default: only accept 200)."
     def parse
-      say "Starting the parser...", :green
+      start_message = "Parser started"
+      start_message += options[:simulate] ? " in simulation mode." : "."
+      logger.info(start_message)
       mysql = Mysql.new(options)
       if !options[:simulate] && mysql.parser_running?
-        say "Exit. Another instance of the parser is already running.", :yellow
+        logger.warn("Exit. Another instance of the parser is already running.")
         exit
       end
       state = {
         last_event_id: mysql.get_last_parse_id, 
         match_count: 0,
         status: STATUS[:running],
-        run_at: Time.now,
+        started_at: Time.now,
         run_time: 0.0
       }
       state = mysql.start_run(state) unless options[:simulate]
@@ -81,45 +56,42 @@ module SmsLogparser
             if data
               requests = api.send(data)
               state[:match_count] += 1
-              verbose_parser_output(data, requests, entry['ID']) if options[:verbose]
+              verbose_parser_output(entry['ID'], data, requests) if options[:verbose]
+              state[:last_event_id] = entry['ID']
             end
           end
-          state[:last_event_id] = entry['ID']
         end
       end
       state[:status] = STATUS[:ok]
     rescue => e
-      say "Error: #{e.message}", :red
-      say "Aborting parser run...", :red
+      logger.error("#{e.message}. Aborting the parser run.")
       say(e.backtrace.join("\n"), :yellow) if options[:debug]
       state[:status] = STATUS[:api_error] if state
     ensure
       begin
         if mysql && state
-          state[:run_time] = (Time.now - state[:run_at]).round(2)
+          state[:run_time] = (Time.now - state[:started_at]).round(2)
           if state[:id]
             mysql.write_parse_result(state) unless options[:simulate]
           end
-          print_parse_results(state)
+          log_parse_results(state)
         end
       rescue => e
-        say "Error: #{e.message}", :red
+        logger.fatal(e.message)
         say(e.backtrace.join("\n"), :yellow) if options[:debug]
       end
     end
 
     desc "history", "List the last paser runs"
-    option :results,
-      type: :numeric,
-      default: 10,
-      aliases: %w(-n),
+    option :results, type: :numeric, default: 10, aliases: %w(-n),
       desc: "Number of results to display"
     def history
       runs = Mysql.new(options).last_runs(options[:results])
       if runs && runs.size > 0
-        table = [%w(run_at count last_id status run_time)]
+        table = [%w(id started_at count last_id status run_time)]
         runs.to_a.reverse.each do |run|
           table << [
+            run['id'],
             run['run_at'].strftime('%d.%d.%Y %T'),
             run['match_count'],
             run['last_event_id'],
@@ -132,50 +104,46 @@ module SmsLogparser
         say "No parser runs found in the database."
       end
     rescue => e
-      say "Error: #{e.message}", :red
-      exit 1
+      logger.fatal(e.message)
+      say(e.backtrace.join("\n"), :yellow) if options[:debug]
     end
 
     desc "setup", "Create the parser table to track the last logs parsed"
-    option :force,
-      type: :boolean,
-      default: false,
-      aliases: %w(-f),
+    option :force, type: :boolean, default: false, aliases: %w(-f),
       desc: "Drop an existing table if it exists"
     def setup
       case Mysql.new(options).create_parser_table(options[:force])
       when 0
-        say "OK, table created.", :green
+        logger.info("Created database table.")
       when 1
-        say "Table already exists.", :yellow
+        logger.warn("Table already exists.")
+      when 2
+        logger.info("Recreated database tables.")
       end
     rescue => e
-      say "Error: #{e.message}", :red
-      exit 1
+      logger.fatal(e.message)
+      say(e.backtrace.join("\n"), :yellow) if options[:debug]
     end
 
     no_commands do
-      def verbose_parser_output(data, requests, entry_id)
-        say "ID:\t", :cyan
-        say entry_id
-        say "URL:\t", :cyan
-        say requests.map {|req| "#{req[:url]}#{req[:uri]} (Status: #{req[:status] || 'n/a'})"}.join("\n\t")
-        say "Data:\t", :cyan
-        say data.map{|k,v| "#{k}:\t#{v || '-'}"}.join("\n\t") || "\n"
-        puts
-        puts "-" * 100
-        puts
+      def logger
+        SmsLogparser::AppLogger.instance.set_log_device(options[:logfile])
+      end
+
+      def verbose_parser_output(entry_id, data, requests)
+        logger.info {
+          "parsing data for #{entry_id} (#{data.map{|k,v| "#{k}=\"#{v || '-'}\""}.join(" ") || ''})"
+        }
+        logger.info {
+          "parser urls for entry #{entry_id} (#{requests.map {|req| "#{req[:url]}#{req[:uri]}"}.join(" ")})"
+        }
       end
 
-      def print_parse_results(res)
-        say "Started:\t", :cyan
-        say res[:run_at].strftime('%d.%d.%Y %T')
-        say "Runtime:\t", :cyan
-        say "#{res[:run_time]}s"
-        say "Status:\t\t", :cyan
-        say STATUS.key(res[:status]).upcase
-        say("Events #{options[:simulate] ? "found" : "sent"}:\t", :cyan)
-        say res[:match_count]
+      def log_parse_results(res)
+        res = res.reject {|k,v| %w(:id :run_at).include? k}
+        message = "Parser #{options[:simulate] ? 'simulation' : 'run'} ended."
+        message +=  " (" + res.map {|k,v| "#{k}=\"#{v}\""}.join(' ') + ")"
+        logger.info(message)
       end
 
       def options

data/lib/sms-logparser/mysql.rb

@@ -24,8 +24,9 @@ module SmsLogparser
     end
 
     def create_parser_table(force = false)
+      status = 0
       if force
-        drop_parser_table
+        status = drop_parser_table ? 2 : 0
       elsif parser_table_exists?
         return 1
       end
@@ -40,7 +41,7 @@ module SmsLogparser
           INDEX `last_event_id_I1` (`last_event_id`)
         )"
       )
-      return 0
+      status 
     end
 
     def parser_running?(running_state = 3)
@@ -57,7 +58,7 @@ module SmsLogparser
       client.query(
         "INSERT INTO sms_logparser_runs (run_at, status)\
         VALUES (\
-          '#{options[:run_at].strftime("%Y-%m-%d %H:%M:%S")}',\
+          '#{options[:started_at].strftime("%Y-%m-%d %H:%M:%S")}',\
           #{options[:status]}\
         )"
       )
@@ -131,8 +132,9 @@ module SmsLogparser
     end
 
     def drop_parser_table
-      return nil unless parser_table_exists?
+      return false unless parser_table_exists?
       client.query("DROP TABLE sms_logparser_runs")
+      true
     end
 
   end # class

data/lib/sms-logparser/version.rb

@@ -1,3 +1,3 @@
 module SmsLogparser
-  VERSION = "0.8.4"
+  VERSION = "0.9.0"
 end

data/spec/cli_spec.rb

@@ -19,7 +19,7 @@ describe SmsLogparser::Cli do
     out, err = capture_io do
       parser.setup
     end
-    out.must_match /OK.*/
+    out.must_match /.*Created database table./
   end
 
   it "can parse a log database and find matches" do
@@ -31,7 +31,7 @@ describe SmsLogparser::Cli do
       parser.setup
       parser.parse  
     end
-    out.must_match /\s+10$/
+    out.must_match /.*match_count=10.*/
   end
 
   # it "skips over already parsed logs" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sms-logparser
 version: !ruby/object:Gem::Version
-  version: 0.8.4
+  version: 0.9.0
 platform: ruby
 authors:
 - niwo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-08 00:00:00.000000000 Z
+date: 2014-04-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -125,6 +125,7 @@ files:
 - bin/sms-logparser
 - lib/sms-logparser.rb
 - lib/sms-logparser/api.rb
+- lib/sms-logparser/app_logger.rb
 - lib/sms-logparser/cli.rb
 - lib/sms-logparser/mysql.rb
 - lib/sms-logparser/parser.rb