sms-logparser 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9f0b662e2f8fc572e36fb90d70772df4719bd74e
-  data.tar.gz: b689ed872012488f135a78ddd38e3583025a02e5
+  metadata.gz: 7d8d1676ea63d17f63d196dd955d52bcd822c28b
+  data.tar.gz: 6a0a0341ed6a5050cefa8871835247122f2a2934
 SHA512:
-  metadata.gz: 141e69433bd1a57508777f58f6f21f72c7d6456c56bf13b284ac84e85fd75045ab04df0e47aa608db8f0ade2136c9a44e14bcca9c268988d5e7e4acffe50616f
-  data.tar.gz: 73fd2b7ddef48779755558005dbe50ee4bf53fc7c4d7f3e0c2f4f5d3dfd3f3d271bed72fd15e83c2f4e4f9dcaf1ef09d46cdf2ca61474f5c0641305c9dea0bd1
+  metadata.gz: dabe200a09f9dd8f508726c6b8ba8471847c713ae0e142c0121ea8eafb526321cc023972c6c0d830996de5107158981fafa987c53f44074af772947fe9a9bae8
+  data.tar.gz: b5e7aa89a019a2582fbf0296fc38a3350da2a7aba1cc41c03b7fed1f01948b768f570807fce987b81a9cf6842b7fb770c68ce363946ce55f4a1ed3a14c19825e

data/README.md

@@ -1,46 +1,60 @@
-# SmsLogparser
+# SMS Logparser
 
-sms-logparser - DB-Logparser for Simplex Media Server (SMS). Reads access logs stored in a MySQL database (coming from the SWISS TXT CDN) and pushes them to the SMS API.
+sms-logparser - Logparser for Simplex Media Server (SMS). Reads access logs stored in a MySQL database (coming from the SWISS TXT CDN) and sends them to the SMS API.
 
 [![Gem Version](https://badge.fury.io/rb/sms-logparser.png)](http://badge.fury.io/rb/sms-logparser)
 
 ## Installation
 
-Add this line to your application's Gemfile:
+Install the sms-logpaser gem:
 
-    gem 'sms-logparser'
-
-And then execute:
-
-    $ bundle
-
-Or install it yourself as:
-
-    $ gem install sms-logparser
+```bash
+$ gem install sms-logparser
+```
 
 ## Setup
 
 Create the database table to track which logs have been parsed:
 
-    $ sms-logparser setup
+```bash
+$ sms-logparser setup
+```
 
 Make a test run:
 
-    $ sms-logparser parse --simulate --verbose
+```bash
+$ sms-logparser parse --simulate --verbose
+```
 
 ## Usage
 
 See available commands:
 
-    $ sms-logparser help
+```bash
+$ sms-logparser help
+```
 
 Parse logs from database and send them to the API
 
-    $ sms-logparser parse
+```bash
+$ sms-logparser parse
+```
 
 Show the last parser runs:
 
-    $ sms-logparser last_runs
+```bash
+$ sms-logparser history
+``
+
+## Configuration file
+
+sms-logparser tries to read default options from a yaml file named '.sms-logparser.yml' placed in your home directory. Using the "-c/--config" flag you can adapt the path to the configuration file.
+
+An configuration for adapting the default MySQL password could look like this:
+
+```yaml
+:mysql_password: "my!secret"
+```
 
 ## Development
 

data/lib/sms-logparser/api.rb

@@ -6,12 +6,17 @@ module SmsLogparser
     end
 
     def send(data)
+      urls = []
       base_url = "#{@options[:api_base_path]}/"
       base_url += "#{data[:customer_id]}/"
       base_url += "#{data[:author_id]}/"
       base_url += "#{data[:project_id]}"
-      urls = ["#{base_url}/#{data[:traffic_type]}/#{data[:bytes]}"]
-      urls << "#{base_url}/#{data[:visitor_type]}/1" if data[:visitor_type]
+      unless data[:file] =~ /.*\.m3u8$/
+        urls = ["#{base_url}/#{data[:traffic_type]}/#{data[:bytes]}"]
+      end
+      if data[:visitor_type]
+        urls << "#{base_url}/#{data[:visitor_type]}/1"
+      end
       unless @options[:simulate]
         urls.each do |url|
           begin

data/lib/sms-logparser/cli.rb

@@ -1,22 +1,56 @@
 module SmsLogparser
   class Cli < Thor
+    require 'yaml'
 
-    class_option :mysql_host, :default => 'localhost', aliases: %w(-h)
-    class_option :mysql_user, :default => 'root', aliases: %w(-u)
-    class_option :mysql_password, aliases: %w(-p)
-    class_option :mysql_db, :default => 'Syslog', aliases: %w(-d)
+    STATUS = {
+      :ok => 0,
+      :api_error => 1
+    }
 
-    desc "version", "print cloudstack-cli version number"
+    class_option :config, 
+      :default => File.join(Dir.home, '.sms-logparser.yml'),
+      :aliases => %w(-c),
+      :desc => "Configuration file for default options"
+
+    class_option :mysql_host, 
+      :default => 'localhost',
+      :aliases => %w(-h),
+      :desc => "MySQL host"
+    
+    class_option :mysql_user,
+      :default => 'root',
+      :aliases => %w(-u),
+      :desc => "MySQL user"
+    
+    class_option :mysql_password,
+      :aliases => %w(-p),
+      :desc => "MySQL password"
+    
+    class_option :mysql_db,
+      :default => 'Syslog',
+      :aliases => %w(-d),
+      :desc => "MySQL database"
+
+    desc "version", "Print cloudstack-cli version number"
     def version
       say "sms-logparser version #{SmsLogparser::VERSION}"
     end
     map %w(-v --version) => :version
 
-    desc "parse", "Check the database for pcache logs and send them to SMS"
-    option :api_base_path, :default => 'http://dev.simplex.tv/creator/rest', aliases: %w(-a)
-    option :api_key, aliases: %w(-k)
-    option :simulate, :type => :boolean, :default => false, aliases: %w(-s)
-    option :verbose, :type => :boolean, :default => false, aliases: %w(-v)
+    desc "parse", "Check the database for pcache logs and send them to the SMS-API"
+    option :api_base_path,
+      :default => 'http://dev.simplex.tv/creator/rest',
+      :aliases => %w(-a)
+    option :api_key,
+      :aliases => %w(-k)
+    option :simulate,
+      :type => :boolean,
+      :default => false,
+      :aliases => %w(-s)
+    option :verbose,
+      :type => :boolean,
+      :default => false,
+      :aliases => %w(-v)
     def parse
       start_time = Time.now
       count = 0
@@ -25,44 +59,61 @@ module SmsLogparser
         entries = mysql.get_entries
         api = Api.new(options)
         last_id = mysql.get_last_parse_id
+        status = STATUS[:ok]
         entries.each do |entry|
-          if Parser.match(entry['Message'])
+          if Parser.match?(entry['Message'])
             data = Parser.extract_data_from_msg(entry['Message'])
             begin
               urls = api.send(data)
             rescue => e
               say "Error: #{e.message}", :red
               say "Aborting parser run...", :red
+              status = STATUS[:api_error]
               break
             end
             last_id = entry['ID']
             count += 1
-            verbose_parser_output(data, urls, entry) if options[:verbose]
+            if options[:verbose]
+              verbose_parser_output(data, urls, entry)
+            end
           end
         end
-        mysql.write_parse_result(last_id, count) unless options[:simulate]
+        unless options[:simulate]
+          mysql.write_parse_result(last_id, count, status)
+        end
         say "Started:\t", :cyan
         say start_time.strftime('%d.%d.%Y %T')
         say "Runtime:\t", :cyan
         say "#{(Time.now - start_time).round(2)}s"
-        options[:simulate] ? say("Events found:\t", :cyan) : say("Events sent:\t", :cyan)
+        if options[:simulate]
+          say("Events found:\t", :cyan)
+        else
+          say("Events sent:\t", :cyan)
+        end
         say count
       rescue => e
         say "Error: #{e.message}", :red
       end
     end
 
-    desc "last_runs", "List the last paser runs"
-    def last_runs
+    desc "history", "List the last paser runs"
+    option :results,
+      :type => :numeric,
+      :default => 10,
+      :aliases => %w(-n),
+      :desc => "Number of results to display"
+    def history
+      puts options
       begin
-        runs = Mysql.new(options).last_runs
+        runs = Mysql.new(options).last_runs(options[:results])
         if runs.size > 0
-          table = [%w(RunAt #Events LastEventID)]
-          runs.each do |run|
+          table = [%w(RunAt #Events LastEventID Status)]
+          runs.to_a.reverse.each do |run|
             table << [
               run['RunAt'],
               run['EventsFound'],
-              run['LastEventID']
+              run['LastEventID'],
+              STATUS.index(run['Status']).upcase
             ]
           end
           print_table table
@@ -75,26 +126,46 @@ module SmsLogparser
     end
 
     desc "setup", "Create the parser table to track the last logs parsed"
+    option :force,
+      :type => :boolean,
+      :default => false,
+      :aliases => %w(-f),
+      :desc => "Drop an existing table if it exists"
     def setup
       begin
-        Mysql.new(options).create_parser_table
-        say "OK", :green
+        case Mysql.new(options).create_parser_table(options[:force])
+        when 0
+          say "OK, table created.", :green
+        when 1
+          say "Table already exists.", :yellow
+        end
       rescue => e
         say "Error: #{e.message}", :red
       end
     end
 
     no_commands do
+
       def verbose_parser_output(data, urls, entry)
         say "ID:\t", :cyan
         say entry['ID']
         say "URL:\t", :cyan
         say urls.join("\n\t")
         say "Data:\t", :cyan
-        say data.map{|k,v| "#{k}:\t#{v}"}.join("\n\t")
+        say data.map{|k,v| "#{k}:\t#{v}"}.join("\n\t") || "\n"
+        puts
         puts "-" * 100
         puts
       end
+
+      def options
+        original_options = super
+        filename = original_options[:config] || File.join(Dir.home, '.sms-logparser.yml')
+        return original_options unless File.exists?(filename)
+        defaults = ::YAML::load_file(filename) || {}
+        defaults.merge(original_options)
+      end
+
     end
 
   end

data/lib/sms-logparser/mysql.rb

@@ -14,27 +14,54 @@ module SmsLogparser
       )
     end
 
-    def last_runs
+    def last_runs(results = 10)
       begin 
         runs = client.query(
-          "SELECT * FROM SmsParserRuns ORDER BY ID ASC LIMIT 10"
+          "SELECT * FROM SmsParserRuns ORDER BY ID DESC LIMIT #{results}"
         )
       rescue Mysql2::Error => e
         raise e
       end
     end
 
-    def create_parser_table
+    def parser_table_exists?
       begin
         return client.query(
-          "CREATE TABLE IF NOT EXISTS\
-            SmsParserRuns(\
-              ID INT PRIMARY KEY AUTO_INCREMENT,\
-              RunAt datetime DEFAULT NULL,\
-              LastEventID INT DEFAULT NULL,\
-              EventsFound INT DEFAULT 0,\
-              INDEX `LastEventID_I1` (`LastEventID`)
-            )"
+          "SHOW TABLES LIKE 'SmsParserRuns'"
+        ).size > 0
+      rescue Mysql2::Error => e
+        raise e
+      end
+    end
+
+    def create_parser_table(force = false)
+      if force
+        drop_parser_table
+      elsif parser_table_exists?
+        return 1
+      end
+      begin
+        client.query(
+          "CREATE TABLE SmsParserRuns(\
+            ID SERIAL PRIMARY KEY AUTO_INCREMENT,\
+            RunAt datetime DEFAULT NULL,\
+            LastEventID BIGINT(20) UNSIGNED DEFAULT 0,\
+            EventsFound INT DEFAULT 0,\
+            Status TINYINT UNSIGNED DEFAULT 0,\
+            INDEX `LastEventID_I1` (`LastEventID`)
+          )"
+        )
+      rescue Mysql2::Error => e
+        raise e
+      end
+      return 0
+    end
+
+    def drop_parser_table
+      return nil unless parser_table_exists?
+      begin
+        return client.query(
+          "DROP TABLE SmsParserRuns"
         )
       rescue Mysql2::Error => e
         raise e
@@ -53,12 +80,13 @@ module SmsLogparser
       end
     end
 
-    def write_parse_result(id, count)
-      client.query("INSERT INTO SmsParserRuns(RunAt, LastEventID, EventsFound)\
+    def write_parse_result(id, count, status)
+      client.query("INSERT INTO SmsParserRuns(RunAt, LastEventID, EventsFound, Status)\
         VALUES(\
           '#{Time.now.strftime("%Y-%m-%d %H:%M:%S")}',\
           #{id},\
-          #{count}
+          #{count},\
+          #{status}
         )"
       )
     end

data/lib/sms-logparser/parser.rb

@@ -18,8 +18,14 @@ module SmsLogparser
       }
     end
 
-    def self.match(message)
-      message =~ /\/content\/.*\.(f4v|flv|mp4|mp3|ts|m3u8) .* (200|206)/
+    def self.match?(message)
+      match = message.match(
+        /\/content\/.+\/(\w+\.(f4v|flv|mp4|mp3|ts|m3u8)) .+ (200|206)/i
+      )
+      if match
+        return true unless match[1] =~ /detect.mp4|index.m3u8/i
+      end
+      false
     end
 
     # see https://developer.mozilla.org/en-US/docs/Browser_detection_using_the_user_agent

data/lib/sms-logparser/version.rb

@@ -1,3 +1,3 @@
 module SmsLogparser
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/sms-logparser.gemspec

@@ -18,6 +18,8 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
+  spec.required_ruby_version = '>= 1.9.3'
+  
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"
 

data/spec/api_spec.rb

@@ -0,0 +1,60 @@
+require 'spec_helper'
+
+describe SmsLogparser::Api do
+
+  before do
+    @api = SmsLogparser::Api.new(
+      simulate: true,
+      api_base_path: "http://myapi.com/rest/"
+    )
+  end
+
+  it "sends the correct information to the api" do
+    data = {
+      :customer_id => 1,
+      :author_id => 2,
+      :project_id => 3,
+      :file =>  'myfile.mp4',
+      :bytes => 128,
+      :user_agent => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko)",
+      :traffic_type => 'TRAFFIC_WEBCAST',
+      :visitor_type => 'VISITORS_WEBCAST'
+    }
+    urls = @api.send(data)
+    urls.size.must_equal 2
+    urls[0].must_match /.+\/1\/2\/3\/TRAFFIC_WEBCAST\/128$/
+    urls[1].must_match /.+\/1\/2\/3\/VISITORS_WEBCAST\/1$/
+  end
+
+  it "does not send traffic for m3u8 files" do
+    data = {
+      :customer_id => 100,
+      :author_id => 200,
+      :project_id => 300,
+      :file =>  'myfile.m3u8',
+      :bytes => 512,
+      :user_agent => 'Mozilla/5.0 (Android; Mobile; rv:13.0) Gecko/13.0 Firefox/13.0',
+      :traffic_type => 'TRAFFIC_MOBILE',
+      :visitor_type => 'VISITORS_MOBILE'
+    }
+    urls = @api.send(data)
+    urls.size.must_equal 1
+    urls[0].must_match /.+\/100\/200\/300\/VISITORS_MOBILE\/1$/
+  end
+
+  it "does not send visitor info if no visitor_type" do
+    data = {
+      :customer_id => 101,
+      :author_id => 202,
+      :project_id => 303,
+      :file =>  'myfile.mp4',
+      :bytes => 48,
+      :user_agent => 'Mozilla/5.0 (Android; Mobile; rv:13.0) Gecko/13.0 Firefox/13.0',
+      :traffic_type => 'TRAFFIC_MOBILE',
+    }
+    urls = @api.send(data)
+    urls.size.must_equal 1
+    urls[0].must_match /.+\/101\/202\/303\/TRAFFIC_MOBILE\/48$/
+  end
+
+end

data/spec/cli_spec.rb

@@ -14,7 +14,7 @@ describe SmsLogparser::Cli do
     out, err = capture_io do
       TestHelper.sms_logparser.setup
     end
-    out.must_match /OK/
+    out.must_match /OK.*/
   end
 
   it "can parse a log database and find matches" do
@@ -40,15 +40,15 @@ describe SmsLogparser::Cli do
     out.must_match /\s+0$/
   end
 
-  it "lists parser runs" do
+  it "can show the parser history" do
     TestHelper.seed_db(1)
     parser = TestHelper.sms_logparser
     parser.options[:api_base_path] = 'http://devnull-as-a-service.com/dev/null/'
     out, err = capture_io do
       TestHelper.sms_logparser.setup
       parser.parse
-      parser.last_runs 
+      parser.history 
     end
-    assert_equal(err, "")
+    err.must_equal ""
   end
 end

data/spec/parser_spec.rb

@@ -4,41 +4,49 @@ describe SmsLogparser::Parser do
 
   %w(f4v flv mp4 mp3 ts m3u8).each do |extension|
     it "matches #{extension} files" do
-      SmsLogparser::Parser.match(
+      SmsLogparser::Parser.match?(
         "GET /content/2/719/54986/file.#{extension} HTTP/1.1\" 200 6741309 "
-      ).wont_be_nil
+      ).must_equal true
     end
   end
 
   %w(jpg js css m4a docx).each do |extension|
     it "does not matche #{extension} files" do
-      SmsLogparser::Parser.match(
+      SmsLogparser::Parser.match?(
         "GET /content/2/719/54986/file.#{extension} HTTP/1.1\" 200 6741309 "
-      ).must_be_nil
+      ).must_equal false
     end
   end
 
   %w(200 206).each do |status|
     it "does match status code #{status}" do
-      SmsLogparser::Parser.match(
+      SmsLogparser::Parser.match?(
         "GET /content/2/719/54986/file.mp4 HTTP/1.1\" #{status} 50000 "
-      ).wont_be_nil
+      ).must_equal true
     end
   end
 
   %w(404 500 304).each do |status|
     it "does not match status code #{status}" do
-      SmsLogparser::Parser.match(
+      SmsLogparser::Parser.match?(
         "GET /content/2/719/54986/file.mp4 HTTP/1.1\" #{status} 50000 "
-      ).must_be_nil
+      ).must_equal false
     end
   end
 
-  %w(contents public index CONTENT).each do |dir|
+  %w(contents public index assets).each do |dir|
     it "does not match directories other than /content" do
-      SmsLogparser::Parser.match(
+      SmsLogparser::Parser.match?(
         "GET /#{dir}/2/719/54986/file.mp4 HTTP/1.1\" 200 50000 "
-      ).must_be_nil
+      ).must_equal false
+    end
+  end
+
+  %w(detect.mp4 index.m3u8).each do |file|
+    it "does not match excluded files" do
+      SmsLogparser::Parser.match?(
+        "GET /content/2/719/54986/#{file} HTTP/1.1\" 200 128 "
+      ).must_equal false
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sms-logparser
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - niwo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-25 00:00:00.000000000 Z
+date: 2014-03-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -101,6 +101,7 @@ files:
 - lib/sms-logparser/parser.rb
 - lib/sms-logparser/version.rb
 - sms-logparser.gemspec
+- spec/api_spec.rb
 - spec/cli_spec.rb
 - spec/parser_spec.rb
 - spec/spec_helper.rb
@@ -116,7 +117,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - '>='
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.9.3
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - '>='
@@ -129,6 +130,7 @@ signing_key:
 specification_version: 4
 summary: SMS Logparser
 test_files:
+- spec/api_spec.rb
 - spec/cli_spec.rb
 - spec/parser_spec.rb
 - spec/spec_helper.rb