sms-logparser 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2e9efb4712d1d9d54c9904f57993ced13742bd51
-  data.tar.gz: 2fa877596b733aac7a13a27cd76ac22902978564
+  metadata.gz: 484db0ec68fb533d509b336f6d5258947a1550cb
+  data.tar.gz: c826cb94003ff3beea5b579e1967a4f3a45cd548
 SHA512:
-  metadata.gz: ecb5588a86ae77f592f67117172b4b1d259fec82d7bd6e80305c70de60032db646c8751734eed245d5d90867a1e875bf08e366465a1db23dade2400df1744b61
-  data.tar.gz: 1202cb4ff8e9d3820682e5cb6bbc0b032c0a5168dfc32c963915b5055f5bfe7a344e64870eb10a1d972e3d0f413e236b5e243f1fcd7b06638b94ea7ab6e5f098
+  metadata.gz: f1638d239b7bd9ef872f92b2c87271de78eb6ffe46b187609c132607adb7921f2637d0605f3a101c9ae808678b0fe2f995d4dcaff5d868c5e145d8e50f019441
+  data.tar.gz: 53b8660b214c5a8cd97de7160372b8b5082a5b13c851db48702aae481f0e2151de7118b9fb4024e6979d4c33b318f33988b108625248842ef40e8c7669b0a210

data/README.md

@@ -20,11 +20,11 @@ Or install it yourself as:
 
 Create the database table to track which logs have been parsed:
 
-    $ sms-logparser create_parser_table
+    $ sms-logparser setup
 
 Make a test run:
 
-    $ sms-logparser parse --simulate
+    $ sms-logparser parse --simulate --verbose
 
 ## Usage
 
@@ -40,6 +40,13 @@ Show the last parser runs:
 
     $ sms-logparser last_runs
 
+# Development
+
+  - check out the git repo (git clone ...)
+  - implement your changes
+  - run the tests `rake test`
+  - bump the version number commit your changes and release a new version of the gem `rake release`
+
 ## Contributing
 
 1. Fork it

data/lib/sms-logparser/api.rb

@@ -6,20 +6,22 @@ module SmsLogparser
     end
 
     def send(data)
-      url = "#{@options[:api_base_path]}/"
-      url += "#{data[:customer_id]}/"
-      url += "#{data[:author_id]}/"
-      url += "#{data[:project_id]}/"
-      url += "#{data[:traffic_type]}/"
-      url += "#{data[:bytes]}"
+      base_url = "#{@options[:api_base_path]}/"
+      base_url += "#{data[:customer_id]}/"
+      base_url += "#{data[:author_id]}/"
+      base_url += "#{data[:project_id]}"
+      urls = ["#{base_url}/#{data[:traffic_type]}/#{data[:bytes]}"]
+      urls << "#{base_url}/#{data[:visitor_type]}/1" if data[:visitor_type]
       unless @options[:simulate]
-        begin
-          RestClient.get(url)
-        rescue
-          raise "Can't send log to #{url}"
+        urls.each do |url|
+          begin
+            RestClient.get(url)
+          rescue
+            raise "Can't send request to #{url}"
+          end
         end
       end
-      url
+      urls
     end
 
   end # class

data/lib/sms-logparser/cli.rb

@@ -1,9 +1,10 @@
 module SmsLogparser
   class Cli < Thor
 
-    class_option :mysql_host, :default => 'localhost'
-    class_option :mysql_user, :default => 'root'
-    class_option :mysql_db, :default => 'Syslog'
+    class_option :mysql_host, :default => 'localhost', aliases: %w(-h)
+    class_option :mysql_user, :default => 'root', aliases: %w(-u)
+    class_option :mysql_password, aliases: %w(-p)
+    class_option :mysql_db, :default => 'Syslog', aliases: %w(-d)
 
     desc "version", "print cloudstack-cli version number"
     def version
@@ -12,8 +13,9 @@ module SmsLogparser
     map %w(-v --version) => :version
 
     desc "parse", "Check the database for pcache logs and send them to SMS"
-    option :api_base_path, :default => 'http://dev.simplex.tv/creator/rest'
-    option :simulate, :type => :boolean, :default => false
+    option :api_base_path, :default => 'http://dev.simplex.tv/creator/rest', aliases: %w(-a)
+    option :simulate, :type => :boolean, :default => false, aliases: %w(-s)
+    option :verbose, :type => :boolean, :default => false, aliases: %w(-v)
     def parse
       start_time = Time.now
       count = 0
@@ -23,18 +25,27 @@ module SmsLogparser
         api = Api.new(options)
         last_id = mysql.get_last_parse_id
         entries.each do |entry|
-          if Parser.match(entry)
+          if Parser.match(entry['Message'])
             data = Parser.extract_data_from_msg(entry['Message'])
-            url = api.send(data)
+            begin
+              urls = api.send(data)
+            rescue => e
+              say "Error: #{e.message}", :red
+              say "Aborting parser run...", :red
+              break
+            end
             last_id = entry['ID']
             count += 1
-            debug_parser_output(data, url, entry) if options[:simulate]
+            verbose_parser_output(data, urls, entry) if options[:verbose]
           end
         end
         mysql.write_parse_result(last_id, count) unless options[:simulate]
-        puts "Started: #{start_time.strftime('%d.%d.%Y %T')}"
-        puts "Runtime: #{(Time.now - start_time).round(2)}s"
-        puts "Matches: #{count}"
+        say "Started:\t", :cyan
+        say start_time.strftime('%d.%d.%Y %T')
+        say "Runtime:\t", :cyan
+        say "#{(Time.now - start_time).round(2)}s"
+        options[:simulate] ? say("Events found:\t", :cyan) : say("Events sent:\t", :cyan)
+        say count
       rescue => e
         say "Error: #{e.message}", :red
       end
@@ -73,14 +84,14 @@ module SmsLogparser
     end
 
     no_commands do
-      def debug_parser_output(data, url, entry)
-        puts
-        say "Message ID: ", :green
+      def verbose_parser_output(data, urls, entry)
+        say "ID:\t", :cyan
         say entry['ID']
-        say "URL: ", :green
-        say url
-        say "Data: ", :green
-        say data
+        say "URL:\t", :cyan
+        say urls.join("\n\t")
+        say "Data:\t", :cyan
+        say data.map{|k,v| "#{k}:\t#{v}"}.join("\n\t")
+        puts "-" * 100
         puts
       end
     end

data/lib/sms-logparser/mysql.rb

@@ -9,6 +9,7 @@ module SmsLogparser
       @client ||= Mysql2::Client.new(
         :host => @options[:mysql_host],
         :username => @options[:mysql_user],
+        :password => @options[:mysql_password],
         :database => @options[:mysql_db]
       )
     end

data/lib/sms-logparser/parser.rb

@@ -1,34 +1,55 @@
 module SmsLogparser
   class Parser
 
-    def self.extract_data_from_msg(msg)
-      m = msg.match /\/content\/(\d+)\/(\d+)\/(\d+)\/.+\.(\S+)\s.*\"\s\d+\s(\d+).+"(.*)"$/
+    def self.extract_data_from_msg(message)
+      m = message.match /\/content\/(\d+)\/(\d+)\/(\d+)\/(\w+\.\w+)\s.*\"\s\d+\s(\d+).+"(.*)"$/
       raise "No match found." unless m
+      traffic_type = Parser.get_traffic_type(m[6])
+      visitor_type = Parser.get_visitor_type(traffic_type, m[4])
       data = {
         :customer_id => m[1],
         :author_id => m[2],
         :project_id => m[3],
-        :ext => m[4],
-        :traffic_type => Parser.get_traffic_type(m[4]),
+        :file =>  m[4],
         :bytes => m[5],
-        :user_agent => m[6]
+        :user_agent => m[6],
+        :traffic_type => traffic_type,
+        :visitor_type => visitor_type
       }
     end
 
-    def self.match(entry)
-      entry['Message'] =~ /\/content\/.*\.(f4v|flv|mp4|mp3|ts) .*/
+    def self.match(message)
+      message =~ /\/content\/.*\.(f4v|flv|mp4|mp3|ts|m3u8) .* (200|206)/
     end
 
+    # see https://developer.mozilla.org/en-US/docs/Browser_detection_using_the_user_agent
+    # for mobile browser detection
     def self.get_traffic_type(user_agent)
       case user_agent
       when /.*(iTunes).*/
         "TRAFFIC_PODCAST"
-      when /.*(IEMobile|Mobile Safari|iPhone|iPod|iPad).*/
+      when /.*(Mobi|IEMobile|Mobile Safari|iPhone|iPod|iPad|Android|BlackBerry|Opera Mini).*/
         "TRAFFIC_MOBILE"
       else
         "TRAFFIC_WEBCAST"
       end
     end
 
+    def self.get_visitor_type(traffic_type, file)
+      return "VISITORS_MOBILE" if file == 'index.m3u8'
+      case traffic_type
+      when "TRAFFIC_PODCAST"
+        "VISITORS_PODCAST"
+      when "TRAFFIC_MOBILE"
+        if File.extname(file) != ".ts"
+          "VISITORS_MOBILE"
+        else
+          nil
+        end
+      else
+        "VISITORS_WEBCAST"
+      end
+    end
+
   end # class
 end # module

data/lib/sms-logparser/version.rb

@@ -1,3 +1,3 @@
 module SmsLogparser
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/spec/cli_spec.rb

@@ -25,7 +25,7 @@ describe SmsLogparser::Cli do
       TestHelper.sms_logparser.setup
       parser.parse  
     end
-    out.must_match /.*Matches: 10$/
+    out.must_match /\s+10$/
   end
 
   it "skips over already parsed logs" do
@@ -37,7 +37,7 @@ describe SmsLogparser::Cli do
       parser.parse
       parser.parse  
     end
-    out.must_match /.*Matches: 0$/
+    out.must_match /\s+0$/
   end
 
   it "lists parser runs" do

data/spec/parser_spec.rb

@@ -0,0 +1,82 @@
+require 'spec_helper'
+
+describe SmsLogparser::Parser do
+
+  %w(f4v flv mp4 mp3 ts m3u8).each do |extension|
+    it "matches #{extension} files" do
+      SmsLogparser::Parser.match(
+        "GET /content/2/719/54986/file.#{extension} HTTP/1.1\" 200 6741309 "
+      ).wont_be_nil
+    end
+  end
+
+  %w(jpg js css m4a docx).each do |extension|
+    it "does not matche #{extension} files" do
+      SmsLogparser::Parser.match(
+        "GET /content/2/719/54986/file.#{extension} HTTP/1.1\" 200 6741309 "
+      ).must_be_nil
+    end
+  end
+
+  %w(200 206).each do |status|
+    it "does match status code #{status}" do
+      SmsLogparser::Parser.match(
+        "GET /content/2/719/54986/file.mp4 HTTP/1.1\" #{status} 50000 "
+      ).wont_be_nil
+    end
+  end
+
+  %w(404 500 304).each do |status|
+    it "does not match status code #{status}" do
+      SmsLogparser::Parser.match(
+        "GET /content/2/719/54986/file.mp4 HTTP/1.1\" #{status} 50000 "
+      ).must_be_nil
+    end
+  end
+
+  %w(contents public index CONTENT).each do |dir|
+    it "does not match directories other than /content" do
+      SmsLogparser::Parser.match(
+        "GET /#{dir}/2/719/54986/file.mp4 HTTP/1.1\" 200 50000 "
+      ).must_be_nil
+    end
+  end
+
+  [
+    "Mozilla/5.0(iPad; U; CPU iPhone OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B314 Safari/531.21.10",
+    "Mozilla/5.0 (Android; Mobile; rv:13.0) Gecko/13.0 Firefox/13.0",
+    "Opera/9.80 (Android 2.3.3; Linux; Opera Mobi/ADR-1111101157; U; es-ES) Presto/2.9.201 Version/11.50",
+    "Mozilla/5.0 (Linux; Android 4.4.2); Nexus 5 Build/KOT49H) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.117 Mobile Safari/537.36 OPR/20.0.1396.72047",
+    "Mozilla/5.0 (compatible; MSIE 9.0; Windows Phone OS 7.5; Trident/5.0; IEMobile/9.0)",
+    "Mozilla/5.0 (iPhone; U; CPU iPhone OS 5_1_1 like Mac OS X; en) AppleWebKit/534.46.0 (KHTML, like Gecko) CriOS/19.0.1084.60 Mobile/9B206 Safari/7534.48.3"
+  ].each do |mobile_agent|
+    it "traffic type for mobile user agents is TRAFFIC_MOBILE (#{mobile_agent})" do
+      SmsLogparser::Parser.get_traffic_type(mobile_agent).must_equal "TRAFFIC_MOBILE"
+    end
+  end
+
+  it "should set visitor_type to VISITORS_MOBILE for index.m3u8 files" do
+    SmsLogparser::Parser.get_visitor_type(
+      "TRAFFIC_PODCAST", "index.m3u8"
+    ).must_equal "VISITORS_MOBILE"
+  end
+
+  it "should NOT set visitor_type to VISITORS_MOBILE for TRAFFIC_MOBILE and .ts files" do
+    SmsLogparser::Parser.get_visitor_type(
+      "TRAFFIC_MOBILE", "file.ts"
+    ).must_be_nil
+  end
+
+  it "should set visitor_type to VISITORS_MOBILE for TRAFFIC_MOBILE and file not .ts" do
+    SmsLogparser::Parser.get_visitor_type(
+      "TRAFFIC_MOBILE", "file.mp3"
+    ).must_equal "VISITORS_MOBILE"
+  end
+
+  it "should set visitor_type to VISITORS_PODCAST for TRAFFIC_PODCAST" do
+    SmsLogparser::Parser.get_visitor_type(
+      "TRAFFIC_PODCAST", "file.mp4"
+    ).must_equal "VISITORS_PODCAST"
+  end
+
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sms-logparser
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - niwo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-13 00:00:00.000000000 Z
+date: 2014-03-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -102,6 +102,7 @@ files:
 - lib/sms-logparser/version.rb
 - sms-logparser.gemspec
 - spec/cli_spec.rb
+- spec/parser_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/swisstxt/sms-logparser
 licenses:
@@ -129,4 +130,5 @@ specification_version: 4
 summary: SMS Logparser
 test_files:
 - spec/cli_spec.rb
+- spec/parser_spec.rb
 - spec/spec_helper.rb