sms-logparser 0.14.1 → 0.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 146a3d01abb1ef42fd0c65a5ab993616f9ea20bf
-  data.tar.gz: b066180907bca69e2a76deed5aa500104d209a2d
+  metadata.gz: 7ad478bf9a5f80ba72ab9e0f48303ce03e850511
+  data.tar.gz: 95d931e94749fcdb7876a7ecece52e4ce225675e
 SHA512:
-  metadata.gz: eaf673b707de500d4dc316b21cf2b045e8d2c784551b1236d80aca4a914b667a0a3b16bc7ed5647f9f605661fff5e2bb50615a1ad886638ac1be36d53f4ca219
-  data.tar.gz: a82b49914a4357543c733f3b91089dd5b3923813bee3273bd223cab1072d2cd4bdbe46ea85102154b460b19dca7e30d151cc403e981a4db36c8ae3aff7e7eb20
+  metadata.gz: 19b52cef0cf0181912da7075cb91deb76f9f7a7bd2aeba49ce6409ed730ecc623a0c67927ab059a67bfb1ab31270201fa33c449c5b152470595ba8c1798404fe
+  data.tar.gz: 4cc3e8d1135625a43129d5f6f9a460b97e1fa20e7d55671b17017ebf70a9becc4e7b00b0908a3bc48de075895e8683e0483112eea4937784e533ae0bb1c9a6a4

data/lib/sms-logparser.rb

@@ -9,4 +9,5 @@ require "sms-logparser/parser"
 require "sms-logparser/api"
 require "sms-logparser/loggster"
 require "sms-logparser/data_cache"
+require "sms-logparser/log_message"
 require "sms-logparser/cli"

data/lib/sms-logparser/api.rb

@@ -14,25 +14,22 @@ module SmsLogparser
     end
 
     def send(data)
-      requests = build_urls(data)
-      return requests if @options[:simulate]
-
-      threads = requests.map do |request|
-        Thread.new do
-          begin
-            response = @connection.post(request[:uri])
-            request[:status] = response.status
-          rescue => e
-            raise RuntimeError, "Can't send request to #{request[:uri]}. #{e.message}", caller
-          end
-          unless @accepted_responses.include?(response.status)
-            msg = "Received HTTP status #{response.status} from API. Only accepting #{@accepted_responses.join(', ')}."
-            raise RuntimeError, msg, caller
-          end
+      path = data_to_path(data)
+      begin
+        if @options[:simulate]
+          status = 200
+        else
+          response = @connection.post(path)
+          status = response.status
         end
+      rescue => e
+        raise RuntimeError, "Can't send request to #{path}. #{e.message}", caller
       end
-      threads.each {|thread| thread.join }
-      requests
+      unless @accepted_responses.include?(status)
+        msg = "Received HTTP status #{status} from API. Only accepting #{@accepted_responses.join(', ')}."
+        raise RuntimeError, msg, caller
+      end
+      return path, status
     end
 
     def send_sets(data_sets, concurrency=4)
@@ -42,30 +39,9 @@ module SmsLogparser
       threads = concurrency.times.map do
         Thread.new do
           while !queue.empty?
-            begin
-              data = queue.pop
-              url = @base_path + [
-                data[:customer_id],
-                data[:author_id],
-                data[:project_id],
-                data[:type],
-                data[:value]
-              ].join('/')
-              if @options[:simulate]
-                status = 200
-              else
-                response = @connection.post(url)
-                status = response.status
-              end
-            rescue => e
-              raise RuntimeError, "Can't send request to #{url}. #{e.message}", caller
-            end
-            unless @accepted_responses.include?(status)
-              msg = "Received HTTP status #{status} from API. Only accepting #{@accepted_responses.join(', ')}."
-              raise RuntimeError, msg, caller
-            end
+            path, status = send(queue.pop)
             semaphore.synchronize {
-              yield url, status
+              yield path, status
             }
           end
         end
@@ -73,22 +49,14 @@ module SmsLogparser
       threads.each {|thread| thread.join }
     end
 
-    def build_urls(data)
-      requests = []
-      path = @base_path + [data[:customer_id], data[:author_id], data[:project_id]].join('/')
-      unless data[:file] =~ /.*\.m3u8$/
-        requests << {
-          url: @url, 
-          uri: [path, data[:traffic_type], data[:bytes]].join('/'),
-        }
-      end
-      if data[:visitor_type]
-        requests << {
-          url: @url,
-          uri: [path, data[:visitor_type], 1].join('/')
-        }
-      end
-      requests
+    def data_to_path(data)
+      @base_path + [
+        data[:customer_id],
+        data[:author_id],
+        data[:project_id],
+        data[:type],
+        data[:value]
+      ].join('/')
     end
 
     private

data/lib/sms-logparser/cli.rb

@@ -69,13 +69,15 @@ module SmsLogparser
         logger.info { "Getting log messages from database..." }
         entries.each do |entry| 
           parser.extract_data_from_msg(entry['Message']) do |data|
-            if data
-              if options[:accumulate]
-                cache.add(data)
-                logger.debug {"Cached data: #{data}"}
-              else
-                requests = api.send(data)
-                verbose_parser_output(entry['ID'], data, requests)
+            if data.size > 0
+              data.each do |data_entry|
+                if options[:accumulate]
+                  cache.add(data_entry)
+                  logger.debug {"Cached data: #{data_entry}"}
+                else
+                  url, status = api.send(data_entry)
+                  verbose_parser_output(entry['ID'], data_entry, url, status)
+                end
               end
               state[:last_event_id] = entry['ID']
               state[:match_count] += 1
@@ -85,9 +87,9 @@ module SmsLogparser
       end
       if options[:accumulate]
         resp_stats = {}
-        api.send_sets(cache.data_sets, options[:concurrency]) do |url, response|
-          logger.debug { "POST #{url} (#{response})" }
-          resp_stats[response] = resp_stats[response].to_i + 1
+        api.send_sets(cache.data_sets, options[:concurrency]) do |url, status|
+          logger.debug { "POST #{url} (#{status})" }
+          resp_stats[status] = resp_stats[status].to_i + 1
         end
         logger.info { "Usage commited: #{resp_stats.map {|k,v| "#{v} x status #{k}" }.join(" : ")}" }
       end
@@ -195,15 +197,11 @@ module SmsLogparser
         SmsLogparser::Loggster.instance.set_log_device options[:logfile]
       end
 
-      def verbose_parser_output(entry_id, data, requests)
+      def verbose_parser_output(entry_id, data, url, status)
         logger.debug {
-          "parsing data for #{entry_id} (#{data.map{|k,v| "#{k}=\"#{v || '-'}\""}.join(" ") || ''})"
+          "Parsing data for #{entry_id} (#{data.map{|k,v| "#{k}=\"#{v || '-'}\""}.join(" ") || ''})"
         }
-        requests.each_with_index do |req, i|
-          logger.debug {
-            "URL #{i + 1} for entry #{entry_id} #{req[:url]}#{req[:uri]}"
-          }
-        end
+        logger.debug {"URL for entry #{entry_id}: #{url} (#{status})"}
       end
 
       def table_to_csv(table)

data/lib/sms-logparser/data_cache.rb

@@ -5,42 +5,25 @@ module SmsLogparser
 
     def initialize
       @cache = Hash.new
-      @wanted_keys = [:customer_id, :author_id, :project_id]
     end
 
     def add(data)
-      key = [data[:customer_id], data[:author_id], data[:project_id]].join('.')
-      @cache[key] = initialize_value(data) unless @cache.has_key?(key)
-      unless data[:file] =~ /.*\.m3u8$/
-        @cache[key][data[:traffic_type]] = @cache[key][data[:traffic_type]].to_i + data[:bytes].to_i
-      end
-      if data[:visitor_type]
-        @cache[key][data[:visitor_type]] = @cache[key][data[:visitor_type]].to_i + 1
-      end
+      key = [data[:customer_id], data[:author_id], data[:project_id], data[:type]].join('.')
+      @cache[key] = @cache[key].to_i + data[:value].to_i
       @cache
     end
 
     def data_sets
-      sets = []
-      types = %w(TRAFFIC_PODCAST TRAFFIC_MOBILE TRAFFIC_WEBCAST VISITORS_PODCAST VISITORS_MOBILE VISITORS_WEBCAST)
-      @cache.each do |key, values|
-        types.each do |type|
-          sets << {
-            customer_id: values[:customer_id],
-            author_id: values[:author_id],
-            project_id: values[:project_id],
-            type: type,
-            value: values[type]
-          } if values[type]
-        end
+      @cache.map do |key, value|
+        key_components = key.split('.')
+        {
+          customer_id: key_components[0],
+          author_id: key_components[1],
+          project_id: key_components[2],
+          type: key_components[3],
+          value: value
+        }
       end
-      sets
-    end
-
-    private
-
-    def initialize_value(data)
-      data.select { |key,_| @wanted_keys.include? key }
     end
 
   end # class

data/lib/sms-logparser/log_message.rb

@@ -0,0 +1,67 @@
+module SmsLogparser
+  class LogMessage
+    def initialize(message)
+      @message = message
+    end
+
+    def customer_id
+      match[1]
+    end
+
+    def author_id
+      match[2]
+    end
+              
+    def project_id
+      match[3]
+    end
+
+    def status
+      match[5].to_i
+    end
+
+    def bytes
+      match[6].to_i
+    end
+
+    def file
+      match[4]
+    end
+
+    def file_extname
+      File.extname(file)
+    end
+    
+    def user_agent
+      match[7]
+    end
+
+    def account_info
+      {
+        customer_id: customer_id,
+        author_id: author_id,
+        project_id: project_id
+      }
+    end
+
+    def transfer_info
+      {
+        status: status,
+        bytes: bytes,
+        file: file,
+        file_extname: file_extname,
+        user_agent: user_agent
+      }
+    end
+
+    def to_h
+      account_info.merge(transfer_info)
+    end
+
+    private 
+
+    def match
+      @match ||= @message.match /\/content\/(\d+)\/(\d+)\/(\d+)\/(\w+\.\w+)\s.*\"\s(\d+)\s(\d+).+"(.*)"$/
+    end
+  end
+end 

data/lib/sms-logparser/parser.rb

@@ -1,29 +1,28 @@
 module SmsLogparser
   class Parser
 
-    def initialize(options)
+    def initialize(options = {})
       @options = options
       @logger = SmsLogparser::Loggster.instance
     end
 
     def extract_data_from_msg(message)
-      data = nil
+      data = []
       if Parser.match?(message)
         @logger.debug { "Parser MATCH: #{message}" }
-        match = message.match /\/content\/(\d+)\/(\d+)\/(\d+)\/(\w+\.\w+)\s.*\"\s\d+\s(\d+).+"(.*)"$/
-        if match
-          traffic_type = Parser.get_traffic_type(match[6])
-          visitor_type = Parser.get_visitor_type(traffic_type, match[4])
-          data = {
-            :customer_id => match[1],
-            :author_id => match[2],
-            :project_id => match[3],
-            :file =>  match[4],
-            :bytes => (match[5].to_f * traffic_correction_factor(traffic_type)).round(0),
-            :user_agent => match[6],
-            :traffic_type => traffic_type,
-            :visitor_type => visitor_type
-          }
+        log_message = LogMessage.new(message)
+        type = Parser.get_type(log_message.user_agent)
+        data << log_message.account_info.merge(
+          type: "TRAFFIC_#{type}",
+          value: (log_message.bytes * traffic_correction_factor(type)).round(0)
+        )
+        if log_message.status == 200 &&
+          (log_message.file_extname =~ /\.(mp3|mp4|flv|f4v)/ ||
+           log_message.file == 'index.m3u8')
+          data << log_message.account_info.merge(
+            type: "VISITORS_#{type}",
+            value: 1,
+          )
         end
       else
         @logger.debug { "Parser IGNORE: #{message}" }
@@ -32,52 +31,40 @@ module SmsLogparser
       yield data
     end
 
-    def traffic_correction_factor(traffic_type)
-      factor = case traffic_type
-      when 'TRAFFIC_WEBCAST'
-        @options[:webcast_traffic_correction] || 1.0
-      when 'TRAFFIC_MOBILE'
-        @options[:mobile_traffic_correction] || 1.0
-      when 'TRAFFIC_PODCAST'
-        @options[:podcast_traffic_correction] || 1.0
-      else
-        1.0
-      end
-      factor.to_f
-    end
-
     def self.match?(message)
       match = message.match(/\/content\/.+\/(\S+) .+ (200|206)/i)
-      # ignore detect.mp4 and index.m3u8 
+      # ignore detect.mp4 
       if match
-        return true unless match[1] =~ /detect.mp4|index.m3u8/i
+        return true unless match[1] =~ /detect.mp4/i
       end
       false
     end
 
     # see https://developer.mozilla.org/en-US/docs/Browser_detection_using_the_user_agent
     # for mobile browser detection
-    def self.get_traffic_type(user_agent)
+    def self.get_type(user_agent)
       case user_agent
       when /.*(iTunes).*/i
-        'TRAFFIC_PODCAST'
+        'PODCAST'
       when /.*(Mobi|IEMobile|Mobile Safari|iPhone|iPod|iPad|Android|BlackBerry|Opera Mini).*/
-        'TRAFFIC_MOBILE'
+        'MOBILE'
       else
-        'TRAFFIC_WEBCAST'
+        'WEBCAST'
       end
     end
 
-    def self.get_visitor_type(traffic_type, file)
-      return 'VISITORS_MOBILE' if File.extname(file) == '.m3u8'
-      case traffic_type
-      when 'TRAFFIC_PODCAST'
-        'VISITORS_PODCAST'
-      when 'TRAFFIC_MOBILE'
-        File.extname(file) != '.ts' ? 'VISITORS_MOBILE' : nil
+    def traffic_correction_factor(traffic_type)
+      factor = case traffic_type
+      when 'WEBCAST'
+        @options[:webcast_traffic_correction] || 1.0
+      when 'MOBILE'
+        @options[:mobile_traffic_correction] || 1.0
+      when 'PODCAST'
+        @options[:podcast_traffic_correction] || 1.0
       else
-        'VISITORS_WEBCAST'
+        1.0
       end
+      factor.to_f
     end
 
   end # class

data/lib/sms-logparser/version.rb

@@ -1,3 +1,3 @@
 module SmsLogparser
-  VERSION = "0.14.1"
+  VERSION = "0.15.0"
 end

data/spec/api_spec.rb

@@ -9,52 +9,26 @@ describe SmsLogparser::Api do
     )
   end
 
-  it "sends the correct information to the api" do
+  it "builds the correct path for TRAFFIC data" do
     data = {
-      :customer_id => 1,
-      :author_id => 2,
-      :project_id => 3,
-      :file =>  'myfile.mp4',
-      :bytes => 128,
-      :user_agent => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko)",
-      :traffic_type => 'TRAFFIC_WEBCAST',
-      :visitor_type => 'VISITORS_WEBCAST'
+      customer_id: 1,
+      author_id: 2,
+      project_id: 3,
+      value: 128,
+      type: 'TRAFFIC_WEBCAST',
     }
-    requests = @api.send(data)
-    requests.size.must_equal 2
-    requests[0][:uri].must_match /\/1\/2\/3\/TRAFFIC_WEBCAST\/128$/
-    requests[1][:uri].must_match /\/1\/2\/3\/VISITORS_WEBCAST\/1$/
+    @api.data_to_path(data).must_match /\/1\/2\/3\/TRAFFIC_WEBCAST\/128$/
   end
 
-  it "does not send traffic for m3u8 files" do
+  it "builds the correct path for VISITOR data" do
     data = {
-      :customer_id => 100,
-      :author_id => 200,
-      :project_id => 300,
-      :file =>  'myfile.m3u8',
-      :bytes => 512,
-      :user_agent => 'Mozilla/5.0 (Android; Mobile; rv:13.0) Gecko/13.0 Firefox/13.0',
-      :traffic_type => 'TRAFFIC_MOBILE',
-      :visitor_type => 'VISITORS_MOBILE'
+      customer_id: 101,
+      author_id: 202,
+      project_id: 303,
+      type: 'TRAFFIC_MOBILE',
+      value: 48
     }
-    requests = @api.send(data)
-    requests.size.must_equal 1
-    requests[0][:uri].must_match /\/100\/200\/300\/VISITORS_MOBILE\/1$/
-  end
-
-  it "does not send visitor info if no visitor_type" do
-    data = {
-      :customer_id => 101,
-      :author_id => 202,
-      :project_id => 303,
-      :file =>  'myfile.mp4',
-      :bytes => 48,
-      :user_agent => 'Mozilla/5.0 (Android; Mobile; rv:13.0) Gecko/13.0 Firefox/13.0',
-      :traffic_type => 'TRAFFIC_MOBILE',
-    }
-    requests = @api.send(data)
-    requests.size.must_equal 1
-    requests[0][:uri].must_match /\/101\/202\/303\/TRAFFIC_MOBILE\/48$/
+    @api.data_to_path(data).must_match /\/101\/202\/303\/TRAFFIC_MOBILE\/48$/
   end
 
 end

data/spec/data_cache_spec.rb

@@ -0,0 +1,28 @@
+require 'spec_helper'
+
+describe SmsLogparser::DataCache do
+
+  before do
+    @cache = SmsLogparser::DataCache.new
+  end
+
+  it "returns correct totals from cache" do
+    @cache.add(customer_id: 101, author_id: 202, project_id: 303, type: 'TRAFFIC_MOBILE', value: 10)
+    @cache.add(customer_id: 101, author_id: 202, project_id: 303, type: 'TRAFFIC_MOBILE', value: 400)
+    @cache.add(customer_id: 101, author_id: 202, project_id: 303, type: 'TRAFFIC_MOBILE', value: 1000)
+
+    @cache.add(customer_id: 101, author_id: 202, project_id: 303, type: 'TRAFFIC_WEBCAST', value: 1000)
+
+    1000.times do 
+      @cache.add(customer_id: 101, author_id: 300, project_id: 303, type: 'VISIT_WEBCAST', value: 1)
+    end
+
+    @cache.add(customer_id: 1, author_id: 10, project_id: 600, type: 'TRAFFIC_MOBILE', value: 500)
+
+    @cache.data_sets.must_include(customer_id: "101", author_id: "202", project_id: "303", type: "TRAFFIC_MOBILE", value: 1410)
+    @cache.data_sets.must_include(customer_id: "101", author_id: "202", project_id: "303", type: "TRAFFIC_WEBCAST", value: 1000)
+    @cache.data_sets.must_include(customer_id: "101", author_id: "300", project_id: "303", type: "VISIT_WEBCAST", value: 1000)
+    @cache.data_sets.must_include(customer_id: "1", author_id: "10", project_id: "600", type: "TRAFFIC_MOBILE", value: 500)
+  end
+
+end

data/spec/log_message_spec.rb

@@ -0,0 +1,17 @@
+require 'spec_helper'
+
+describe SmsLogparser::LogMessage do
+
+  it "can extract the correct values from messages" do
+    log_message = SmsLogparser::LogMessage.new('127.0.0.1 - - [13/Apr/2014:05:33:23 +0200] "GET /content/51/102/42481/simvid_1.mp4 HTTP/1.1" 206 7865189 "-" "iTunes/11.1.5 (Windows; Microsoft Windows 7 Home Premium Edition Service Pack 1 (Build 7601)) AppleWebKit/537.60.11"')
+    log_message.customer_id.must_equal '51'
+    log_message.author_id.must_equal '102'
+    log_message.project_id.must_equal '42481'
+    log_message.status.must_equal 206
+    log_message.bytes.must_equal 7865189
+    log_message.file.must_equal 'simvid_1.mp4'
+    log_message.file_extname.must_equal '.mp4'
+    log_message.user_agent.must_equal 'iTunes/11.1.5 (Windows; Microsoft Windows 7 Home Premium Edition Service Pack 1 (Build 7601)) AppleWebKit/537.60.11'
+  end
+
+end

data/spec/parser_spec.rb

@@ -34,12 +34,10 @@ describe SmsLogparser::Parser do
     end
   end
 
-  %w(detect.mp4 index.m3u8).each do |file|
-    it "does not match excluded files" do
-      SmsLogparser::Parser.match?(
-        "GET /content/2/719/54986/#{file} HTTP/1.1\" 200 128 "
-      ).must_equal false
-    end
+  it "does not match for 'detect.mp4' files" do
+    SmsLogparser::Parser.match?(
+      "GET /content/2/719/54986/detect.mp4 HTTP/1.1\" 200 128 "
+    ).must_equal false
   end
 
   [
@@ -51,7 +49,7 @@ describe SmsLogparser::Parser do
     "Mozilla/5.0 (iPhone; U; CPU iPhone OS 5_1_1 like Mac OS X; en) AppleWebKit/534.46.0 (KHTML, like Gecko) CriOS/19.0.1084.60 Mobile/9B206 Safari/7534.48.3"
   ].each do |mobile_agent|
     it "traffic type for mobile user agents is TRAFFIC_MOBILE (#{mobile_agent})" do
-      SmsLogparser::Parser.get_traffic_type(mobile_agent).must_equal "TRAFFIC_MOBILE"
+      SmsLogparser::Parser.get_type(mobile_agent).must_equal "MOBILE"
     end
   end
 
@@ -59,32 +57,60 @@ describe SmsLogparser::Parser do
     '127.0.0.1 - - [13/Apr/2014:05:33:23 +0200] "GET /content/51/52/42481/simvid_1.mp4 HTTP/1.1" 206 7865189 "-" "iTunes/11.1.5 (Windows; Microsoft Windows 7 Home Premium Edition Service Pack 1 (Build 7601)) AppleWebKit/537.60.11"'
   ].each do |podcast_agent|
     it "traffic type for mobile user agents is TRAFFIC_PODCAST (#{podcast_agent})" do
-      SmsLogparser::Parser.get_traffic_type(podcast_agent).must_equal "TRAFFIC_PODCAST"
+      SmsLogparser::Parser.get_type(podcast_agent).must_equal "PODCAST"
     end
   end
 
-  it "should set visitor_type to VISITORS_MOBILE for index.m3u8 files" do
-    SmsLogparser::Parser.get_visitor_type(
-      "TRAFFIC_PODCAST", "index.m3u8"
-    ).must_equal "VISITORS_MOBILE"
+  it "count index.m3u8 with status 200 and user agent iPhone as mobile visit" do
+    message = '- - [22/Apr/2014:17:44:17 +0200] "GET /content/51/52/42701/index.m3u8 HTTP/1.1" 200 319 "-" "AppleCoreMedia/1.0.0.11D167 (iPhone; U; CPU OS 7_1 like Mac OS X; de_de)"'
+    data = SmsLogparser::Parser.new.extract_data_from_msg(message)
+    data[1][:customer_id].must_equal "51"
+    data[1][:author_id].must_equal "52"
+    data[1][:project_id].must_equal "42701"
+    data[1][:type].must_equal 'VISITORS_MOBILE'
+    data[1][:value].must_equal 1
+  end
+
+  it "count *.flv with status 200 and user agent Android as mobile visit" do
+    message = ' - - [22/Apr/2014:17:44:27 +0200] "GET /content/51/52/42709/simvid_1_40.flv HTTP/1.1" 200 96259 "http://blick.simplex.tv/NubesPlayer/index.html?cID=51&aID=52&pID=42709&autostart=false&themeColor=d6081c&embed=1&configUrl=http://f.blick.ch/resources/61786/ver1-0/js/xtendxIframeStatsSmartphone.js?adtechID=3522740&language=de&quality=40&hideHD=true&progressiveDownload=true" "Mozilla/5.0 (Linux; Android 4.4.2; C6903 Build/14.3.A.0.757) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.114 Mobile Safari/537.36"'
+    data = SmsLogparser::Parser.new.extract_data_from_msg(message)
+    data[1][:customer_id].must_equal "51"
+    data[1][:author_id].must_equal "52"
+    data[1][:project_id].must_equal "42709"
+    data[1][:type].must_equal 'VISITORS_MOBILE'
+    data[1][:value].must_equal 1
+  end
+
+  it "count *.mp4 with status 200 and user agent Android as mobile visit" do
+    message = '- - [22/Apr/2014:17:44:21 +0200] "GET /content/51/52/42701/simvid_1.mp4 HTTP/1.1" 200 2644715 "-" "Samsung GT-I9505 stagefright/1.2 (Linux;Android 4.4.2)"'
+    data = SmsLogparser::Parser.new.extract_data_from_msg(message)
+    data[1][:customer_id].must_equal "51"
+    data[1][:author_id].must_equal "52"
+    data[1][:project_id].must_equal "42701"
+    data[1][:type].must_equal 'VISITORS_MOBILE'
+    data[1][:value].must_equal 1
   end
 
-  it "should NOT set visitor_type to VISITORS_MOBILE for TRAFFIC_MOBILE and .ts files" do
-    SmsLogparser::Parser.get_visitor_type(
-      "TRAFFIC_MOBILE", "file.ts"
-    ).must_be_nil
+  it "count *.flv with status 200 and user agent Firefox on Windows as webcast visit" do
+    message = '- - [22/Apr/2014:18:00:50 +0200] "GET /content/51/52/42431/simvid_1_40.flv HTTP/1.1" 200 6742274 "http://blick.simplex.tv/NubesPlayer/player.swf" "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko"'
+    data = SmsLogparser::Parser.new.extract_data_from_msg(message)
+    data[1][:customer_id].must_equal "51"
+    data[1][:author_id].must_equal "52"
+    data[1][:project_id].must_equal "42431"
+    data[1][:type].must_equal 'VISITORS_WEBCAST'
+    data[1][:value].must_equal 1
   end
 
-  it "should set visitor_type to VISITORS_MOBILE for TRAFFIC_MOBILE and file not .ts" do
-    SmsLogparser::Parser.get_visitor_type(
-      "TRAFFIC_MOBILE", "file.mp3"
-    ).must_equal "VISITORS_MOBILE"
+  it "do not count *.css with status 200 as visit" do
+    message = '- - [22/Apr/2014:18:00:50 +0200] "GET /content/51/52/42431/application.css HTTP/1.1" 200 192 "http://blick.simplex.tv/NubesPlayer/player.swf" "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko"'
+    data = SmsLogparser::Parser.new.extract_data_from_msg(message)
+    data.size.must_equal 1
   end
 
-  it "should set visitor_type to VISITORS_PODCAST for TRAFFIC_PODCAST" do
-    SmsLogparser::Parser.get_visitor_type(
-      "TRAFFIC_PODCAST", "file.mp4"
-    ).must_equal "VISITORS_PODCAST"
+  it "do not count status 206 as visit" do
+    message = '- - [22/Apr/2014:18:00:50 +0200] "GET /content/51/52/42431/simvid_1_40.flv HTTP/1.1" 206 19289 "http://blick.simplex.tv/NubesPlayer/player.swf" "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko"'
+    data = SmsLogparser::Parser.new.extract_data_from_msg(message)
+    data.size.must_equal 1
   end
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sms-logparser
 version: !ruby/object:Gem::Version
-  version: 0.14.1
+  version: 0.15.0
 platform: ruby
 authors:
 - niwo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-22 00:00:00.000000000 Z
+date: 2014-04-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -127,6 +127,7 @@ files:
 - lib/sms-logparser/api.rb
 - lib/sms-logparser/cli.rb
 - lib/sms-logparser/data_cache.rb
+- lib/sms-logparser/log_message.rb
 - lib/sms-logparser/loggster.rb
 - lib/sms-logparser/mysql.rb
 - lib/sms-logparser/parser.rb
@@ -134,6 +135,8 @@ files:
 - sms-logparser.gemspec
 - spec/api_spec.rb
 - spec/cli_spec.rb
+- spec/data_cache_spec.rb
+- spec/log_message_spec.rb
 - spec/parser_spec.rb
 - spec/spec_helper.rb
 - spec/syslog_events.sql
@@ -164,6 +167,8 @@ summary: sms-logparser - Logparser for Simplex Media Server (SMS)
 test_files:
 - spec/api_spec.rb
 - spec/cli_spec.rb
+- spec/data_cache_spec.rb
+- spec/log_message_spec.rb
 - spec/parser_spec.rb
 - spec/spec_helper.rb
 - spec/syslog_events.sql