sms-logparser 0.14.1 → 0.15.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/sms-logparser.rb +1 -0
- data/lib/sms-logparser/api.rb +24 -56
- data/lib/sms-logparser/cli.rb +15 -17
- data/lib/sms-logparser/data_cache.rb +11 -28
- data/lib/sms-logparser/log_message.rb +67 -0
- data/lib/sms-logparser/parser.rb +31 -44
- data/lib/sms-logparser/version.rb +1 -1
- data/spec/api_spec.rb +14 -40
- data/spec/data_cache_spec.rb +28 -0
- data/spec/log_message_spec.rb +17 -0
- data/spec/parser_spec.rb +50 -24
- metadata +7 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7ad478bf9a5f80ba72ab9e0f48303ce03e850511
|
4
|
+
data.tar.gz: 95d931e94749fcdb7876a7ecece52e4ce225675e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 19b52cef0cf0181912da7075cb91deb76f9f7a7bd2aeba49ce6409ed730ecc623a0c67927ab059a67bfb1ab31270201fa33c449c5b152470595ba8c1798404fe
|
7
|
+
data.tar.gz: 4cc3e8d1135625a43129d5f6f9a460b97e1fa20e7d55671b17017ebf70a9becc4e7b00b0908a3bc48de075895e8683e0483112eea4937784e533ae0bb1c9a6a4
|
data/lib/sms-logparser.rb
CHANGED
data/lib/sms-logparser/api.rb
CHANGED
@@ -14,25 +14,22 @@ module SmsLogparser
|
|
14
14
|
end
|
15
15
|
|
16
16
|
def send(data)
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
request[:status] = response.status
|
25
|
-
rescue => e
|
26
|
-
raise RuntimeError, "Can't send request to #{request[:uri]}. #{e.message}", caller
|
27
|
-
end
|
28
|
-
unless @accepted_responses.include?(response.status)
|
29
|
-
msg = "Received HTTP status #{response.status} from API. Only accepting #{@accepted_responses.join(', ')}."
|
30
|
-
raise RuntimeError, msg, caller
|
31
|
-
end
|
17
|
+
path = data_to_path(data)
|
18
|
+
begin
|
19
|
+
if @options[:simulate]
|
20
|
+
status = 200
|
21
|
+
else
|
22
|
+
response = @connection.post(path)
|
23
|
+
status = response.status
|
32
24
|
end
|
25
|
+
rescue => e
|
26
|
+
raise RuntimeError, "Can't send request to #{path}. #{e.message}", caller
|
33
27
|
end
|
34
|
-
|
35
|
-
|
28
|
+
unless @accepted_responses.include?(status)
|
29
|
+
msg = "Received HTTP status #{status} from API. Only accepting #{@accepted_responses.join(', ')}."
|
30
|
+
raise RuntimeError, msg, caller
|
31
|
+
end
|
32
|
+
return path, status
|
36
33
|
end
|
37
34
|
|
38
35
|
def send_sets(data_sets, concurrency=4)
|
@@ -42,30 +39,9 @@ module SmsLogparser
|
|
42
39
|
threads = concurrency.times.map do
|
43
40
|
Thread.new do
|
44
41
|
while !queue.empty?
|
45
|
-
|
46
|
-
data = queue.pop
|
47
|
-
url = @base_path + [
|
48
|
-
data[:customer_id],
|
49
|
-
data[:author_id],
|
50
|
-
data[:project_id],
|
51
|
-
data[:type],
|
52
|
-
data[:value]
|
53
|
-
].join('/')
|
54
|
-
if @options[:simulate]
|
55
|
-
status = 200
|
56
|
-
else
|
57
|
-
response = @connection.post(url)
|
58
|
-
status = response.status
|
59
|
-
end
|
60
|
-
rescue => e
|
61
|
-
raise RuntimeError, "Can't send request to #{url}. #{e.message}", caller
|
62
|
-
end
|
63
|
-
unless @accepted_responses.include?(status)
|
64
|
-
msg = "Received HTTP status #{status} from API. Only accepting #{@accepted_responses.join(', ')}."
|
65
|
-
raise RuntimeError, msg, caller
|
66
|
-
end
|
42
|
+
path, status = send(queue.pop)
|
67
43
|
semaphore.synchronize {
|
68
|
-
yield
|
44
|
+
yield path, status
|
69
45
|
}
|
70
46
|
end
|
71
47
|
end
|
@@ -73,22 +49,14 @@ module SmsLogparser
|
|
73
49
|
threads.each {|thread| thread.join }
|
74
50
|
end
|
75
51
|
|
76
|
-
def
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
end
|
85
|
-
if data[:visitor_type]
|
86
|
-
requests << {
|
87
|
-
url: @url,
|
88
|
-
uri: [path, data[:visitor_type], 1].join('/')
|
89
|
-
}
|
90
|
-
end
|
91
|
-
requests
|
52
|
+
def data_to_path(data)
|
53
|
+
@base_path + [
|
54
|
+
data[:customer_id],
|
55
|
+
data[:author_id],
|
56
|
+
data[:project_id],
|
57
|
+
data[:type],
|
58
|
+
data[:value]
|
59
|
+
].join('/')
|
92
60
|
end
|
93
61
|
|
94
62
|
private
|
data/lib/sms-logparser/cli.rb
CHANGED
@@ -69,13 +69,15 @@ module SmsLogparser
|
|
69
69
|
logger.info { "Getting log messages from database..." }
|
70
70
|
entries.each do |entry|
|
71
71
|
parser.extract_data_from_msg(entry['Message']) do |data|
|
72
|
-
if data
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
72
|
+
if data.size > 0
|
73
|
+
data.each do |data_entry|
|
74
|
+
if options[:accumulate]
|
75
|
+
cache.add(data_entry)
|
76
|
+
logger.debug {"Cached data: #{data_entry}"}
|
77
|
+
else
|
78
|
+
url, status = api.send(data_entry)
|
79
|
+
verbose_parser_output(entry['ID'], data_entry, url, status)
|
80
|
+
end
|
79
81
|
end
|
80
82
|
state[:last_event_id] = entry['ID']
|
81
83
|
state[:match_count] += 1
|
@@ -85,9 +87,9 @@ module SmsLogparser
|
|
85
87
|
end
|
86
88
|
if options[:accumulate]
|
87
89
|
resp_stats = {}
|
88
|
-
api.send_sets(cache.data_sets, options[:concurrency]) do |url,
|
89
|
-
logger.debug { "POST #{url} (#{
|
90
|
-
resp_stats[
|
90
|
+
api.send_sets(cache.data_sets, options[:concurrency]) do |url, status|
|
91
|
+
logger.debug { "POST #{url} (#{status})" }
|
92
|
+
resp_stats[status] = resp_stats[status].to_i + 1
|
91
93
|
end
|
92
94
|
logger.info { "Usage commited: #{resp_stats.map {|k,v| "#{v} x status #{k}" }.join(" : ")}" }
|
93
95
|
end
|
@@ -195,15 +197,11 @@ module SmsLogparser
|
|
195
197
|
SmsLogparser::Loggster.instance.set_log_device options[:logfile]
|
196
198
|
end
|
197
199
|
|
198
|
-
def verbose_parser_output(entry_id, data,
|
200
|
+
def verbose_parser_output(entry_id, data, url, status)
|
199
201
|
logger.debug {
|
200
|
-
"
|
202
|
+
"Parsing data for #{entry_id} (#{data.map{|k,v| "#{k}=\"#{v || '-'}\""}.join(" ") || ''})"
|
201
203
|
}
|
202
|
-
|
203
|
-
logger.debug {
|
204
|
-
"URL #{i + 1} for entry #{entry_id} #{req[:url]}#{req[:uri]}"
|
205
|
-
}
|
206
|
-
end
|
204
|
+
logger.debug {"URL for entry #{entry_id}: #{url} (#{status})"}
|
207
205
|
end
|
208
206
|
|
209
207
|
def table_to_csv(table)
|
@@ -5,42 +5,25 @@ module SmsLogparser
|
|
5
5
|
|
6
6
|
def initialize
|
7
7
|
@cache = Hash.new
|
8
|
-
@wanted_keys = [:customer_id, :author_id, :project_id]
|
9
8
|
end
|
10
9
|
|
11
10
|
def add(data)
|
12
|
-
key = [data[:customer_id], data[:author_id], data[:project_id]].join('.')
|
13
|
-
@cache[key] =
|
14
|
-
unless data[:file] =~ /.*\.m3u8$/
|
15
|
-
@cache[key][data[:traffic_type]] = @cache[key][data[:traffic_type]].to_i + data[:bytes].to_i
|
16
|
-
end
|
17
|
-
if data[:visitor_type]
|
18
|
-
@cache[key][data[:visitor_type]] = @cache[key][data[:visitor_type]].to_i + 1
|
19
|
-
end
|
11
|
+
key = [data[:customer_id], data[:author_id], data[:project_id], data[:type]].join('.')
|
12
|
+
@cache[key] = @cache[key].to_i + data[:value].to_i
|
20
13
|
@cache
|
21
14
|
end
|
22
15
|
|
23
16
|
def data_sets
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
value: values[type]
|
34
|
-
} if values[type]
|
35
|
-
end
|
17
|
+
@cache.map do |key, value|
|
18
|
+
key_components = key.split('.')
|
19
|
+
{
|
20
|
+
customer_id: key_components[0],
|
21
|
+
author_id: key_components[1],
|
22
|
+
project_id: key_components[2],
|
23
|
+
type: key_components[3],
|
24
|
+
value: value
|
25
|
+
}
|
36
26
|
end
|
37
|
-
sets
|
38
|
-
end
|
39
|
-
|
40
|
-
private
|
41
|
-
|
42
|
-
def initialize_value(data)
|
43
|
-
data.select { |key,_| @wanted_keys.include? key }
|
44
27
|
end
|
45
28
|
|
46
29
|
end # class
|
@@ -0,0 +1,67 @@
|
|
1
|
+
module SmsLogparser
|
2
|
+
class LogMessage
|
3
|
+
def initialize(message)
|
4
|
+
@message = message
|
5
|
+
end
|
6
|
+
|
7
|
+
def customer_id
|
8
|
+
match[1]
|
9
|
+
end
|
10
|
+
|
11
|
+
def author_id
|
12
|
+
match[2]
|
13
|
+
end
|
14
|
+
|
15
|
+
def project_id
|
16
|
+
match[3]
|
17
|
+
end
|
18
|
+
|
19
|
+
def status
|
20
|
+
match[5].to_i
|
21
|
+
end
|
22
|
+
|
23
|
+
def bytes
|
24
|
+
match[6].to_i
|
25
|
+
end
|
26
|
+
|
27
|
+
def file
|
28
|
+
match[4]
|
29
|
+
end
|
30
|
+
|
31
|
+
def file_extname
|
32
|
+
File.extname(file)
|
33
|
+
end
|
34
|
+
|
35
|
+
def user_agent
|
36
|
+
match[7]
|
37
|
+
end
|
38
|
+
|
39
|
+
def account_info
|
40
|
+
{
|
41
|
+
customer_id: customer_id,
|
42
|
+
author_id: author_id,
|
43
|
+
project_id: project_id
|
44
|
+
}
|
45
|
+
end
|
46
|
+
|
47
|
+
def transfer_info
|
48
|
+
{
|
49
|
+
status: status,
|
50
|
+
bytes: bytes,
|
51
|
+
file: file,
|
52
|
+
file_extname: file_extname,
|
53
|
+
user_agent: user_agent
|
54
|
+
}
|
55
|
+
end
|
56
|
+
|
57
|
+
def to_h
|
58
|
+
account_info.merge(transfer_info)
|
59
|
+
end
|
60
|
+
|
61
|
+
private
|
62
|
+
|
63
|
+
def match
|
64
|
+
@match ||= @message.match /\/content\/(\d+)\/(\d+)\/(\d+)\/(\w+\.\w+)\s.*\"\s(\d+)\s(\d+).+"(.*)"$/
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
data/lib/sms-logparser/parser.rb
CHANGED
@@ -1,29 +1,28 @@
|
|
1
1
|
module SmsLogparser
|
2
2
|
class Parser
|
3
3
|
|
4
|
-
def initialize(options)
|
4
|
+
def initialize(options = {})
|
5
5
|
@options = options
|
6
6
|
@logger = SmsLogparser::Loggster.instance
|
7
7
|
end
|
8
8
|
|
9
9
|
def extract_data_from_msg(message)
|
10
|
-
data =
|
10
|
+
data = []
|
11
11
|
if Parser.match?(message)
|
12
12
|
@logger.debug { "Parser MATCH: #{message}" }
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
:
|
24
|
-
:
|
25
|
-
|
26
|
-
}
|
13
|
+
log_message = LogMessage.new(message)
|
14
|
+
type = Parser.get_type(log_message.user_agent)
|
15
|
+
data << log_message.account_info.merge(
|
16
|
+
type: "TRAFFIC_#{type}",
|
17
|
+
value: (log_message.bytes * traffic_correction_factor(type)).round(0)
|
18
|
+
)
|
19
|
+
if log_message.status == 200 &&
|
20
|
+
(log_message.file_extname =~ /\.(mp3|mp4|flv|f4v)/ ||
|
21
|
+
log_message.file == 'index.m3u8')
|
22
|
+
data << log_message.account_info.merge(
|
23
|
+
type: "VISITORS_#{type}",
|
24
|
+
value: 1,
|
25
|
+
)
|
27
26
|
end
|
28
27
|
else
|
29
28
|
@logger.debug { "Parser IGNORE: #{message}" }
|
@@ -32,52 +31,40 @@ module SmsLogparser
|
|
32
31
|
yield data
|
33
32
|
end
|
34
33
|
|
35
|
-
def traffic_correction_factor(traffic_type)
|
36
|
-
factor = case traffic_type
|
37
|
-
when 'TRAFFIC_WEBCAST'
|
38
|
-
@options[:webcast_traffic_correction] || 1.0
|
39
|
-
when 'TRAFFIC_MOBILE'
|
40
|
-
@options[:mobile_traffic_correction] || 1.0
|
41
|
-
when 'TRAFFIC_PODCAST'
|
42
|
-
@options[:podcast_traffic_correction] || 1.0
|
43
|
-
else
|
44
|
-
1.0
|
45
|
-
end
|
46
|
-
factor.to_f
|
47
|
-
end
|
48
|
-
|
49
34
|
def self.match?(message)
|
50
35
|
match = message.match(/\/content\/.+\/(\S+) .+ (200|206)/i)
|
51
|
-
# ignore detect.mp4
|
36
|
+
# ignore detect.mp4
|
52
37
|
if match
|
53
|
-
return true unless match[1] =~ /detect.mp4
|
38
|
+
return true unless match[1] =~ /detect.mp4/i
|
54
39
|
end
|
55
40
|
false
|
56
41
|
end
|
57
42
|
|
58
43
|
# see https://developer.mozilla.org/en-US/docs/Browser_detection_using_the_user_agent
|
59
44
|
# for mobile browser detection
|
60
|
-
def self.
|
45
|
+
def self.get_type(user_agent)
|
61
46
|
case user_agent
|
62
47
|
when /.*(iTunes).*/i
|
63
|
-
'
|
48
|
+
'PODCAST'
|
64
49
|
when /.*(Mobi|IEMobile|Mobile Safari|iPhone|iPod|iPad|Android|BlackBerry|Opera Mini).*/
|
65
|
-
'
|
50
|
+
'MOBILE'
|
66
51
|
else
|
67
|
-
'
|
52
|
+
'WEBCAST'
|
68
53
|
end
|
69
54
|
end
|
70
55
|
|
71
|
-
def
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
56
|
+
def traffic_correction_factor(traffic_type)
|
57
|
+
factor = case traffic_type
|
58
|
+
when 'WEBCAST'
|
59
|
+
@options[:webcast_traffic_correction] || 1.0
|
60
|
+
when 'MOBILE'
|
61
|
+
@options[:mobile_traffic_correction] || 1.0
|
62
|
+
when 'PODCAST'
|
63
|
+
@options[:podcast_traffic_correction] || 1.0
|
78
64
|
else
|
79
|
-
|
65
|
+
1.0
|
80
66
|
end
|
67
|
+
factor.to_f
|
81
68
|
end
|
82
69
|
|
83
70
|
end # class
|
data/spec/api_spec.rb
CHANGED
@@ -9,52 +9,26 @@ describe SmsLogparser::Api do
|
|
9
9
|
)
|
10
10
|
end
|
11
11
|
|
12
|
-
it "
|
12
|
+
it "builds the correct path for TRAFFIC data" do
|
13
13
|
data = {
|
14
|
-
:
|
15
|
-
:
|
16
|
-
:
|
17
|
-
:
|
18
|
-
:
|
19
|
-
:user_agent => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko)",
|
20
|
-
:traffic_type => 'TRAFFIC_WEBCAST',
|
21
|
-
:visitor_type => 'VISITORS_WEBCAST'
|
14
|
+
customer_id: 1,
|
15
|
+
author_id: 2,
|
16
|
+
project_id: 3,
|
17
|
+
value: 128,
|
18
|
+
type: 'TRAFFIC_WEBCAST',
|
22
19
|
}
|
23
|
-
|
24
|
-
requests.size.must_equal 2
|
25
|
-
requests[0][:uri].must_match /\/1\/2\/3\/TRAFFIC_WEBCAST\/128$/
|
26
|
-
requests[1][:uri].must_match /\/1\/2\/3\/VISITORS_WEBCAST\/1$/
|
20
|
+
@api.data_to_path(data).must_match /\/1\/2\/3\/TRAFFIC_WEBCAST\/128$/
|
27
21
|
end
|
28
22
|
|
29
|
-
it "
|
23
|
+
it "builds the correct path for VISITOR data" do
|
30
24
|
data = {
|
31
|
-
:
|
32
|
-
:
|
33
|
-
:
|
34
|
-
:
|
35
|
-
:
|
36
|
-
:user_agent => 'Mozilla/5.0 (Android; Mobile; rv:13.0) Gecko/13.0 Firefox/13.0',
|
37
|
-
:traffic_type => 'TRAFFIC_MOBILE',
|
38
|
-
:visitor_type => 'VISITORS_MOBILE'
|
25
|
+
customer_id: 101,
|
26
|
+
author_id: 202,
|
27
|
+
project_id: 303,
|
28
|
+
type: 'TRAFFIC_MOBILE',
|
29
|
+
value: 48
|
39
30
|
}
|
40
|
-
|
41
|
-
requests.size.must_equal 1
|
42
|
-
requests[0][:uri].must_match /\/100\/200\/300\/VISITORS_MOBILE\/1$/
|
43
|
-
end
|
44
|
-
|
45
|
-
it "does not send visitor info if no visitor_type" do
|
46
|
-
data = {
|
47
|
-
:customer_id => 101,
|
48
|
-
:author_id => 202,
|
49
|
-
:project_id => 303,
|
50
|
-
:file => 'myfile.mp4',
|
51
|
-
:bytes => 48,
|
52
|
-
:user_agent => 'Mozilla/5.0 (Android; Mobile; rv:13.0) Gecko/13.0 Firefox/13.0',
|
53
|
-
:traffic_type => 'TRAFFIC_MOBILE',
|
54
|
-
}
|
55
|
-
requests = @api.send(data)
|
56
|
-
requests.size.must_equal 1
|
57
|
-
requests[0][:uri].must_match /\/101\/202\/303\/TRAFFIC_MOBILE\/48$/
|
31
|
+
@api.data_to_path(data).must_match /\/101\/202\/303\/TRAFFIC_MOBILE\/48$/
|
58
32
|
end
|
59
33
|
|
60
34
|
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe SmsLogparser::DataCache do
|
4
|
+
|
5
|
+
before do
|
6
|
+
@cache = SmsLogparser::DataCache.new
|
7
|
+
end
|
8
|
+
|
9
|
+
it "returns correct totals from cache" do
|
10
|
+
@cache.add(customer_id: 101, author_id: 202, project_id: 303, type: 'TRAFFIC_MOBILE', value: 10)
|
11
|
+
@cache.add(customer_id: 101, author_id: 202, project_id: 303, type: 'TRAFFIC_MOBILE', value: 400)
|
12
|
+
@cache.add(customer_id: 101, author_id: 202, project_id: 303, type: 'TRAFFIC_MOBILE', value: 1000)
|
13
|
+
|
14
|
+
@cache.add(customer_id: 101, author_id: 202, project_id: 303, type: 'TRAFFIC_WEBCAST', value: 1000)
|
15
|
+
|
16
|
+
1000.times do
|
17
|
+
@cache.add(customer_id: 101, author_id: 300, project_id: 303, type: 'VISIT_WEBCAST', value: 1)
|
18
|
+
end
|
19
|
+
|
20
|
+
@cache.add(customer_id: 1, author_id: 10, project_id: 600, type: 'TRAFFIC_MOBILE', value: 500)
|
21
|
+
|
22
|
+
@cache.data_sets.must_include(customer_id: "101", author_id: "202", project_id: "303", type: "TRAFFIC_MOBILE", value: 1410)
|
23
|
+
@cache.data_sets.must_include(customer_id: "101", author_id: "202", project_id: "303", type: "TRAFFIC_WEBCAST", value: 1000)
|
24
|
+
@cache.data_sets.must_include(customer_id: "101", author_id: "300", project_id: "303", type: "VISIT_WEBCAST", value: 1000)
|
25
|
+
@cache.data_sets.must_include(customer_id: "1", author_id: "10", project_id: "600", type: "TRAFFIC_MOBILE", value: 500)
|
26
|
+
end
|
27
|
+
|
28
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe SmsLogparser::LogMessage do
|
4
|
+
|
5
|
+
it "can extract the correct values from messages" do
|
6
|
+
log_message = SmsLogparser::LogMessage.new('127.0.0.1 - - [13/Apr/2014:05:33:23 +0200] "GET /content/51/102/42481/simvid_1.mp4 HTTP/1.1" 206 7865189 "-" "iTunes/11.1.5 (Windows; Microsoft Windows 7 Home Premium Edition Service Pack 1 (Build 7601)) AppleWebKit/537.60.11"')
|
7
|
+
log_message.customer_id.must_equal '51'
|
8
|
+
log_message.author_id.must_equal '102'
|
9
|
+
log_message.project_id.must_equal '42481'
|
10
|
+
log_message.status.must_equal 206
|
11
|
+
log_message.bytes.must_equal 7865189
|
12
|
+
log_message.file.must_equal 'simvid_1.mp4'
|
13
|
+
log_message.file_extname.must_equal '.mp4'
|
14
|
+
log_message.user_agent.must_equal 'iTunes/11.1.5 (Windows; Microsoft Windows 7 Home Premium Edition Service Pack 1 (Build 7601)) AppleWebKit/537.60.11'
|
15
|
+
end
|
16
|
+
|
17
|
+
end
|
data/spec/parser_spec.rb
CHANGED
@@ -34,12 +34,10 @@ describe SmsLogparser::Parser do
|
|
34
34
|
end
|
35
35
|
end
|
36
36
|
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
).must_equal false
|
42
|
-
end
|
37
|
+
it "does not match for 'detect.mp4' files" do
|
38
|
+
SmsLogparser::Parser.match?(
|
39
|
+
"GET /content/2/719/54986/detect.mp4 HTTP/1.1\" 200 128 "
|
40
|
+
).must_equal false
|
43
41
|
end
|
44
42
|
|
45
43
|
[
|
@@ -51,7 +49,7 @@ describe SmsLogparser::Parser do
|
|
51
49
|
"Mozilla/5.0 (iPhone; U; CPU iPhone OS 5_1_1 like Mac OS X; en) AppleWebKit/534.46.0 (KHTML, like Gecko) CriOS/19.0.1084.60 Mobile/9B206 Safari/7534.48.3"
|
52
50
|
].each do |mobile_agent|
|
53
51
|
it "traffic type for mobile user agents is TRAFFIC_MOBILE (#{mobile_agent})" do
|
54
|
-
SmsLogparser::Parser.
|
52
|
+
SmsLogparser::Parser.get_type(mobile_agent).must_equal "MOBILE"
|
55
53
|
end
|
56
54
|
end
|
57
55
|
|
@@ -59,32 +57,60 @@ describe SmsLogparser::Parser do
|
|
59
57
|
'127.0.0.1 - - [13/Apr/2014:05:33:23 +0200] "GET /content/51/52/42481/simvid_1.mp4 HTTP/1.1" 206 7865189 "-" "iTunes/11.1.5 (Windows; Microsoft Windows 7 Home Premium Edition Service Pack 1 (Build 7601)) AppleWebKit/537.60.11"'
|
60
58
|
].each do |podcast_agent|
|
61
59
|
it "traffic type for mobile user agents is TRAFFIC_PODCAST (#{podcast_agent})" do
|
62
|
-
SmsLogparser::Parser.
|
60
|
+
SmsLogparser::Parser.get_type(podcast_agent).must_equal "PODCAST"
|
63
61
|
end
|
64
62
|
end
|
65
63
|
|
66
|
-
it "
|
67
|
-
|
68
|
-
|
69
|
-
|
64
|
+
it "count index.m3u8 with status 200 and user agent iPhone as mobile visit" do
|
65
|
+
message = '- - [22/Apr/2014:17:44:17 +0200] "GET /content/51/52/42701/index.m3u8 HTTP/1.1" 200 319 "-" "AppleCoreMedia/1.0.0.11D167 (iPhone; U; CPU OS 7_1 like Mac OS X; de_de)"'
|
66
|
+
data = SmsLogparser::Parser.new.extract_data_from_msg(message)
|
67
|
+
data[1][:customer_id].must_equal "51"
|
68
|
+
data[1][:author_id].must_equal "52"
|
69
|
+
data[1][:project_id].must_equal "42701"
|
70
|
+
data[1][:type].must_equal 'VISITORS_MOBILE'
|
71
|
+
data[1][:value].must_equal 1
|
72
|
+
end
|
73
|
+
|
74
|
+
it "count *.flv with status 200 and user agent Android as mobile visit" do
|
75
|
+
message = ' - - [22/Apr/2014:17:44:27 +0200] "GET /content/51/52/42709/simvid_1_40.flv HTTP/1.1" 200 96259 "http://blick.simplex.tv/NubesPlayer/index.html?cID=51&aID=52&pID=42709&autostart=false&themeColor=d6081c&embed=1&configUrl=http://f.blick.ch/resources/61786/ver1-0/js/xtendxIframeStatsSmartphone.js?adtechID=3522740&language=de&quality=40&hideHD=true&progressiveDownload=true" "Mozilla/5.0 (Linux; Android 4.4.2; C6903 Build/14.3.A.0.757) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.114 Mobile Safari/537.36"'
|
76
|
+
data = SmsLogparser::Parser.new.extract_data_from_msg(message)
|
77
|
+
data[1][:customer_id].must_equal "51"
|
78
|
+
data[1][:author_id].must_equal "52"
|
79
|
+
data[1][:project_id].must_equal "42709"
|
80
|
+
data[1][:type].must_equal 'VISITORS_MOBILE'
|
81
|
+
data[1][:value].must_equal 1
|
82
|
+
end
|
83
|
+
|
84
|
+
it "count *.mp4 with status 200 and user agent Android as mobile visit" do
|
85
|
+
message = '- - [22/Apr/2014:17:44:21 +0200] "GET /content/51/52/42701/simvid_1.mp4 HTTP/1.1" 200 2644715 "-" "Samsung GT-I9505 stagefright/1.2 (Linux;Android 4.4.2)"'
|
86
|
+
data = SmsLogparser::Parser.new.extract_data_from_msg(message)
|
87
|
+
data[1][:customer_id].must_equal "51"
|
88
|
+
data[1][:author_id].must_equal "52"
|
89
|
+
data[1][:project_id].must_equal "42701"
|
90
|
+
data[1][:type].must_equal 'VISITORS_MOBILE'
|
91
|
+
data[1][:value].must_equal 1
|
70
92
|
end
|
71
93
|
|
72
|
-
it "
|
73
|
-
|
74
|
-
|
75
|
-
|
94
|
+
it "count *.flv with status 200 and user agent Firefox on Windows as webcast visit" do
|
95
|
+
message = '- - [22/Apr/2014:18:00:50 +0200] "GET /content/51/52/42431/simvid_1_40.flv HTTP/1.1" 200 6742274 "http://blick.simplex.tv/NubesPlayer/player.swf" "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko"'
|
96
|
+
data = SmsLogparser::Parser.new.extract_data_from_msg(message)
|
97
|
+
data[1][:customer_id].must_equal "51"
|
98
|
+
data[1][:author_id].must_equal "52"
|
99
|
+
data[1][:project_id].must_equal "42431"
|
100
|
+
data[1][:type].must_equal 'VISITORS_WEBCAST'
|
101
|
+
data[1][:value].must_equal 1
|
76
102
|
end
|
77
103
|
|
78
|
-
it "
|
79
|
-
|
80
|
-
|
81
|
-
|
104
|
+
it "do not count *.css with status 200 as visit" do
|
105
|
+
message = '- - [22/Apr/2014:18:00:50 +0200] "GET /content/51/52/42431/application.css HTTP/1.1" 200 192 "http://blick.simplex.tv/NubesPlayer/player.swf" "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko"'
|
106
|
+
data = SmsLogparser::Parser.new.extract_data_from_msg(message)
|
107
|
+
data.size.must_equal 1
|
82
108
|
end
|
83
109
|
|
84
|
-
it "
|
85
|
-
|
86
|
-
|
87
|
-
|
110
|
+
it "do not count status 206 as visit" do
|
111
|
+
message = '- - [22/Apr/2014:18:00:50 +0200] "GET /content/51/52/42431/simvid_1_40.flv HTTP/1.1" 206 19289 "http://blick.simplex.tv/NubesPlayer/player.swf" "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko"'
|
112
|
+
data = SmsLogparser::Parser.new.extract_data_from_msg(message)
|
113
|
+
data.size.must_equal 1
|
88
114
|
end
|
89
115
|
|
90
116
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sms-logparser
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.15.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- niwo
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-04-
|
11
|
+
date: 2014-04-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -127,6 +127,7 @@ files:
|
|
127
127
|
- lib/sms-logparser/api.rb
|
128
128
|
- lib/sms-logparser/cli.rb
|
129
129
|
- lib/sms-logparser/data_cache.rb
|
130
|
+
- lib/sms-logparser/log_message.rb
|
130
131
|
- lib/sms-logparser/loggster.rb
|
131
132
|
- lib/sms-logparser/mysql.rb
|
132
133
|
- lib/sms-logparser/parser.rb
|
@@ -134,6 +135,8 @@ files:
|
|
134
135
|
- sms-logparser.gemspec
|
135
136
|
- spec/api_spec.rb
|
136
137
|
- spec/cli_spec.rb
|
138
|
+
- spec/data_cache_spec.rb
|
139
|
+
- spec/log_message_spec.rb
|
137
140
|
- spec/parser_spec.rb
|
138
141
|
- spec/spec_helper.rb
|
139
142
|
- spec/syslog_events.sql
|
@@ -164,6 +167,8 @@ summary: sms-logparser - Logparser for Simplex Media Server (SMS)
|
|
164
167
|
test_files:
|
165
168
|
- spec/api_spec.rb
|
166
169
|
- spec/cli_spec.rb
|
170
|
+
- spec/data_cache_spec.rb
|
171
|
+
- spec/log_message_spec.rb
|
167
172
|
- spec/parser_spec.rb
|
168
173
|
- spec/spec_helper.rb
|
169
174
|
- spec/syslog_events.sql
|