sms-logparser 0.0.4 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/sms-logparser/api.rb +26 -0
- data/lib/sms-logparser/cli.rb +53 -118
- data/lib/sms-logparser/mysql.rb +79 -0
- data/lib/sms-logparser/parser.rb +34 -0
- data/lib/sms-logparser/version.rb +1 -1
- data/lib/sms-logparser.rb +3 -0
- data/spec/{logparser_spec.rb → cli_spec.rb} +7 -7
- data/spec/spec_helper.rb +3 -3
- metadata +7 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2e9efb4712d1d9d54c9904f57993ced13742bd51
|
4
|
+
data.tar.gz: 2fa877596b733aac7a13a27cd76ac22902978564
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ecb5588a86ae77f592f67117172b4b1d259fec82d7bd6e80305c70de60032db646c8751734eed245d5d90867a1e875bf08e366465a1db23dade2400df1744b61
|
7
|
+
data.tar.gz: 1202cb4ff8e9d3820682e5cb6bbc0b032c0a5168dfc32c963915b5055f5bfe7a344e64870eb10a1d972e3d0f413e236b5e243f1fcd7b06638b94ea7ab6e5f098
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module SmsLogparser
|
2
|
+
class Api
|
3
|
+
|
4
|
+
def initialize(options)
|
5
|
+
@options = options
|
6
|
+
end
|
7
|
+
|
8
|
+
def send(data)
|
9
|
+
url = "#{@options[:api_base_path]}/"
|
10
|
+
url += "#{data[:customer_id]}/"
|
11
|
+
url += "#{data[:author_id]}/"
|
12
|
+
url += "#{data[:project_id]}/"
|
13
|
+
url += "#{data[:traffic_type]}/"
|
14
|
+
url += "#{data[:bytes]}"
|
15
|
+
unless @options[:simulate]
|
16
|
+
begin
|
17
|
+
RestClient.get(url)
|
18
|
+
rescue
|
19
|
+
raise "Can't send log to #{url}"
|
20
|
+
end
|
21
|
+
end
|
22
|
+
url
|
23
|
+
end
|
24
|
+
|
25
|
+
end # class
|
26
|
+
end # module
|
data/lib/sms-logparser/cli.rb
CHANGED
@@ -1,14 +1,6 @@
|
|
1
|
-
module SmsLogparser
|
1
|
+
module SmsLogparser
|
2
2
|
class Cli < Thor
|
3
3
|
|
4
|
-
EXTENSION_MAP = {
|
5
|
-
'default' => 'TRAFFIC_PODCAST',
|
6
|
-
'mp4' => 'TRAFFIC_PODCAST',
|
7
|
-
'f4v' => 'TRAFFIC_PODCAST',
|
8
|
-
'flv' => 'TRAFFIC_PODCAST',
|
9
|
-
'ts' => 'TRAFFIC_MOBILE'
|
10
|
-
}
|
11
|
-
|
12
4
|
class_option :mysql_host, :default => 'localhost'
|
13
5
|
class_option :mysql_user, :default => 'root'
|
14
6
|
class_option :mysql_db, :default => 'Syslog'
|
@@ -25,130 +17,73 @@ module SmsLogparser
|
|
25
17
|
def parse
|
26
18
|
start_time = Time.now
|
27
19
|
count = 0
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
if result['Message'] =~ /\/content\/.*\.(f4v|flv|mp4|ts) .*/
|
41
|
-
data = extract_data_from_msg(result['Message'])
|
42
|
-
url = "#{options[:api_base_path]}/"
|
43
|
-
url += "#{data[:customer_id]}/"
|
44
|
-
url += "#{data[:author_id]}/"
|
45
|
-
url += "#{data[:project_id]}/"
|
46
|
-
url += "#{data[:traffic_type]}/"
|
47
|
-
url += "#{data[:bytes]}"
|
48
|
-
if options[:simulate]
|
49
|
-
puts "Message ID: #{result['ID']}"
|
50
|
-
puts "URL: #{url}"
|
51
|
-
puts "Data: #{data}"
|
52
|
-
puts "-----------------------"
|
53
|
-
else
|
54
|
-
begin
|
55
|
-
RestClient.get(url)
|
56
|
-
rescue
|
57
|
-
say "Error: Can't send log to #{url}", :red
|
58
|
-
say "Aborting.", :red
|
59
|
-
break
|
60
|
-
end
|
20
|
+
begin
|
21
|
+
mysql = Mysql.new(options)
|
22
|
+
entries = mysql.get_entries
|
23
|
+
api = Api.new(options)
|
24
|
+
last_id = mysql.get_last_parse_id
|
25
|
+
entries.each do |entry|
|
26
|
+
if Parser.match(entry)
|
27
|
+
data = Parser.extract_data_from_msg(entry['Message'])
|
28
|
+
url = api.send(data)
|
29
|
+
last_id = entry['ID']
|
30
|
+
count += 1
|
31
|
+
debug_parser_output(data, url, entry) if options[:simulate]
|
61
32
|
end
|
62
|
-
count += 1
|
63
33
|
end
|
64
|
-
last_id
|
34
|
+
mysql.write_parse_result(last_id, count) unless options[:simulate]
|
35
|
+
puts "Started: #{start_time.strftime('%d.%d.%Y %T')}"
|
36
|
+
puts "Runtime: #{(Time.now - start_time).round(2)}s"
|
37
|
+
puts "Matches: #{count}"
|
38
|
+
rescue => e
|
39
|
+
say "Error: #{e.message}", :red
|
65
40
|
end
|
66
|
-
write_parse_result(last_id, count) unless options[:simulate]
|
67
|
-
puts "Started: #{start_time.strftime('%d.%d.%Y %T')}"
|
68
|
-
puts "Runtime: #{(Time.now - start_time).round(2)}s"
|
69
|
-
puts "Matches: #{count}"
|
70
41
|
end
|
71
42
|
|
72
43
|
desc "last_runs", "List the last paser runs"
|
73
44
|
def last_runs
|
74
|
-
begin
|
75
|
-
runs =
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
table
|
86
|
-
|
87
|
-
|
88
|
-
run['LastEventID']
|
89
|
-
]
|
45
|
+
begin
|
46
|
+
runs = Mysql.new(options).last_runs
|
47
|
+
if runs.size > 0
|
48
|
+
table = [%w(RunAt #Events LastEventID)]
|
49
|
+
runs.each do |run|
|
50
|
+
table << [
|
51
|
+
run['RunAt'],
|
52
|
+
run['EventsFound'],
|
53
|
+
run['LastEventID']
|
54
|
+
]
|
55
|
+
end
|
56
|
+
print_table table
|
57
|
+
else
|
58
|
+
say "No parser runs found in the database."
|
90
59
|
end
|
91
|
-
|
92
|
-
|
93
|
-
say "No parser runs found in the database."
|
60
|
+
rescue => e
|
61
|
+
say "Error: #{e.message}", :red
|
94
62
|
end
|
95
63
|
end
|
96
64
|
|
97
|
-
desc "
|
98
|
-
def
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
EventsFound INT DEFAULT 0,\
|
106
|
-
INDEX `LastEventID_I1` (`LastEventID`)
|
107
|
-
)"
|
108
|
-
)
|
109
|
-
say "OK", :green
|
65
|
+
desc "setup", "Create the parser table to track the last logs parsed"
|
66
|
+
def setup
|
67
|
+
begin
|
68
|
+
Mysql.new(options).create_parser_table
|
69
|
+
say "OK", :green
|
70
|
+
rescue => e
|
71
|
+
say "Error: #{e.message}", :red
|
72
|
+
end
|
110
73
|
end
|
111
74
|
|
112
75
|
no_commands do
|
113
|
-
def
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
say "
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
def write_parse_result(id, count)
|
124
|
-
client.query("INSERT INTO SmsParserRuns(RunAt, LastEventID, EventsFound)\
|
125
|
-
VALUES(\
|
126
|
-
'#{Time.now.strftime("%Y-%m-%d %H:%M:%S")}',\
|
127
|
-
#{id},\
|
128
|
-
#{count}
|
129
|
-
)"
|
130
|
-
)
|
131
|
-
end
|
132
|
-
|
133
|
-
def extract_data_from_msg(msg)
|
134
|
-
m = msg.match /\/content\/(\d+)\/(\d+)\/(\d+)\/.+\.(\S+)\s.*\"\s\d+\s(\d+)/
|
135
|
-
data = {
|
136
|
-
:customer_id => m[1],
|
137
|
-
:author_id => m[2],
|
138
|
-
:project_id => m[3],
|
139
|
-
:ext => m[4],
|
140
|
-
:traffic_type => (EXTENSION_MAP[m[4]] || EXTENSION_MAP['default']),
|
141
|
-
:bytes => m[5]
|
142
|
-
}
|
143
|
-
end
|
144
|
-
|
145
|
-
def client
|
146
|
-
@client ||= Mysql2::Client.new(
|
147
|
-
:host => options[:mysql_host],
|
148
|
-
:username => options[:mysql_user],
|
149
|
-
:database => options[:mysql_db]
|
150
|
-
)
|
76
|
+
def debug_parser_output(data, url, entry)
|
77
|
+
puts
|
78
|
+
say "Message ID: ", :green
|
79
|
+
say entry['ID']
|
80
|
+
say "URL: ", :green
|
81
|
+
say url
|
82
|
+
say "Data: ", :green
|
83
|
+
say data
|
84
|
+
puts
|
151
85
|
end
|
152
86
|
end
|
87
|
+
|
153
88
|
end
|
154
89
|
end
|
@@ -0,0 +1,79 @@
|
|
1
|
+
module SmsLogparser
|
2
|
+
class Mysql
|
3
|
+
|
4
|
+
def initialize(options)
|
5
|
+
@options = options
|
6
|
+
end
|
7
|
+
|
8
|
+
def client
|
9
|
+
@client ||= Mysql2::Client.new(
|
10
|
+
:host => @options[:mysql_host],
|
11
|
+
:username => @options[:mysql_user],
|
12
|
+
:database => @options[:mysql_db]
|
13
|
+
)
|
14
|
+
end
|
15
|
+
|
16
|
+
def last_runs
|
17
|
+
begin
|
18
|
+
runs = client.query(
|
19
|
+
"SELECT * FROM SmsParserRuns ORDER BY ID ASC LIMIT 10"
|
20
|
+
)
|
21
|
+
rescue Mysql2::Error => e
|
22
|
+
raise e
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def create_parser_table
|
27
|
+
begin
|
28
|
+
return client.query(
|
29
|
+
"CREATE TABLE IF NOT EXISTS\
|
30
|
+
SmsParserRuns(\
|
31
|
+
ID INT PRIMARY KEY AUTO_INCREMENT,\
|
32
|
+
RunAt datetime DEFAULT NULL,\
|
33
|
+
LastEventID INT DEFAULT NULL,\
|
34
|
+
EventsFound INT DEFAULT 0,\
|
35
|
+
INDEX `LastEventID_I1` (`LastEventID`)
|
36
|
+
)"
|
37
|
+
)
|
38
|
+
rescue Mysql2::Error => e
|
39
|
+
raise e
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def get_entries(last_id = get_last_parse_id)
|
44
|
+
begin
|
45
|
+
return client.query(
|
46
|
+
"SELECT * FROM SystemEvents\
|
47
|
+
WHERE `FromHost` like 'pcache%'\
|
48
|
+
AND ID > #{last_id} ORDER BY ID ASC"
|
49
|
+
)
|
50
|
+
rescue Mysql2::Error => e
|
51
|
+
raise e
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def write_parse_result(id, count)
|
56
|
+
client.query("INSERT INTO SmsParserRuns(RunAt, LastEventID, EventsFound)\
|
57
|
+
VALUES(\
|
58
|
+
'#{Time.now.strftime("%Y-%m-%d %H:%M:%S")}',\
|
59
|
+
#{id},\
|
60
|
+
#{count}
|
61
|
+
)"
|
62
|
+
)
|
63
|
+
end
|
64
|
+
|
65
|
+
def get_last_parse_id
|
66
|
+
id = 0
|
67
|
+
begin
|
68
|
+
last_parse = client.query(
|
69
|
+
"SELECT LastEventID FROM SmsParserRuns ORDER BY ID DESC LIMIT 1"
|
70
|
+
)
|
71
|
+
id = last_parse.first ? last_parse.first['LastEventID'] : 0
|
72
|
+
rescue Mysql2::Error => e
|
73
|
+
raise e
|
74
|
+
end
|
75
|
+
id
|
76
|
+
end
|
77
|
+
|
78
|
+
end # class
|
79
|
+
end # module
|
@@ -0,0 +1,34 @@
|
|
1
|
+
module SmsLogparser
|
2
|
+
class Parser
|
3
|
+
|
4
|
+
def self.extract_data_from_msg(msg)
|
5
|
+
m = msg.match /\/content\/(\d+)\/(\d+)\/(\d+)\/.+\.(\S+)\s.*\"\s\d+\s(\d+).+"(.*)"$/
|
6
|
+
raise "No match found." unless m
|
7
|
+
data = {
|
8
|
+
:customer_id => m[1],
|
9
|
+
:author_id => m[2],
|
10
|
+
:project_id => m[3],
|
11
|
+
:ext => m[4],
|
12
|
+
:traffic_type => Parser.get_traffic_type(m[4]),
|
13
|
+
:bytes => m[5],
|
14
|
+
:user_agent => m[6]
|
15
|
+
}
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.match(entry)
|
19
|
+
entry['Message'] =~ /\/content\/.*\.(f4v|flv|mp4|mp3|ts) .*/
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.get_traffic_type(user_agent)
|
23
|
+
case user_agent
|
24
|
+
when /.*(iTunes).*/
|
25
|
+
"TRAFFIC_PODCAST"
|
26
|
+
when /.*(IEMobile|Mobile Safari|iPhone|iPod|iPad).*/
|
27
|
+
"TRAFFIC_MOBILE"
|
28
|
+
else
|
29
|
+
"TRAFFIC_WEBCAST"
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
end # class
|
34
|
+
end # module
|
data/lib/sms-logparser.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
|
-
describe SmsLogparser do
|
3
|
+
describe SmsLogparser::Cli do
|
4
4
|
before do
|
5
5
|
TestHelper.create_test_db
|
6
6
|
TestHelper.create_sylog_db_table
|
@@ -12,7 +12,7 @@ describe SmsLogparser do
|
|
12
12
|
|
13
13
|
it "can create the parser_runs database table" do
|
14
14
|
out, err = capture_io do
|
15
|
-
TestHelper.sms_logparser.
|
15
|
+
TestHelper.sms_logparser.setup
|
16
16
|
end
|
17
17
|
out.must_match /OK/
|
18
18
|
end
|
@@ -22,7 +22,7 @@ describe SmsLogparser do
|
|
22
22
|
parser = TestHelper.sms_logparser
|
23
23
|
parser.options[:api_base_path] = 'http://devnull-as-a-service.com/dev/null/'
|
24
24
|
out, err = capture_io do
|
25
|
-
TestHelper.sms_logparser.
|
25
|
+
TestHelper.sms_logparser.setup
|
26
26
|
parser.parse
|
27
27
|
end
|
28
28
|
out.must_match /.*Matches: 10$/
|
@@ -33,22 +33,22 @@ describe SmsLogparser do
|
|
33
33
|
parser = TestHelper.sms_logparser
|
34
34
|
parser.options[:api_base_path] = 'http://devnull-as-a-service.com/dev/null/'
|
35
35
|
out, err = capture_io do
|
36
|
-
TestHelper.sms_logparser.
|
36
|
+
TestHelper.sms_logparser.setup
|
37
37
|
parser.parse
|
38
38
|
parser.parse
|
39
39
|
end
|
40
40
|
out.must_match /.*Matches: 0$/
|
41
41
|
end
|
42
42
|
|
43
|
-
it "
|
43
|
+
it "lists parser runs" do
|
44
44
|
TestHelper.seed_db(1)
|
45
45
|
parser = TestHelper.sms_logparser
|
46
46
|
parser.options[:api_base_path] = 'http://devnull-as-a-service.com/dev/null/'
|
47
47
|
out, err = capture_io do
|
48
|
-
TestHelper.sms_logparser.
|
48
|
+
TestHelper.sms_logparser.setup
|
49
49
|
parser.parse
|
50
50
|
parser.last_runs
|
51
51
|
end
|
52
52
|
assert_equal(err, "")
|
53
53
|
end
|
54
|
-
end
|
54
|
+
end
|
data/spec/spec_helper.rb
CHANGED
@@ -41,13 +41,13 @@ module TestHelper
|
|
41
41
|
end
|
42
42
|
|
43
43
|
def self.insert_logs(host = "pcache", message = "", number_of_inserts = 1)
|
44
|
-
values =
|
44
|
+
values = []
|
45
45
|
number_of_inserts.times do
|
46
|
-
values
|
46
|
+
values << "('#{host}', '#{message}')"
|
47
47
|
end
|
48
48
|
self.client.query(
|
49
49
|
"INSERT INTO #{@@mysql_db}.SystemEvents(FromHost, Message)\
|
50
|
-
VALUES #{values.
|
50
|
+
VALUES #{values.join(', ')}"
|
51
51
|
)
|
52
52
|
end
|
53
53
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sms-logparser
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- niwo
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-03-
|
11
|
+
date: 2014-03-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -95,10 +95,13 @@ files:
|
|
95
95
|
- Rakefile
|
96
96
|
- bin/sms-logparser
|
97
97
|
- lib/sms-logparser.rb
|
98
|
+
- lib/sms-logparser/api.rb
|
98
99
|
- lib/sms-logparser/cli.rb
|
100
|
+
- lib/sms-logparser/mysql.rb
|
101
|
+
- lib/sms-logparser/parser.rb
|
99
102
|
- lib/sms-logparser/version.rb
|
100
103
|
- sms-logparser.gemspec
|
101
|
-
- spec/
|
104
|
+
- spec/cli_spec.rb
|
102
105
|
- spec/spec_helper.rb
|
103
106
|
homepage: https://github.com/swisstxt/sms-logparser
|
104
107
|
licenses:
|
@@ -125,5 +128,5 @@ signing_key:
|
|
125
128
|
specification_version: 4
|
126
129
|
summary: SMS Logparser
|
127
130
|
test_files:
|
128
|
-
- spec/
|
131
|
+
- spec/cli_spec.rb
|
129
132
|
- spec/spec_helper.rb
|