sms-logparser 0.0.4 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/sms-logparser/api.rb +26 -0
- data/lib/sms-logparser/cli.rb +53 -118
- data/lib/sms-logparser/mysql.rb +79 -0
- data/lib/sms-logparser/parser.rb +34 -0
- data/lib/sms-logparser/version.rb +1 -1
- data/lib/sms-logparser.rb +3 -0
- data/spec/{logparser_spec.rb → cli_spec.rb} +7 -7
- data/spec/spec_helper.rb +3 -3
- metadata +7 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2e9efb4712d1d9d54c9904f57993ced13742bd51
|
4
|
+
data.tar.gz: 2fa877596b733aac7a13a27cd76ac22902978564
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ecb5588a86ae77f592f67117172b4b1d259fec82d7bd6e80305c70de60032db646c8751734eed245d5d90867a1e875bf08e366465a1db23dade2400df1744b61
|
7
|
+
data.tar.gz: 1202cb4ff8e9d3820682e5cb6bbc0b032c0a5168dfc32c963915b5055f5bfe7a344e64870eb10a1d972e3d0f413e236b5e243f1fcd7b06638b94ea7ab6e5f098
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module SmsLogparser
|
2
|
+
class Api
|
3
|
+
|
4
|
+
def initialize(options)
|
5
|
+
@options = options
|
6
|
+
end
|
7
|
+
|
8
|
+
def send(data)
|
9
|
+
url = "#{@options[:api_base_path]}/"
|
10
|
+
url += "#{data[:customer_id]}/"
|
11
|
+
url += "#{data[:author_id]}/"
|
12
|
+
url += "#{data[:project_id]}/"
|
13
|
+
url += "#{data[:traffic_type]}/"
|
14
|
+
url += "#{data[:bytes]}"
|
15
|
+
unless @options[:simulate]
|
16
|
+
begin
|
17
|
+
RestClient.get(url)
|
18
|
+
rescue
|
19
|
+
raise "Can't send log to #{url}"
|
20
|
+
end
|
21
|
+
end
|
22
|
+
url
|
23
|
+
end
|
24
|
+
|
25
|
+
end # class
|
26
|
+
end # module
|
data/lib/sms-logparser/cli.rb
CHANGED
@@ -1,14 +1,6 @@
|
|
1
|
-
module SmsLogparser
|
1
|
+
module SmsLogparser
|
2
2
|
class Cli < Thor
|
3
3
|
|
4
|
-
EXTENSION_MAP = {
|
5
|
-
'default' => 'TRAFFIC_PODCAST',
|
6
|
-
'mp4' => 'TRAFFIC_PODCAST',
|
7
|
-
'f4v' => 'TRAFFIC_PODCAST',
|
8
|
-
'flv' => 'TRAFFIC_PODCAST',
|
9
|
-
'ts' => 'TRAFFIC_MOBILE'
|
10
|
-
}
|
11
|
-
|
12
4
|
class_option :mysql_host, :default => 'localhost'
|
13
5
|
class_option :mysql_user, :default => 'root'
|
14
6
|
class_option :mysql_db, :default => 'Syslog'
|
@@ -25,130 +17,73 @@ module SmsLogparser
|
|
25
17
|
def parse
|
26
18
|
start_time = Time.now
|
27
19
|
count = 0
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
if result['Message'] =~ /\/content\/.*\.(f4v|flv|mp4|ts) .*/
|
41
|
-
data = extract_data_from_msg(result['Message'])
|
42
|
-
url = "#{options[:api_base_path]}/"
|
43
|
-
url += "#{data[:customer_id]}/"
|
44
|
-
url += "#{data[:author_id]}/"
|
45
|
-
url += "#{data[:project_id]}/"
|
46
|
-
url += "#{data[:traffic_type]}/"
|
47
|
-
url += "#{data[:bytes]}"
|
48
|
-
if options[:simulate]
|
49
|
-
puts "Message ID: #{result['ID']}"
|
50
|
-
puts "URL: #{url}"
|
51
|
-
puts "Data: #{data}"
|
52
|
-
puts "-----------------------"
|
53
|
-
else
|
54
|
-
begin
|
55
|
-
RestClient.get(url)
|
56
|
-
rescue
|
57
|
-
say "Error: Can't send log to #{url}", :red
|
58
|
-
say "Aborting.", :red
|
59
|
-
break
|
60
|
-
end
|
20
|
+
begin
|
21
|
+
mysql = Mysql.new(options)
|
22
|
+
entries = mysql.get_entries
|
23
|
+
api = Api.new(options)
|
24
|
+
last_id = mysql.get_last_parse_id
|
25
|
+
entries.each do |entry|
|
26
|
+
if Parser.match(entry)
|
27
|
+
data = Parser.extract_data_from_msg(entry['Message'])
|
28
|
+
url = api.send(data)
|
29
|
+
last_id = entry['ID']
|
30
|
+
count += 1
|
31
|
+
debug_parser_output(data, url, entry) if options[:simulate]
|
61
32
|
end
|
62
|
-
count += 1
|
63
33
|
end
|
64
|
-
last_id
|
34
|
+
mysql.write_parse_result(last_id, count) unless options[:simulate]
|
35
|
+
puts "Started: #{start_time.strftime('%d.%d.%Y %T')}"
|
36
|
+
puts "Runtime: #{(Time.now - start_time).round(2)}s"
|
37
|
+
puts "Matches: #{count}"
|
38
|
+
rescue => e
|
39
|
+
say "Error: #{e.message}", :red
|
65
40
|
end
|
66
|
-
write_parse_result(last_id, count) unless options[:simulate]
|
67
|
-
puts "Started: #{start_time.strftime('%d.%d.%Y %T')}"
|
68
|
-
puts "Runtime: #{(Time.now - start_time).round(2)}s"
|
69
|
-
puts "Matches: #{count}"
|
70
41
|
end
|
71
42
|
|
72
43
|
desc "last_runs", "List the last paser runs"
|
73
44
|
def last_runs
|
74
|
-
begin
|
75
|
-
runs =
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
table
|
86
|
-
|
87
|
-
|
88
|
-
run['LastEventID']
|
89
|
-
]
|
45
|
+
begin
|
46
|
+
runs = Mysql.new(options).last_runs
|
47
|
+
if runs.size > 0
|
48
|
+
table = [%w(RunAt #Events LastEventID)]
|
49
|
+
runs.each do |run|
|
50
|
+
table << [
|
51
|
+
run['RunAt'],
|
52
|
+
run['EventsFound'],
|
53
|
+
run['LastEventID']
|
54
|
+
]
|
55
|
+
end
|
56
|
+
print_table table
|
57
|
+
else
|
58
|
+
say "No parser runs found in the database."
|
90
59
|
end
|
91
|
-
|
92
|
-
|
93
|
-
say "No parser runs found in the database."
|
60
|
+
rescue => e
|
61
|
+
say "Error: #{e.message}", :red
|
94
62
|
end
|
95
63
|
end
|
96
64
|
|
97
|
-
desc "
|
98
|
-
def
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
EventsFound INT DEFAULT 0,\
|
106
|
-
INDEX `LastEventID_I1` (`LastEventID`)
|
107
|
-
)"
|
108
|
-
)
|
109
|
-
say "OK", :green
|
65
|
+
desc "setup", "Create the parser table to track the last logs parsed"
|
66
|
+
def setup
|
67
|
+
begin
|
68
|
+
Mysql.new(options).create_parser_table
|
69
|
+
say "OK", :green
|
70
|
+
rescue => e
|
71
|
+
say "Error: #{e.message}", :red
|
72
|
+
end
|
110
73
|
end
|
111
74
|
|
112
75
|
no_commands do
|
113
|
-
def
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
say "
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
def write_parse_result(id, count)
|
124
|
-
client.query("INSERT INTO SmsParserRuns(RunAt, LastEventID, EventsFound)\
|
125
|
-
VALUES(\
|
126
|
-
'#{Time.now.strftime("%Y-%m-%d %H:%M:%S")}',\
|
127
|
-
#{id},\
|
128
|
-
#{count}
|
129
|
-
)"
|
130
|
-
)
|
131
|
-
end
|
132
|
-
|
133
|
-
def extract_data_from_msg(msg)
|
134
|
-
m = msg.match /\/content\/(\d+)\/(\d+)\/(\d+)\/.+\.(\S+)\s.*\"\s\d+\s(\d+)/
|
135
|
-
data = {
|
136
|
-
:customer_id => m[1],
|
137
|
-
:author_id => m[2],
|
138
|
-
:project_id => m[3],
|
139
|
-
:ext => m[4],
|
140
|
-
:traffic_type => (EXTENSION_MAP[m[4]] || EXTENSION_MAP['default']),
|
141
|
-
:bytes => m[5]
|
142
|
-
}
|
143
|
-
end
|
144
|
-
|
145
|
-
def client
|
146
|
-
@client ||= Mysql2::Client.new(
|
147
|
-
:host => options[:mysql_host],
|
148
|
-
:username => options[:mysql_user],
|
149
|
-
:database => options[:mysql_db]
|
150
|
-
)
|
76
|
+
def debug_parser_output(data, url, entry)
|
77
|
+
puts
|
78
|
+
say "Message ID: ", :green
|
79
|
+
say entry['ID']
|
80
|
+
say "URL: ", :green
|
81
|
+
say url
|
82
|
+
say "Data: ", :green
|
83
|
+
say data
|
84
|
+
puts
|
151
85
|
end
|
152
86
|
end
|
87
|
+
|
153
88
|
end
|
154
89
|
end
|
@@ -0,0 +1,79 @@
|
|
1
|
+
module SmsLogparser
|
2
|
+
class Mysql
|
3
|
+
|
4
|
+
def initialize(options)
|
5
|
+
@options = options
|
6
|
+
end
|
7
|
+
|
8
|
+
def client
|
9
|
+
@client ||= Mysql2::Client.new(
|
10
|
+
:host => @options[:mysql_host],
|
11
|
+
:username => @options[:mysql_user],
|
12
|
+
:database => @options[:mysql_db]
|
13
|
+
)
|
14
|
+
end
|
15
|
+
|
16
|
+
def last_runs
|
17
|
+
begin
|
18
|
+
runs = client.query(
|
19
|
+
"SELECT * FROM SmsParserRuns ORDER BY ID ASC LIMIT 10"
|
20
|
+
)
|
21
|
+
rescue Mysql2::Error => e
|
22
|
+
raise e
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def create_parser_table
|
27
|
+
begin
|
28
|
+
return client.query(
|
29
|
+
"CREATE TABLE IF NOT EXISTS\
|
30
|
+
SmsParserRuns(\
|
31
|
+
ID INT PRIMARY KEY AUTO_INCREMENT,\
|
32
|
+
RunAt datetime DEFAULT NULL,\
|
33
|
+
LastEventID INT DEFAULT NULL,\
|
34
|
+
EventsFound INT DEFAULT 0,\
|
35
|
+
INDEX `LastEventID_I1` (`LastEventID`)
|
36
|
+
)"
|
37
|
+
)
|
38
|
+
rescue Mysql2::Error => e
|
39
|
+
raise e
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def get_entries(last_id = get_last_parse_id)
|
44
|
+
begin
|
45
|
+
return client.query(
|
46
|
+
"SELECT * FROM SystemEvents\
|
47
|
+
WHERE `FromHost` like 'pcache%'\
|
48
|
+
AND ID > #{last_id} ORDER BY ID ASC"
|
49
|
+
)
|
50
|
+
rescue Mysql2::Error => e
|
51
|
+
raise e
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def write_parse_result(id, count)
|
56
|
+
client.query("INSERT INTO SmsParserRuns(RunAt, LastEventID, EventsFound)\
|
57
|
+
VALUES(\
|
58
|
+
'#{Time.now.strftime("%Y-%m-%d %H:%M:%S")}',\
|
59
|
+
#{id},\
|
60
|
+
#{count}
|
61
|
+
)"
|
62
|
+
)
|
63
|
+
end
|
64
|
+
|
65
|
+
def get_last_parse_id
|
66
|
+
id = 0
|
67
|
+
begin
|
68
|
+
last_parse = client.query(
|
69
|
+
"SELECT LastEventID FROM SmsParserRuns ORDER BY ID DESC LIMIT 1"
|
70
|
+
)
|
71
|
+
id = last_parse.first ? last_parse.first['LastEventID'] : 0
|
72
|
+
rescue Mysql2::Error => e
|
73
|
+
raise e
|
74
|
+
end
|
75
|
+
id
|
76
|
+
end
|
77
|
+
|
78
|
+
end # class
|
79
|
+
end # module
|
@@ -0,0 +1,34 @@
|
|
1
|
+
module SmsLogparser
|
2
|
+
class Parser
|
3
|
+
|
4
|
+
def self.extract_data_from_msg(msg)
|
5
|
+
m = msg.match /\/content\/(\d+)\/(\d+)\/(\d+)\/.+\.(\S+)\s.*\"\s\d+\s(\d+).+"(.*)"$/
|
6
|
+
raise "No match found." unless m
|
7
|
+
data = {
|
8
|
+
:customer_id => m[1],
|
9
|
+
:author_id => m[2],
|
10
|
+
:project_id => m[3],
|
11
|
+
:ext => m[4],
|
12
|
+
:traffic_type => Parser.get_traffic_type(m[4]),
|
13
|
+
:bytes => m[5],
|
14
|
+
:user_agent => m[6]
|
15
|
+
}
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.match(entry)
|
19
|
+
entry['Message'] =~ /\/content\/.*\.(f4v|flv|mp4|mp3|ts) .*/
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.get_traffic_type(user_agent)
|
23
|
+
case user_agent
|
24
|
+
when /.*(iTunes).*/
|
25
|
+
"TRAFFIC_PODCAST"
|
26
|
+
when /.*(IEMobile|Mobile Safari|iPhone|iPod|iPad).*/
|
27
|
+
"TRAFFIC_MOBILE"
|
28
|
+
else
|
29
|
+
"TRAFFIC_WEBCAST"
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
end # class
|
34
|
+
end # module
|
data/lib/sms-logparser.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
|
-
describe SmsLogparser do
|
3
|
+
describe SmsLogparser::Cli do
|
4
4
|
before do
|
5
5
|
TestHelper.create_test_db
|
6
6
|
TestHelper.create_sylog_db_table
|
@@ -12,7 +12,7 @@ describe SmsLogparser do
|
|
12
12
|
|
13
13
|
it "can create the parser_runs database table" do
|
14
14
|
out, err = capture_io do
|
15
|
-
TestHelper.sms_logparser.
|
15
|
+
TestHelper.sms_logparser.setup
|
16
16
|
end
|
17
17
|
out.must_match /OK/
|
18
18
|
end
|
@@ -22,7 +22,7 @@ describe SmsLogparser do
|
|
22
22
|
parser = TestHelper.sms_logparser
|
23
23
|
parser.options[:api_base_path] = 'http://devnull-as-a-service.com/dev/null/'
|
24
24
|
out, err = capture_io do
|
25
|
-
TestHelper.sms_logparser.
|
25
|
+
TestHelper.sms_logparser.setup
|
26
26
|
parser.parse
|
27
27
|
end
|
28
28
|
out.must_match /.*Matches: 10$/
|
@@ -33,22 +33,22 @@ describe SmsLogparser do
|
|
33
33
|
parser = TestHelper.sms_logparser
|
34
34
|
parser.options[:api_base_path] = 'http://devnull-as-a-service.com/dev/null/'
|
35
35
|
out, err = capture_io do
|
36
|
-
TestHelper.sms_logparser.
|
36
|
+
TestHelper.sms_logparser.setup
|
37
37
|
parser.parse
|
38
38
|
parser.parse
|
39
39
|
end
|
40
40
|
out.must_match /.*Matches: 0$/
|
41
41
|
end
|
42
42
|
|
43
|
-
it "
|
43
|
+
it "lists parser runs" do
|
44
44
|
TestHelper.seed_db(1)
|
45
45
|
parser = TestHelper.sms_logparser
|
46
46
|
parser.options[:api_base_path] = 'http://devnull-as-a-service.com/dev/null/'
|
47
47
|
out, err = capture_io do
|
48
|
-
TestHelper.sms_logparser.
|
48
|
+
TestHelper.sms_logparser.setup
|
49
49
|
parser.parse
|
50
50
|
parser.last_runs
|
51
51
|
end
|
52
52
|
assert_equal(err, "")
|
53
53
|
end
|
54
|
-
end
|
54
|
+
end
|
data/spec/spec_helper.rb
CHANGED
@@ -41,13 +41,13 @@ module TestHelper
|
|
41
41
|
end
|
42
42
|
|
43
43
|
def self.insert_logs(host = "pcache", message = "", number_of_inserts = 1)
|
44
|
-
values =
|
44
|
+
values = []
|
45
45
|
number_of_inserts.times do
|
46
|
-
values
|
46
|
+
values << "('#{host}', '#{message}')"
|
47
47
|
end
|
48
48
|
self.client.query(
|
49
49
|
"INSERT INTO #{@@mysql_db}.SystemEvents(FromHost, Message)\
|
50
|
-
VALUES #{values.
|
50
|
+
VALUES #{values.join(', ')}"
|
51
51
|
)
|
52
52
|
end
|
53
53
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sms-logparser
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- niwo
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-03-
|
11
|
+
date: 2014-03-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -95,10 +95,13 @@ files:
|
|
95
95
|
- Rakefile
|
96
96
|
- bin/sms-logparser
|
97
97
|
- lib/sms-logparser.rb
|
98
|
+
- lib/sms-logparser/api.rb
|
98
99
|
- lib/sms-logparser/cli.rb
|
100
|
+
- lib/sms-logparser/mysql.rb
|
101
|
+
- lib/sms-logparser/parser.rb
|
99
102
|
- lib/sms-logparser/version.rb
|
100
103
|
- sms-logparser.gemspec
|
101
|
-
- spec/
|
104
|
+
- spec/cli_spec.rb
|
102
105
|
- spec/spec_helper.rb
|
103
106
|
homepage: https://github.com/swisstxt/sms-logparser
|
104
107
|
licenses:
|
@@ -125,5 +128,5 @@ signing_key:
|
|
125
128
|
specification_version: 4
|
126
129
|
summary: SMS Logparser
|
127
130
|
test_files:
|
128
|
-
- spec/
|
131
|
+
- spec/cli_spec.rb
|
129
132
|
- spec/spec_helper.rb
|