sms-logparser 0.5.0 → 0.6.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/sms-logparser/api.rb +11 -13
- data/lib/sms-logparser/cli.rb +35 -29
- data/lib/sms-logparser/mysql.rb +67 -67
- data/lib/sms-logparser/parser.rb +17 -14
- data/lib/sms-logparser/version.rb +1 -1
- data/sms-logparser.gemspec +2 -1
- metadata +19 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ac7a2c7bd9b90b797a3ee9a0200117c47139f54b
|
4
|
+
data.tar.gz: 88b67958f0fd4b641414a028b329be5bbd50e0da
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8e2fca495d375a1edf97a182552bdb38b15001929dacc3ae777c70a8a33ec377ced9c481d6b636acdb75aab90e6fe3384ee981b92841835fc2c78ad9b369b665
|
7
|
+
data.tar.gz: 40f73533a55aa2deb3e2f76a1fda30037e375e179cee86401df26b7551c0027b12c289dae30043419b691e117b507eaab8464dd44f33061b601b1e0be1721522
|
data/lib/sms-logparser/api.rb
CHANGED
@@ -8,20 +8,18 @@ module SmsLogparser
|
|
8
8
|
end
|
9
9
|
|
10
10
|
def connection
|
11
|
-
@connection
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
if @options[:api_key]
|
22
|
-
conn.headers['X-simplex-api-key'] = @options[:api_key]
|
11
|
+
unless @connection
|
12
|
+
@connection = Faraday.new(url: @base_url) do |faraday|
|
13
|
+
faraday.request :url_encoded
|
14
|
+
faraday.response :logger if @options[:debug]
|
15
|
+
faraday.adapter :net_http_persistent
|
16
|
+
end
|
17
|
+
@connection.headers[:user_agent] = "sms-logparser v#{SmsLogparser::VERSION}"
|
18
|
+
if @options[:api_key]
|
19
|
+
@connection.headers['X-simplex-api-key'] = @options[:api_key]
|
20
|
+
end
|
23
21
|
end
|
24
|
-
|
22
|
+
@connection
|
25
23
|
end
|
26
24
|
|
27
25
|
def send(data)
|
data/lib/sms-logparser/cli.rb
CHANGED
@@ -55,47 +55,52 @@ module SmsLogparser
|
|
55
55
|
type: :boolean,
|
56
56
|
default: false
|
57
57
|
def parse
|
58
|
+
say "Starting the parser...", :green
|
58
59
|
start_time = Time.now
|
59
60
|
count = 0
|
60
61
|
begin
|
61
62
|
mysql = Mysql.new(options)
|
62
|
-
entries = mysql.get_entries(limit: options[:limit])
|
63
63
|
api = Api.new(options)
|
64
64
|
last_id = mysql.get_last_parse_id
|
65
65
|
status = STATUS[:ok]
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
last_id = entry['ID']
|
78
|
-
count += 1
|
79
|
-
if options[:verbose]
|
80
|
-
verbose_parser_output(data, uris, entry)
|
66
|
+
begin
|
67
|
+
mysql.get_entries(last_id: last_id, limit: options[:limit]) do |entries|
|
68
|
+
entries.each do |entry|
|
69
|
+
if data = Parser.extract_data_from_msg(entry['Message'])
|
70
|
+
uris = api.send(data)
|
71
|
+
last_id = entry['ID']
|
72
|
+
count += 1
|
73
|
+
if options[:verbose]
|
74
|
+
verbose_parser_output(data, uris, entry)
|
75
|
+
end
|
76
|
+
end
|
81
77
|
end
|
82
78
|
end
|
83
|
-
|
84
|
-
|
85
|
-
|
79
|
+
rescue => e
|
80
|
+
say "Error: #{e.message}", :red
|
81
|
+
say "Aborting parser run...", :red
|
82
|
+
status = STATUS[:api_error]
|
83
|
+
ensure
|
84
|
+
mysql.write_parse_result(
|
85
|
+
last_event_id: last_id,
|
86
|
+
match_count: count,
|
87
|
+
status: status,
|
88
|
+
run_at: start_time,
|
89
|
+
run_time: (Time.now - start_time).round(2)
|
90
|
+
) unless options[:simulate]
|
86
91
|
end
|
87
92
|
say "Started:\t", :cyan
|
88
93
|
say start_time.strftime('%d.%d.%Y %T')
|
89
94
|
say "Runtime:\t", :cyan
|
90
95
|
say "#{(Time.now - start_time).round(2)}s"
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
end
|
96
|
+
say "Status:\t\t", :cyan
|
97
|
+
say STATUS.key(status).upcase
|
98
|
+
action = options[:simulate] ? "found" : "sent"
|
99
|
+
say("Events #{action}:\t", :cyan)
|
96
100
|
say count
|
97
101
|
rescue => e
|
98
102
|
say "Error: #{e.message}", :red
|
103
|
+
say e.backtrace; :yellow
|
99
104
|
end
|
100
105
|
end
|
101
106
|
|
@@ -109,13 +114,14 @@ module SmsLogparser
|
|
109
114
|
begin
|
110
115
|
runs = Mysql.new(options).last_runs(options[:results])
|
111
116
|
if runs.size > 0
|
112
|
-
table = [%w(
|
117
|
+
table = [%w(run_at run_time match_count last_event_id status)]
|
113
118
|
runs.to_a.reverse.each do |run|
|
114
119
|
table << [
|
115
|
-
run['
|
116
|
-
run['
|
117
|
-
run['
|
118
|
-
|
120
|
+
run['run_at'],
|
121
|
+
"#{run['run_time']}s",
|
122
|
+
run['match_count'],
|
123
|
+
run['last_event_id'],
|
124
|
+
STATUS.key(run['status']).upcase
|
119
125
|
]
|
120
126
|
end
|
121
127
|
print_table table
|
data/lib/sms-logparser/mysql.rb
CHANGED
@@ -3,6 +3,9 @@ module SmsLogparser
|
|
3
3
|
|
4
4
|
def initialize(options)
|
5
5
|
@options = options
|
6
|
+
@host_filter = options[:host_filter] || 'pcache%'
|
7
|
+
@query_limit = options[:query_limit] || 1000
|
8
|
+
@client = client
|
6
9
|
end
|
7
10
|
|
8
11
|
def client
|
@@ -15,23 +18,9 @@ module SmsLogparser
|
|
15
18
|
end
|
16
19
|
|
17
20
|
def last_runs(results = 10)
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
)
|
22
|
-
rescue Mysql2::Error => e
|
23
|
-
raise e
|
24
|
-
end
|
25
|
-
end
|
26
|
-
|
27
|
-
def parser_table_exists?
|
28
|
-
begin
|
29
|
-
return client.query(
|
30
|
-
"SHOW TABLES LIKE 'SmsParserRuns'"
|
31
|
-
).size > 0
|
32
|
-
rescue Mysql2::Error => e
|
33
|
-
raise e
|
34
|
-
end
|
21
|
+
runs = client.query(
|
22
|
+
"SELECT * FROM sms_logparser_runs ORDER BY id DESC LIMIT #{results}"
|
23
|
+
)
|
35
24
|
end
|
36
25
|
|
37
26
|
def create_parser_table(force = false)
|
@@ -40,69 +29,80 @@ module SmsLogparser
|
|
40
29
|
elsif parser_table_exists?
|
41
30
|
return 1
|
42
31
|
end
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
rescue Mysql2::Error => e
|
55
|
-
raise e
|
56
|
-
end
|
32
|
+
client.query(
|
33
|
+
"CREATE TABLE sms_logparser_runs(\
|
34
|
+
id SERIAL PRIMARY KEY AUTO_INCREMENT,\
|
35
|
+
run_at datetime DEFAULT NULL,\
|
36
|
+
last_event_id BIGINT(20) UNSIGNED DEFAULT 0,\
|
37
|
+
match_count INT UNSIGNED DEFAULT 0,\
|
38
|
+
status TINYINT UNSIGNED DEFAULT 0,\
|
39
|
+
run_time DOUBLE(10,2) NOT NULL DEFAULT '0.00',\
|
40
|
+
INDEX `last_event_id_I1` (`last_event_id`)
|
41
|
+
)"
|
42
|
+
)
|
57
43
|
return 0
|
58
44
|
end
|
59
45
|
|
60
|
-
def
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
"
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
46
|
+
def write_parse_result(options={})
|
47
|
+
client.query(
|
48
|
+
"INSERT INTO sms_logparser_runs(run_at, last_event_id, match_count, status, run_time)\
|
49
|
+
VALUES(\
|
50
|
+
'#{options[:run_at].strftime("%Y-%m-%d %H:%M:%S")}',\
|
51
|
+
#{options[:last_event_id]},\
|
52
|
+
#{options[:match_count]},\
|
53
|
+
#{options[:status]},\
|
54
|
+
#{options[:run_time]}
|
55
|
+
)"
|
56
|
+
)
|
69
57
|
end
|
70
58
|
|
71
59
|
def get_entries(options={})
|
72
60
|
last_id = options[:last_id] || get_last_parse_id
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
61
|
+
if options[:limit]
|
62
|
+
max_id = last_id + options[:limit] + 1
|
63
|
+
else
|
64
|
+
max_id = get_last_event_id
|
65
|
+
end
|
66
|
+
while last_id < max_id
|
67
|
+
entries = select_entries(last_id)
|
68
|
+
last_id = entries.to_a[-1]['ID']
|
69
|
+
yield entries
|
81
70
|
end
|
82
71
|
end
|
83
72
|
|
84
|
-
def
|
85
|
-
client.query(
|
86
|
-
|
87
|
-
'#{Time.now.strftime("%Y-%m-%d %H:%M:%S")}',\
|
88
|
-
#{id},\
|
89
|
-
#{count},\
|
90
|
-
#{status}
|
91
|
-
)"
|
73
|
+
def get_last_parse_id
|
74
|
+
last_parse = client.query(
|
75
|
+
"SELECT last_event_id FROM sms_logparser_runs ORDER BY id DESC LIMIT 1"
|
92
76
|
)
|
77
|
+
last_parse.first ? last_parse.first['last_event_id'] : 0
|
93
78
|
end
|
94
79
|
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
80
|
+
private
|
81
|
+
|
82
|
+
def select_entries(last_id)
|
83
|
+
sql = "SELECT * FROM SystemEvents\
|
84
|
+
WHERE `FromHost` like '#{@host_filter}'\
|
85
|
+
ORDER BY ID ASC\
|
86
|
+
LIMIT #{last_id},#{@query_limit};"
|
87
|
+
client.query(sql)
|
88
|
+
end
|
89
|
+
|
90
|
+
def get_last_event_id
|
91
|
+
last_event = client.query(
|
92
|
+
"SELECT ID FROM SystemEvents ORDER BY ID DESC LIMIT 1"
|
93
|
+
)
|
94
|
+
last_event.first ? last_event.first['ID'] : 0
|
95
|
+
end
|
96
|
+
|
97
|
+
def parser_table_exists?
|
98
|
+
client.query(
|
99
|
+
"SHOW TABLES LIKE 'sms_logparser_runs'"
|
100
|
+
).size > 0
|
101
|
+
end
|
102
|
+
|
103
|
+
def drop_parser_table
|
104
|
+
return nil unless parser_table_exists?
|
105
|
+
client.query("DROP TABLE sms_logparser_runs")
|
106
106
|
end
|
107
107
|
|
108
108
|
end # class
|
data/lib/sms-logparser/parser.rb
CHANGED
@@ -2,20 +2,23 @@ module SmsLogparser
|
|
2
2
|
class Parser
|
3
3
|
|
4
4
|
def self.extract_data_from_msg(message)
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
5
|
+
if self.match?(message)
|
6
|
+
m = message.match /\/content\/(\d+)\/(\d+)\/(\d+)\/(\w+\.\w+)\s.*\"\s\d+\s(\d+).+"(.*)"$/
|
7
|
+
raise "No match found." unless m
|
8
|
+
traffic_type = Parser.get_traffic_type(m[6])
|
9
|
+
visitor_type = Parser.get_visitor_type(traffic_type, m[4])
|
10
|
+
return {
|
11
|
+
:customer_id => m[1],
|
12
|
+
:author_id => m[2],
|
13
|
+
:project_id => m[3],
|
14
|
+
:file => m[4],
|
15
|
+
:bytes => m[5],
|
16
|
+
:user_agent => m[6],
|
17
|
+
:traffic_type => traffic_type,
|
18
|
+
:visitor_type => visitor_type
|
19
|
+
}
|
20
|
+
end
|
21
|
+
nil
|
19
22
|
end
|
20
23
|
|
21
24
|
def self.match?(message)
|
data/sms-logparser.gemspec
CHANGED
@@ -24,6 +24,7 @@ Gem::Specification.new do |spec|
|
|
24
24
|
spec.add_development_dependency 'rake'
|
25
25
|
|
26
26
|
spec.add_dependency 'thor', '~> 0.19.1'
|
27
|
-
spec.add_dependency 'faraday', '~> 0.9.0'
|
28
27
|
spec.add_dependency 'mysql2'
|
28
|
+
spec.add_dependency 'faraday', '~> 0.9.0'
|
29
|
+
spec.add_dependency 'net-http-persistent', '>= 2.9.4'
|
29
30
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sms-logparser
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- niwo
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-04-
|
11
|
+
date: 2014-04-04 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -52,6 +52,20 @@ dependencies:
|
|
52
52
|
- - ~>
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: 0.19.1
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: mysql2
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - '>='
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
55
69
|
- !ruby/object:Gem::Dependency
|
56
70
|
name: faraday
|
57
71
|
requirement: !ruby/object:Gem::Requirement
|
@@ -67,19 +81,19 @@ dependencies:
|
|
67
81
|
- !ruby/object:Gem::Version
|
68
82
|
version: 0.9.0
|
69
83
|
- !ruby/object:Gem::Dependency
|
70
|
-
name:
|
84
|
+
name: net-http-persistent
|
71
85
|
requirement: !ruby/object:Gem::Requirement
|
72
86
|
requirements:
|
73
87
|
- - '>='
|
74
88
|
- !ruby/object:Gem::Version
|
75
|
-
version:
|
89
|
+
version: 2.9.4
|
76
90
|
type: :runtime
|
77
91
|
prerelease: false
|
78
92
|
version_requirements: !ruby/object:Gem::Requirement
|
79
93
|
requirements:
|
80
94
|
- - '>='
|
81
95
|
- !ruby/object:Gem::Version
|
82
|
-
version:
|
96
|
+
version: 2.9.4
|
83
97
|
description: Reads access logs stored in a MySQL database (coming from the SWISS TXT
|
84
98
|
CDN) and sends them to the SMS API.
|
85
99
|
email:
|