sms-logparser 0.6.2 → 0.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/sms-logparser/cli.rb +17 -11
- data/lib/sms-logparser/mysql.rb +33 -11
- data/lib/sms-logparser/parser.rb +4 -2
- data/lib/sms-logparser/version.rb +1 -1
- metadata +20 -20
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9d9aa56b379edf1fef5f9322463a6fa581d38ff8
|
4
|
+
data.tar.gz: ef45045d3538b34b9612f1fca2b7fe546e5d2583
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3a5bd6d467611916e182b95428f2fa49c7ef4131d92e2cd675559344db3b46b20a8376d0f8f9df85b145459fcaf120a2bfca4229fa8fc6f2c8fc76b55960ee3f
|
7
|
+
data.tar.gz: 2324fde02fd0598af49909792cc928728d79cfd4117598f369e8ee90db43c658d1ccd8c0bb6a7d761309d12ced9a508f5c7f426dbbbde5059f129b8e56083c98
|
data/lib/sms-logparser/cli.rb
CHANGED
@@ -59,6 +59,10 @@ module SmsLogparser
|
|
59
59
|
def parse
|
60
60
|
say "Starting the parser...", :green
|
61
61
|
mysql = Mysql.new(options)
|
62
|
+
if mysql.parser_running?
|
63
|
+
say "Aborting. Another instance of the parser is already running.", :red
|
64
|
+
exit
|
65
|
+
end
|
62
66
|
state = {
|
63
67
|
last_event_id: mysql.get_last_parse_id,
|
64
68
|
match_count: 0,
|
@@ -66,26 +70,29 @@ module SmsLogparser
|
|
66
70
|
run_at: Time.now,
|
67
71
|
run_time: 0.0
|
68
72
|
}
|
73
|
+
state = mysql.start_run(state) unless options[:simulate]
|
69
74
|
api = Api.new(options)
|
70
75
|
mysql.get_entries(last_id: state[:last_event_id], limit: options[:limit]) do |entries|
|
71
76
|
entries.each do |entry|
|
72
|
-
|
73
|
-
|
74
|
-
state[:last_event_id] = entry['ID']
|
77
|
+
Parser.extract_data_from_msg(entry['Message']) do |data|
|
78
|
+
urls = api.send(data)
|
75
79
|
state[:match_count] += 1
|
76
|
-
verbose_parser_output(data,
|
80
|
+
verbose_parser_output(data, urls, entry) if options[:verbose]
|
77
81
|
end
|
82
|
+
state[:last_event_id] = entry['ID']
|
78
83
|
end
|
79
84
|
end
|
80
85
|
rescue => e
|
81
86
|
say "Error: #{e.message}", :red
|
82
87
|
say "Aborting parser run...", :red
|
83
|
-
state[:status] = STATUS[:api_error]
|
88
|
+
state[:status] = STATUS[:api_error] if state
|
84
89
|
ensure
|
85
90
|
begin
|
86
|
-
|
87
|
-
|
88
|
-
|
91
|
+
if mysql
|
92
|
+
state[:run_time] = (Time.now - state[:run_at]).round(2)
|
93
|
+
mysql.write_parse_result(state) unless options[:simulate]
|
94
|
+
print_parse_results(state)
|
95
|
+
end
|
89
96
|
rescue => e
|
90
97
|
say "Error: #{e.message}", :red
|
91
98
|
say(e.backtrace.join("\n"), :yellow) if options[:debug]
|
@@ -142,7 +149,7 @@ module SmsLogparser
|
|
142
149
|
def verbose_parser_output(data, uris, entry)
|
143
150
|
say "ID:\t", :cyan
|
144
151
|
say entry['ID']
|
145
|
-
say "
|
152
|
+
say "URL:\t", :cyan
|
146
153
|
say uris.join("\n\t")
|
147
154
|
say "Data:\t", :cyan
|
148
155
|
say data.map{|k,v| "#{k}:\t#{v}"}.join("\n\t") || "\n"
|
@@ -158,8 +165,7 @@ module SmsLogparser
|
|
158
165
|
say "#{res[:run_time]}s"
|
159
166
|
say "Status:\t\t", :cyan
|
160
167
|
say STATUS.key(res[:status]).upcase
|
161
|
-
|
162
|
-
say("Events #{action}:\t", :cyan)
|
168
|
+
say("Events #{options[:simulate] ? "found" : "sent"}:\t", :cyan)
|
163
169
|
say res[:match_count]
|
164
170
|
end
|
165
171
|
|
data/lib/sms-logparser/mysql.rb
CHANGED
@@ -43,15 +43,36 @@ module SmsLogparser
|
|
43
43
|
return 0
|
44
44
|
end
|
45
45
|
|
46
|
-
def
|
46
|
+
def parser_running?(running_state = 3)
|
47
|
+
last_parse = client.query(
|
48
|
+
"SELECT status FROM sms_logparser_runs ORDER BY id DESC LIMIT 1"
|
49
|
+
)
|
50
|
+
if entry = last_parse.first
|
51
|
+
return false unless entry['status'] == running_state
|
52
|
+
end
|
53
|
+
false
|
54
|
+
end
|
55
|
+
|
56
|
+
def start_run(options)
|
47
57
|
client.query(
|
48
|
-
"INSERT INTO sms_logparser_runs(run_at,
|
49
|
-
VALUES(\
|
58
|
+
"INSERT INTO sms_logparser_runs (run_at, status)\
|
59
|
+
VALUES (\
|
50
60
|
'#{options[:run_at].strftime("%Y-%m-%d %H:%M:%S")}',\
|
51
|
-
#{options[:
|
52
|
-
|
53
|
-
|
54
|
-
|
61
|
+
#{options[:status]}\
|
62
|
+
)"
|
63
|
+
)
|
64
|
+
options[:id] = client.last_id
|
65
|
+
options
|
66
|
+
end
|
67
|
+
|
68
|
+
def write_parse_result(options)
|
69
|
+
client.query(
|
70
|
+
"UPDATE sms_logparser_runs SET match_count, status, run_time\
|
71
|
+
last_event_id = #{options[:last_event_id]},\
|
72
|
+
match_count = #{options[:match_count]},\
|
73
|
+
status = #{options[:status]},\
|
74
|
+
run_time = #{options[:run_time]}\
|
75
|
+
WHERE id = #{options[id]}
|
55
76
|
)"
|
56
77
|
)
|
57
78
|
end
|
@@ -65,8 +86,8 @@ module SmsLogparser
|
|
65
86
|
end
|
66
87
|
while last_id < max_id
|
67
88
|
entries = select_entries(last_id)
|
68
|
-
last_id = entries.to_a[-1]['ID']
|
69
89
|
yield entries
|
90
|
+
last_id = entries.to_a[-1]['ID']
|
70
91
|
end
|
71
92
|
end
|
72
93
|
|
@@ -79,12 +100,13 @@ module SmsLogparser
|
|
79
100
|
|
80
101
|
private
|
81
102
|
|
103
|
+
|
104
|
+
|
82
105
|
def select_entries(last_id)
|
83
|
-
|
106
|
+
client.query("SELECT * FROM SystemEvents\
|
84
107
|
WHERE `FromHost` like '#{@host_filter}'\
|
85
108
|
ORDER BY ID ASC\
|
86
|
-
LIMIT #{last_id},#{@query_limit};"
|
87
|
-
client.query(sql)
|
109
|
+
LIMIT #{last_id},#{@query_limit};")
|
88
110
|
end
|
89
111
|
|
90
112
|
def get_last_event_id
|
data/lib/sms-logparser/parser.rb
CHANGED
@@ -7,7 +7,7 @@ module SmsLogparser
|
|
7
7
|
raise "No match found." unless m
|
8
8
|
traffic_type = Parser.get_traffic_type(m[6])
|
9
9
|
visitor_type = Parser.get_visitor_type(traffic_type, m[4])
|
10
|
-
|
10
|
+
data = {
|
11
11
|
:customer_id => m[1],
|
12
12
|
:author_id => m[2],
|
13
13
|
:project_id => m[3],
|
@@ -16,7 +16,9 @@ module SmsLogparser
|
|
16
16
|
:user_agent => m[6],
|
17
17
|
:traffic_type => traffic_type,
|
18
18
|
:visitor_type => visitor_type
|
19
|
-
}
|
19
|
+
}
|
20
|
+
return data unless block_given?
|
21
|
+
yield data
|
20
22
|
end
|
21
23
|
nil
|
22
24
|
end
|
metadata
CHANGED
@@ -1,111 +1,111 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sms-logparser
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.7.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- niwo
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-04-
|
11
|
+
date: 2014-04-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - ~>
|
17
|
+
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '1.3'
|
20
20
|
type: :development
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- - ~>
|
24
|
+
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '1.3'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: rake
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- -
|
31
|
+
- - ">="
|
32
32
|
- !ruby/object:Gem::Version
|
33
33
|
version: '0'
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- -
|
38
|
+
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: webmock
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- -
|
45
|
+
- - ">="
|
46
46
|
- !ruby/object:Gem::Version
|
47
47
|
version: '0'
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- -
|
52
|
+
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '0'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: thor
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- - ~>
|
59
|
+
- - "~>"
|
60
60
|
- !ruby/object:Gem::Version
|
61
61
|
version: 0.19.1
|
62
62
|
type: :runtime
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
|
-
- - ~>
|
66
|
+
- - "~>"
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: 0.19.1
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: mysql2
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
|
-
- -
|
73
|
+
- - ">="
|
74
74
|
- !ruby/object:Gem::Version
|
75
75
|
version: '0'
|
76
76
|
type: :runtime
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
|
-
- -
|
80
|
+
- - ">="
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: '0'
|
83
83
|
- !ruby/object:Gem::Dependency
|
84
84
|
name: faraday
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
86
86
|
requirements:
|
87
|
-
- - ~>
|
87
|
+
- - "~>"
|
88
88
|
- !ruby/object:Gem::Version
|
89
89
|
version: 0.9.0
|
90
90
|
type: :runtime
|
91
91
|
prerelease: false
|
92
92
|
version_requirements: !ruby/object:Gem::Requirement
|
93
93
|
requirements:
|
94
|
-
- - ~>
|
94
|
+
- - "~>"
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: 0.9.0
|
97
97
|
- !ruby/object:Gem::Dependency
|
98
98
|
name: net-http-persistent
|
99
99
|
requirement: !ruby/object:Gem::Requirement
|
100
100
|
requirements:
|
101
|
-
- -
|
101
|
+
- - ">="
|
102
102
|
- !ruby/object:Gem::Version
|
103
103
|
version: 2.9.4
|
104
104
|
type: :runtime
|
105
105
|
prerelease: false
|
106
106
|
version_requirements: !ruby/object:Gem::Requirement
|
107
107
|
requirements:
|
108
|
-
- -
|
108
|
+
- - ">="
|
109
109
|
- !ruby/object:Gem::Version
|
110
110
|
version: 2.9.4
|
111
111
|
description: Reads access logs stored in a MySQL database (coming from the SWISS TXT
|
@@ -117,7 +117,7 @@ executables:
|
|
117
117
|
extensions: []
|
118
118
|
extra_rdoc_files: []
|
119
119
|
files:
|
120
|
-
- .gitignore
|
120
|
+
- ".gitignore"
|
121
121
|
- Gemfile
|
122
122
|
- LICENSE.txt
|
123
123
|
- README.md
|
@@ -144,17 +144,17 @@ require_paths:
|
|
144
144
|
- lib
|
145
145
|
required_ruby_version: !ruby/object:Gem::Requirement
|
146
146
|
requirements:
|
147
|
-
- -
|
147
|
+
- - ">="
|
148
148
|
- !ruby/object:Gem::Version
|
149
149
|
version: 1.9.3
|
150
150
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
151
151
|
requirements:
|
152
|
-
- -
|
152
|
+
- - ">="
|
153
153
|
- !ruby/object:Gem::Version
|
154
154
|
version: '0'
|
155
155
|
requirements: []
|
156
156
|
rubyforge_project:
|
157
|
-
rubygems_version: 2.0
|
157
|
+
rubygems_version: 2.2.0
|
158
158
|
signing_key:
|
159
159
|
specification_version: 4
|
160
160
|
summary: sms-logparser - Logparser for Simplex Media Server (SMS)
|