sms-logparser 0.8.4 → 0.9.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/sms-logparser.rb +3 -2
- data/lib/sms-logparser/api.rb +9 -6
- data/lib/sms-logparser/app_logger.rb +24 -0
- data/lib/sms-logparser/cli.rb +51 -83
- data/lib/sms-logparser/mysql.rb +6 -4
- data/lib/sms-logparser/version.rb +1 -1
- data/spec/cli_spec.rb +2 -2
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9f876436c72457987cb31430a76668f0613cde56
|
4
|
+
data.tar.gz: cb5a9b5a5b6d4d2a09bded36130cbe557492dc10
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1c258366166a28499e44ab7867d7a6b4e6cbdfca5a837bd6910322cd8ef8c022127518539954fea02303cc7d22d01708b3a1a2951dc69811e8db37a053ca1825
|
7
|
+
data.tar.gz: b5b2c2199aa31028db755f46cdcaf2a408232a8e6cdd0359b9084ff399db4645f31e3bbbeb97419cda1a7f6445322b96d0b15f43e3060150826f7041a100d59c
|
data/lib/sms-logparser.rb
CHANGED
@@ -1,10 +1,11 @@
|
|
1
1
|
require "rubygems" # ruby1.9 doesn't "require" it though
|
2
2
|
require "thor"
|
3
|
-
require
|
4
|
-
require
|
3
|
+
require "mysql2"
|
4
|
+
require "faraday"
|
5
5
|
|
6
6
|
require "sms-logparser/version"
|
7
7
|
require "sms-logparser/mysql"
|
8
8
|
require "sms-logparser/parser"
|
9
9
|
require "sms-logparser/api"
|
10
|
+
require "sms-logparser/app_logger"
|
10
11
|
require "sms-logparser/cli"
|
data/lib/sms-logparser/api.rb
CHANGED
@@ -4,25 +4,26 @@ module SmsLogparser
|
|
4
4
|
|
5
5
|
def initialize(options)
|
6
6
|
@options = options
|
7
|
-
@base_url = URI(@options[:api_base_url] || 'http://localhost:8080/creator/rest/')
|
7
|
+
@base_url = URI(@options[:api_base_url] || 'http://localhost:8080/creator/rest/')
|
8
8
|
@url = @base_url.to_s.chomp(@base_url.path)
|
9
|
+
@base_path = (@base_url.path << "/").squeeze('/')
|
9
10
|
@accepted_responses = parse_status_codes(options[:accepted_api_responses]) || [200]
|
10
11
|
@connection = connection
|
11
12
|
end
|
12
13
|
|
13
14
|
def send(data)
|
14
15
|
requests = []
|
15
|
-
|
16
|
+
path = @base_path << [data[:customer_id], data[:author_id], data[:project_id]].join('/')
|
16
17
|
unless data[:file] =~ /.*\.m3u8$/
|
17
18
|
requests << {
|
18
19
|
url: @url,
|
19
|
-
uri: [
|
20
|
+
uri: [path, data[:traffic_type], data[:bytes]].join('/'),
|
20
21
|
}
|
21
22
|
end
|
22
23
|
if data[:visitor_type]
|
23
24
|
requests << {
|
24
25
|
url: @url,
|
25
|
-
uri: [
|
26
|
+
uri: [path, data[:visitor_type], 1].join('/')
|
26
27
|
}
|
27
28
|
end
|
28
29
|
unless @options[:simulate]
|
@@ -45,9 +46,11 @@ module SmsLogparser
|
|
45
46
|
private
|
46
47
|
|
47
48
|
def connection
|
48
|
-
connection = Faraday.new(url: @url) do |faraday|
|
49
|
+
connection = Faraday.new(url: @url, request: {timeout: 5}) do |faraday|
|
49
50
|
faraday.request :url_encoded
|
50
|
-
|
51
|
+
if @options[:debug]
|
52
|
+
faraday.use Faraday::Response::Logger, SmsLogparser::AppLogger.instance
|
53
|
+
end
|
51
54
|
faraday.adapter :net_http_persistent
|
52
55
|
end
|
53
56
|
connection.headers[:user_agent] = "sms-logparser v#{SmsLogparser::VERSION}"
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module SmsLogparser
|
2
|
+
require 'logger'
|
3
|
+
require 'singleton'
|
4
|
+
|
5
|
+
class AppLogger < Logger
|
6
|
+
include Singleton
|
7
|
+
|
8
|
+
def initialize
|
9
|
+
@level = INFO
|
10
|
+
@logdev = STDOUT
|
11
|
+
@progname ||= 'sms-logparser'
|
12
|
+
@formatter = proc do |severity, datetime, progname, msg|
|
13
|
+
"#{datetime} #{severity} [sms-logparser v#{SmsLogparser::VERSION}] #{msg}\n"
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def set_log_device(log_file_path = nil)
|
18
|
+
device = log_file_path ? File.open(log_file_path, "a") : STDOUT
|
19
|
+
@logdev = Logger::LogDevice.new(device)
|
20
|
+
self
|
21
|
+
end
|
22
|
+
|
23
|
+
end
|
24
|
+
end
|
data/lib/sms-logparser/cli.rb
CHANGED
@@ -9,21 +9,12 @@ module SmsLogparser
|
|
9
9
|
aliases: %w(-c),
|
10
10
|
desc: "Configuration file for default options"
|
11
11
|
|
12
|
-
class_option :
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
class_option :
|
17
|
-
|
18
|
-
desc: "MySQL user (Default: root)"
|
19
|
-
|
20
|
-
class_option :mysql_password,
|
21
|
-
aliases: %w(-p),
|
22
|
-
desc: "MySQL password"
|
23
|
-
|
24
|
-
class_option :mysql_db,
|
25
|
-
aliases: %w(-d),
|
26
|
-
desc: "MySQL database (Default: Syslog)"
|
12
|
+
class_option :debug, type: :boolean, desc: "Show debug output"
|
13
|
+
class_option :logfile, desc: "Path to the logfile (Default: STDOUT)"
|
14
|
+
class_option :mysql_host, aliases: %w(-h), desc: "MySQL host"
|
15
|
+
class_option :mysql_user, aliases: %w(-u), desc: "MySQL user (Default: root)"
|
16
|
+
class_option :mysql_password, aliases: %w(-p), desc: "MySQL password"
|
17
|
+
class_option :mysql_db, aliases: %w(-d), desc: "MySQL database (Default: Syslog)"
|
27
18
|
|
28
19
|
desc "version", "Print sms-logparser version number"
|
29
20
|
def version
|
@@ -32,45 +23,29 @@ module SmsLogparser
|
|
32
23
|
map %w(-v --version) => :version
|
33
24
|
|
34
25
|
desc "parse", "Check the database for pcache logs and send them to the SMS-API"
|
35
|
-
option :api_base_url,
|
36
|
-
aliases: %w(-a),
|
26
|
+
option :api_base_url, aliases: %w(-a),
|
37
27
|
desc: "Base path of the SMS API (Default: http://localhost:8080/creator/rest/)"
|
38
|
-
option :api_key,
|
39
|
-
|
40
|
-
desc: "SMS API Key"
|
41
|
-
option :simulate,
|
42
|
-
type: :boolean,
|
43
|
-
default: false,
|
44
|
-
aliases: %w(-s),
|
28
|
+
option :api_key, aliases: %w(-k), desc: "SMS API Key"
|
29
|
+
option :simulate, type: :boolean, aliases: %w(-s),
|
45
30
|
desc: "Dry run without submitting any data"
|
46
|
-
option :verbose,
|
47
|
-
|
48
|
-
|
49
|
-
aliases: %w(-v),
|
50
|
-
desc: "Verbose output"
|
51
|
-
option :limit,
|
52
|
-
type: :numeric,
|
53
|
-
aliases: %w(-l),
|
54
|
-
desc: "Limit the number of entries to query"
|
55
|
-
option :debug,
|
56
|
-
type: :boolean,
|
57
|
-
default: false,
|
58
|
-
desc: "Show debug output"
|
59
|
-
option :accepted_api_responses,
|
60
|
-
type: :array,
|
31
|
+
option :verbose, type: :boolean, aliases: %w(-v), desc: "Verbose output"
|
32
|
+
option :limit, type: :numeric, aliases: %w(-l), desc: "Limit the number of entries to query"
|
33
|
+
option :accepted_api_responses, type: :array, aliases: %w(-r),
|
61
34
|
desc: "API HTTP responses which are accepted (Default: only accept 200)."
|
62
35
|
def parse
|
63
|
-
|
36
|
+
start_message = "Parser started"
|
37
|
+
start_message += options[:simulate] ? " in simulation mode." : "."
|
38
|
+
logger.info(start_message)
|
64
39
|
mysql = Mysql.new(options)
|
65
40
|
if !options[:simulate] && mysql.parser_running?
|
66
|
-
|
41
|
+
logger.warn("Exit. Another instance of the parser is already running.")
|
67
42
|
exit
|
68
43
|
end
|
69
44
|
state = {
|
70
45
|
last_event_id: mysql.get_last_parse_id,
|
71
46
|
match_count: 0,
|
72
47
|
status: STATUS[:running],
|
73
|
-
|
48
|
+
started_at: Time.now,
|
74
49
|
run_time: 0.0
|
75
50
|
}
|
76
51
|
state = mysql.start_run(state) unless options[:simulate]
|
@@ -81,45 +56,42 @@ module SmsLogparser
|
|
81
56
|
if data
|
82
57
|
requests = api.send(data)
|
83
58
|
state[:match_count] += 1
|
84
|
-
verbose_parser_output(
|
59
|
+
verbose_parser_output(entry['ID'], data, requests) if options[:verbose]
|
60
|
+
state[:last_event_id] = entry['ID']
|
85
61
|
end
|
86
62
|
end
|
87
|
-
state[:last_event_id] = entry['ID']
|
88
63
|
end
|
89
64
|
end
|
90
65
|
state[:status] = STATUS[:ok]
|
91
66
|
rescue => e
|
92
|
-
|
93
|
-
say "Aborting parser run...", :red
|
67
|
+
logger.error("#{e.message}. Aborting the parser run.")
|
94
68
|
say(e.backtrace.join("\n"), :yellow) if options[:debug]
|
95
69
|
state[:status] = STATUS[:api_error] if state
|
96
70
|
ensure
|
97
71
|
begin
|
98
72
|
if mysql && state
|
99
|
-
state[:run_time] = (Time.now - state[:
|
73
|
+
state[:run_time] = (Time.now - state[:started_at]).round(2)
|
100
74
|
if state[:id]
|
101
75
|
mysql.write_parse_result(state) unless options[:simulate]
|
102
76
|
end
|
103
|
-
|
77
|
+
log_parse_results(state)
|
104
78
|
end
|
105
79
|
rescue => e
|
106
|
-
|
80
|
+
logger.fatal(e.message)
|
107
81
|
say(e.backtrace.join("\n"), :yellow) if options[:debug]
|
108
82
|
end
|
109
83
|
end
|
110
84
|
|
111
85
|
desc "history", "List the last paser runs"
|
112
|
-
option :results,
|
113
|
-
type: :numeric,
|
114
|
-
default: 10,
|
115
|
-
aliases: %w(-n),
|
86
|
+
option :results, type: :numeric, default: 10, aliases: %w(-n),
|
116
87
|
desc: "Number of results to display"
|
117
88
|
def history
|
118
89
|
runs = Mysql.new(options).last_runs(options[:results])
|
119
90
|
if runs && runs.size > 0
|
120
|
-
table = [%w(
|
91
|
+
table = [%w(id started_at count last_id status run_time)]
|
121
92
|
runs.to_a.reverse.each do |run|
|
122
93
|
table << [
|
94
|
+
run['id'],
|
123
95
|
run['run_at'].strftime('%d.%d.%Y %T'),
|
124
96
|
run['match_count'],
|
125
97
|
run['last_event_id'],
|
@@ -132,50 +104,46 @@ module SmsLogparser
|
|
132
104
|
say "No parser runs found in the database."
|
133
105
|
end
|
134
106
|
rescue => e
|
135
|
-
|
136
|
-
|
107
|
+
logger.fatal(e.message)
|
108
|
+
say(e.backtrace.join("\n"), :yellow) if options[:debug]
|
137
109
|
end
|
138
110
|
|
139
111
|
desc "setup", "Create the parser table to track the last logs parsed"
|
140
|
-
option :force,
|
141
|
-
type: :boolean,
|
142
|
-
default: false,
|
143
|
-
aliases: %w(-f),
|
112
|
+
option :force, type: :boolean, default: false, aliases: %w(-f),
|
144
113
|
desc: "Drop an existing table if it exists"
|
145
114
|
def setup
|
146
115
|
case Mysql.new(options).create_parser_table(options[:force])
|
147
116
|
when 0
|
148
|
-
|
117
|
+
logger.info("Created database table.")
|
149
118
|
when 1
|
150
|
-
|
119
|
+
logger.warn("Table already exists.")
|
120
|
+
when 2
|
121
|
+
logger.info("Recreated database tables.")
|
151
122
|
end
|
152
123
|
rescue => e
|
153
|
-
|
154
|
-
|
124
|
+
logger.fatal(e.message)
|
125
|
+
say(e.backtrace.join("\n"), :yellow) if options[:debug]
|
155
126
|
end
|
156
127
|
|
157
128
|
no_commands do
|
158
|
-
def
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
129
|
+
def logger
|
130
|
+
SmsLogparser::AppLogger.instance.set_log_device(options[:logfile])
|
131
|
+
end
|
132
|
+
|
133
|
+
def verbose_parser_output(entry_id, data, requests)
|
134
|
+
logger.info {
|
135
|
+
"parsing data for #{entry_id} (#{data.map{|k,v| "#{k}=\"#{v || '-'}\""}.join(" ") || ''})"
|
136
|
+
}
|
137
|
+
logger.info {
|
138
|
+
"parser urls for entry #{entry_id} (#{requests.map {|req| "#{req[:url]}#{req[:uri]}"}.join(" ")})"
|
139
|
+
}
|
168
140
|
end
|
169
141
|
|
170
|
-
def
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
say "Status:\t\t", :cyan
|
176
|
-
say STATUS.key(res[:status]).upcase
|
177
|
-
say("Events #{options[:simulate] ? "found" : "sent"}:\t", :cyan)
|
178
|
-
say res[:match_count]
|
142
|
+
def log_parse_results(res)
|
143
|
+
res = res.reject {|k,v| %w(:id :run_at).include? k}
|
144
|
+
message = "Parser #{options[:simulate] ? 'simulation' : 'run'} ended."
|
145
|
+
message += " (" + res.map {|k,v| "#{k}=\"#{v}\""}.join(' ') + ")"
|
146
|
+
logger.info(message)
|
179
147
|
end
|
180
148
|
|
181
149
|
def options
|
data/lib/sms-logparser/mysql.rb
CHANGED
@@ -24,8 +24,9 @@ module SmsLogparser
|
|
24
24
|
end
|
25
25
|
|
26
26
|
def create_parser_table(force = false)
|
27
|
+
status = 0
|
27
28
|
if force
|
28
|
-
drop_parser_table
|
29
|
+
status = drop_parser_table ? 2 : 0
|
29
30
|
elsif parser_table_exists?
|
30
31
|
return 1
|
31
32
|
end
|
@@ -40,7 +41,7 @@ module SmsLogparser
|
|
40
41
|
INDEX `last_event_id_I1` (`last_event_id`)
|
41
42
|
)"
|
42
43
|
)
|
43
|
-
|
44
|
+
status
|
44
45
|
end
|
45
46
|
|
46
47
|
def parser_running?(running_state = 3)
|
@@ -57,7 +58,7 @@ module SmsLogparser
|
|
57
58
|
client.query(
|
58
59
|
"INSERT INTO sms_logparser_runs (run_at, status)\
|
59
60
|
VALUES (\
|
60
|
-
'#{options[:
|
61
|
+
'#{options[:started_at].strftime("%Y-%m-%d %H:%M:%S")}',\
|
61
62
|
#{options[:status]}\
|
62
63
|
)"
|
63
64
|
)
|
@@ -131,8 +132,9 @@ module SmsLogparser
|
|
131
132
|
end
|
132
133
|
|
133
134
|
def drop_parser_table
|
134
|
-
return
|
135
|
+
return false unless parser_table_exists?
|
135
136
|
client.query("DROP TABLE sms_logparser_runs")
|
137
|
+
true
|
136
138
|
end
|
137
139
|
|
138
140
|
end # class
|
data/spec/cli_spec.rb
CHANGED
@@ -19,7 +19,7 @@ describe SmsLogparser::Cli do
|
|
19
19
|
out, err = capture_io do
|
20
20
|
parser.setup
|
21
21
|
end
|
22
|
-
out.must_match
|
22
|
+
out.must_match /.*Created database table./
|
23
23
|
end
|
24
24
|
|
25
25
|
it "can parse a log database and find matches" do
|
@@ -31,7 +31,7 @@ describe SmsLogparser::Cli do
|
|
31
31
|
parser.setup
|
32
32
|
parser.parse
|
33
33
|
end
|
34
|
-
out.must_match
|
34
|
+
out.must_match /.*match_count=10.*/
|
35
35
|
end
|
36
36
|
|
37
37
|
# it "skips over already parsed logs" do
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sms-logparser
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.9.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- niwo
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-04-
|
11
|
+
date: 2014-04-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -125,6 +125,7 @@ files:
|
|
125
125
|
- bin/sms-logparser
|
126
126
|
- lib/sms-logparser.rb
|
127
127
|
- lib/sms-logparser/api.rb
|
128
|
+
- lib/sms-logparser/app_logger.rb
|
128
129
|
- lib/sms-logparser/cli.rb
|
129
130
|
- lib/sms-logparser/mysql.rb
|
130
131
|
- lib/sms-logparser/parser.rb
|