sms-logparser 0.9.6 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/sms-logparser.rb +1 -1
- data/lib/sms-logparser/api.rb +25 -20
- data/lib/sms-logparser/cli.rb +20 -16
- data/lib/sms-logparser/{app_logger.rb → loggster.rb} +19 -1
- data/lib/sms-logparser/mysql.rb +2 -2
- data/lib/sms-logparser/version.rb +1 -1
- data/spec/cli_spec.rb +13 -11
- metadata +21 -21
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fb123379f0d4fe32a268365460e4e916d02b962f
|
4
|
+
data.tar.gz: c2502c598e29d0d284ba10c7e2a3395ee3a2a717
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f24537178e814531fd41c5cfe060e65f070df80eff68e2ac48b1fd1ebee1d0f269da03d46317aa120db64643488e0419f20c4933457d2f0c39212cf01491a50d
|
7
|
+
data.tar.gz: a91d0bdb92e8543ecd352798cddb5c1c961f8434c10b312538423bda419069935f8e7e256ab295405b4d29bc37724b19c86b220deea6bc83246f517ea2e51400
|
data/lib/sms-logparser.rb
CHANGED
data/lib/sms-logparser/api.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
module SmsLogparser
|
2
2
|
class Api
|
3
3
|
require 'uri'
|
4
|
+
require 'thread'
|
4
5
|
|
5
6
|
def initialize(options)
|
6
7
|
@options = options
|
@@ -12,6 +13,28 @@ module SmsLogparser
|
|
12
13
|
end
|
13
14
|
|
14
15
|
def send(data)
|
16
|
+
requests = build_urls(data)
|
17
|
+
return requests if @options[:simulate]
|
18
|
+
|
19
|
+
threads = requests.map do |request|
|
20
|
+
Thread.new do
|
21
|
+
begin
|
22
|
+
response = @connection.post(request[:uri])
|
23
|
+
request[:status] = response.status
|
24
|
+
rescue => e
|
25
|
+
raise RuntimeError, "Can't send request to #{request[:uri]}. #{e.message}", caller
|
26
|
+
end
|
27
|
+
unless @accepted_responses.include?(response.status)
|
28
|
+
msg = "Received HTTP status #{response.status} from API. Only accepting #{@accepted_responses.join(', ')}."
|
29
|
+
raise RuntimeError, msg, caller
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
threads.each {|thread| thread.join }
|
34
|
+
requests
|
35
|
+
end
|
36
|
+
|
37
|
+
def build_urls(data)
|
15
38
|
requests = []
|
16
39
|
path = @base_path + [data[:customer_id], data[:author_id], data[:project_id]].join('/')
|
17
40
|
unless data[:file] =~ /.*\.m3u8$/
|
@@ -26,20 +49,6 @@ module SmsLogparser
|
|
26
49
|
uri: [path, data[:visitor_type], 1].join('/')
|
27
50
|
}
|
28
51
|
end
|
29
|
-
unless @options[:simulate]
|
30
|
-
requests.each_with_index do |request, i|
|
31
|
-
begin
|
32
|
-
response = @connection.post(request[:uri])
|
33
|
-
requests[i][:status] = response.status
|
34
|
-
rescue => e
|
35
|
-
raise RuntimeError, "Can't send request to #{request[:uri]}. #{e.message}", caller
|
36
|
-
end
|
37
|
-
unless @accepted_responses.include?(response.status)
|
38
|
-
msg = "Received HTTP status #{response.status} from API. Only accepting #{@accepted_responses.join(', ')}."
|
39
|
-
raise RuntimeError, msg, caller
|
40
|
-
end
|
41
|
-
end
|
42
|
-
end
|
43
52
|
requests
|
44
53
|
end
|
45
54
|
|
@@ -48,15 +57,11 @@ module SmsLogparser
|
|
48
57
|
def connection
|
49
58
|
connection = Faraday.new(url: @url, request: {timeout: 5}) do |faraday|
|
50
59
|
faraday.request :url_encoded
|
51
|
-
|
52
|
-
faraday.use Faraday::Response::Logger, SmsLogparser::AppLogger.instance
|
53
|
-
end
|
60
|
+
faraday.use Faraday::Response::Logger, SmsLogparser::Loggster.instance
|
54
61
|
faraday.adapter :net_http_persistent
|
55
62
|
end
|
56
63
|
connection.headers[:user_agent] = "sms-logparser v#{SmsLogparser::VERSION}"
|
57
|
-
if @options[:api_key]
|
58
|
-
connection.headers['X-simplex-api-key'] = @options[:api_key]
|
59
|
-
end
|
64
|
+
connection.headers['X-simplex-api-key'] = @options[:api_key] if @options[:api_key]
|
60
65
|
connection
|
61
66
|
end
|
62
67
|
|
data/lib/sms-logparser/cli.rb
CHANGED
@@ -9,8 +9,9 @@ module SmsLogparser
|
|
9
9
|
aliases: %w(-c),
|
10
10
|
desc: "Configuration file for default options"
|
11
11
|
|
12
|
-
class_option :
|
13
|
-
|
12
|
+
class_option :severity, type: :string, aliases: %w(-S),
|
13
|
+
desc: "Log severity <debug|info|warn|error|fatal> (Default: warn)"
|
14
|
+
class_option :logfile, desc: "Path to the logfile (Default: STDOUT)", aliases: %w(-l)
|
14
15
|
class_option :mysql_host, aliases: %w(-h), desc: "MySQL host"
|
15
16
|
class_option :mysql_user, aliases: %w(-u), desc: "MySQL user (Default: root)"
|
16
17
|
class_option :mysql_password, aliases: %w(-p), desc: "MySQL password"
|
@@ -29,7 +30,7 @@ module SmsLogparser
|
|
29
30
|
option :simulate, type: :boolean, aliases: %w(-s),
|
30
31
|
desc: "Dry run without submitting any data"
|
31
32
|
option :verbose, type: :boolean, aliases: %w(-v), desc: "Verbose output"
|
32
|
-
option :limit, type: :numeric, aliases: %w(-
|
33
|
+
option :limit, type: :numeric, aliases: %w(-L), desc: "Limit the number of entries to query"
|
33
34
|
option :accepted_api_responses, type: :array, aliases: %w(-r),
|
34
35
|
desc: "API HTTP responses which are accepted (Default: only accept 200)."
|
35
36
|
def parse
|
@@ -56,18 +57,18 @@ module SmsLogparser
|
|
56
57
|
if data
|
57
58
|
requests = api.send(data)
|
58
59
|
state[:match_count] += 1
|
59
|
-
verbose_parser_output(entry['ID'], data, requests)
|
60
|
+
verbose_parser_output(entry['ID'], data, requests)
|
60
61
|
state[:last_event_id] = entry['ID']
|
61
62
|
end
|
62
63
|
end
|
63
64
|
end
|
64
65
|
end
|
65
66
|
rescue SystemExit, Interrupt
|
66
|
-
logger.error("Received
|
67
|
+
logger.error("Received an interrupt. Stopping the parser run.")
|
67
68
|
state[:status] = STATUS[:interrupted] if state
|
68
69
|
rescue => e
|
69
|
-
logger.error
|
70
|
-
|
70
|
+
logger.error "Aborting the parser run."
|
71
|
+
logger.error e
|
71
72
|
state[:status] = STATUS[:api_error] if state
|
72
73
|
else
|
73
74
|
state[:status] = STATUS[:ok]
|
@@ -79,10 +80,10 @@ module SmsLogparser
|
|
79
80
|
mysql.write_parse_result(state) unless options[:simulate]
|
80
81
|
end
|
81
82
|
log_parse_results(state)
|
83
|
+
SmsLogparser::Loggster.instance.close
|
82
84
|
end
|
83
85
|
rescue => e
|
84
|
-
logger.fatal
|
85
|
-
say(e.backtrace.join("\n"), :yellow) if options[:debug]
|
86
|
+
logger.fatal e
|
86
87
|
end
|
87
88
|
end
|
88
89
|
|
@@ -128,23 +129,26 @@ module SmsLogparser
|
|
128
129
|
when 2
|
129
130
|
logger.info("Recreated database tables.")
|
130
131
|
end
|
132
|
+
SmsLogparser::Loggster.instance.close
|
131
133
|
rescue => e
|
132
|
-
logger.fatal
|
133
|
-
say(e.backtrace.join("\n"), :yellow) if options[:debug]
|
134
|
+
logger.fatal e
|
134
135
|
end
|
135
136
|
|
136
137
|
no_commands do
|
137
138
|
def logger
|
138
|
-
SmsLogparser::
|
139
|
+
SmsLogparser::Loggster.instance.set_severity options[:severity]
|
140
|
+
SmsLogparser::Loggster.instance.set_log_device options[:logfile]
|
139
141
|
end
|
140
142
|
|
141
143
|
def verbose_parser_output(entry_id, data, requests)
|
142
|
-
logger.
|
144
|
+
logger.debug {
|
143
145
|
"parsing data for #{entry_id} (#{data.map{|k,v| "#{k}=\"#{v || '-'}\""}.join(" ") || ''})"
|
144
146
|
}
|
145
|
-
|
146
|
-
|
147
|
-
|
147
|
+
requests.each_with_index do |req, i|
|
148
|
+
logger.debug {
|
149
|
+
"URL #{i + 1} for entry #{entry_id} #{req[:url]}#{req[:uri]}"
|
150
|
+
}
|
151
|
+
end
|
148
152
|
end
|
149
153
|
|
150
154
|
def table_to_csv(table)
|
@@ -2,7 +2,7 @@ module SmsLogparser
|
|
2
2
|
require 'logger'
|
3
3
|
require 'singleton'
|
4
4
|
|
5
|
-
class
|
5
|
+
class Loggster < Logger
|
6
6
|
include Singleton
|
7
7
|
|
8
8
|
def initialize
|
@@ -20,5 +20,23 @@ module SmsLogparser
|
|
20
20
|
self
|
21
21
|
end
|
22
22
|
|
23
|
+
def set_severity(severity = "info")
|
24
|
+
self.sev_threshold = case severity
|
25
|
+
when "debug"
|
26
|
+
Logger::DEBUG
|
27
|
+
when "info"
|
28
|
+
Logger::INFO
|
29
|
+
when "warn"
|
30
|
+
Logger::WARN
|
31
|
+
when "error"
|
32
|
+
Logger::ERROR
|
33
|
+
when "fatal"
|
34
|
+
Logger::FATAL
|
35
|
+
else
|
36
|
+
Logger::INFO
|
37
|
+
end
|
38
|
+
self
|
39
|
+
end
|
40
|
+
|
23
41
|
end
|
24
42
|
end
|
data/lib/sms-logparser/mysql.rb
CHANGED
@@ -6,7 +6,7 @@ module SmsLogparser
|
|
6
6
|
@host_filter = options[:host_filter] || 'pcache%'
|
7
7
|
@query_limit = options[:query_limit] || 1000
|
8
8
|
@client = client
|
9
|
-
@logger = SmsLogparser::
|
9
|
+
@logger = SmsLogparser::Loggster.instance
|
10
10
|
end
|
11
11
|
|
12
12
|
def client
|
@@ -116,7 +116,7 @@ module SmsLogparser
|
|
116
116
|
ID > #{last_id}
|
117
117
|
ORDER BY ID ASC
|
118
118
|
LIMIT #{max_entries};}.gsub(/\s+/, " ").strip
|
119
|
-
@logger.
|
119
|
+
@logger.debug { "Querying for events... (#{query})" }
|
120
120
|
client.query(query)
|
121
121
|
end
|
122
122
|
|
data/spec/cli_spec.rb
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
|
+
LOGFILE = '/tmp/sms-logparser.log'
|
4
|
+
|
3
5
|
describe SmsLogparser::Cli do
|
4
6
|
before do
|
5
7
|
TestHelper.create_test_db
|
@@ -11,27 +13,26 @@ describe SmsLogparser::Cli do
|
|
11
13
|
|
12
14
|
after do
|
13
15
|
TestHelper.drop_test_db
|
16
|
+
FileUtils.rm(LOGFILE) if File.exists?(LOGFILE)
|
14
17
|
end
|
15
18
|
|
16
19
|
it "can create the parser_runs database table" do
|
17
20
|
parser = TestHelper.sms_logparser
|
18
21
|
parser.options[:force] = true
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
out.must_match /.*Created database table./
|
22
|
+
parser.options[:logfile] = LOGFILE
|
23
|
+
parser.setup
|
24
|
+
IO.read(LOGFILE).must_include 'Created database table.'
|
23
25
|
end
|
24
26
|
|
25
27
|
it "can parse a log database and find matches" do
|
26
28
|
TestHelper.seed_db(10)
|
27
29
|
parser = TestHelper.sms_logparser
|
28
30
|
parser.options[:simulate] = true
|
31
|
+
parser.options[:logfile] = LOGFILE
|
29
32
|
#parser.options[:api_base_url] = "http://localhost/creator/rest/"
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
end
|
34
|
-
out.must_match /.*match_count=10.*/
|
33
|
+
parser.setup
|
34
|
+
parser.parse
|
35
|
+
IO.read(LOGFILE).must_include 'match_count="10"'
|
35
36
|
end
|
36
37
|
|
37
38
|
# it "skips over already parsed logs" do
|
@@ -51,10 +52,11 @@ describe SmsLogparser::Cli do
|
|
51
52
|
parser = TestHelper.sms_logparser
|
52
53
|
parser.options[:simulate] = true
|
53
54
|
parser.options[:force] = true
|
55
|
+
parser.options[:logfile] = LOGFILE
|
56
|
+
parser.setup
|
57
|
+
parser.parse
|
54
58
|
|
55
59
|
out, err = capture_io do
|
56
|
-
parser.setup
|
57
|
-
parser.parse
|
58
60
|
parser.history
|
59
61
|
end
|
60
62
|
err.must_equal ""
|
metadata
CHANGED
@@ -1,111 +1,111 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sms-logparser
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.10.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- niwo
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-04-
|
11
|
+
date: 2014-04-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - ~>
|
17
|
+
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '1.3'
|
20
20
|
type: :development
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- - ~>
|
24
|
+
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '1.3'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: rake
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- -
|
31
|
+
- - ">="
|
32
32
|
- !ruby/object:Gem::Version
|
33
33
|
version: '0'
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- -
|
38
|
+
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: webmock
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- -
|
45
|
+
- - ">="
|
46
46
|
- !ruby/object:Gem::Version
|
47
47
|
version: '0'
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- -
|
52
|
+
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '0'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: thor
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- - ~>
|
59
|
+
- - "~>"
|
60
60
|
- !ruby/object:Gem::Version
|
61
61
|
version: 0.19.1
|
62
62
|
type: :runtime
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
|
-
- - ~>
|
66
|
+
- - "~>"
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: 0.19.1
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: mysql2
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
|
-
- -
|
73
|
+
- - ">="
|
74
74
|
- !ruby/object:Gem::Version
|
75
75
|
version: '0'
|
76
76
|
type: :runtime
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
|
-
- -
|
80
|
+
- - ">="
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: '0'
|
83
83
|
- !ruby/object:Gem::Dependency
|
84
84
|
name: faraday
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
86
86
|
requirements:
|
87
|
-
- - ~>
|
87
|
+
- - "~>"
|
88
88
|
- !ruby/object:Gem::Version
|
89
89
|
version: 0.9.0
|
90
90
|
type: :runtime
|
91
91
|
prerelease: false
|
92
92
|
version_requirements: !ruby/object:Gem::Requirement
|
93
93
|
requirements:
|
94
|
-
- - ~>
|
94
|
+
- - "~>"
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: 0.9.0
|
97
97
|
- !ruby/object:Gem::Dependency
|
98
98
|
name: net-http-persistent
|
99
99
|
requirement: !ruby/object:Gem::Requirement
|
100
100
|
requirements:
|
101
|
-
- -
|
101
|
+
- - ">="
|
102
102
|
- !ruby/object:Gem::Version
|
103
103
|
version: 2.9.4
|
104
104
|
type: :runtime
|
105
105
|
prerelease: false
|
106
106
|
version_requirements: !ruby/object:Gem::Requirement
|
107
107
|
requirements:
|
108
|
-
- -
|
108
|
+
- - ">="
|
109
109
|
- !ruby/object:Gem::Version
|
110
110
|
version: 2.9.4
|
111
111
|
description: Reads access logs stored in a MySQL database (coming from the SWISS TXT
|
@@ -117,7 +117,7 @@ executables:
|
|
117
117
|
extensions: []
|
118
118
|
extra_rdoc_files: []
|
119
119
|
files:
|
120
|
-
- .gitignore
|
120
|
+
- ".gitignore"
|
121
121
|
- Gemfile
|
122
122
|
- LICENSE.txt
|
123
123
|
- README.md
|
@@ -125,8 +125,8 @@ files:
|
|
125
125
|
- bin/sms-logparser
|
126
126
|
- lib/sms-logparser.rb
|
127
127
|
- lib/sms-logparser/api.rb
|
128
|
-
- lib/sms-logparser/app_logger.rb
|
129
128
|
- lib/sms-logparser/cli.rb
|
129
|
+
- lib/sms-logparser/loggster.rb
|
130
130
|
- lib/sms-logparser/mysql.rb
|
131
131
|
- lib/sms-logparser/parser.rb
|
132
132
|
- lib/sms-logparser/version.rb
|
@@ -146,17 +146,17 @@ require_paths:
|
|
146
146
|
- lib
|
147
147
|
required_ruby_version: !ruby/object:Gem::Requirement
|
148
148
|
requirements:
|
149
|
-
- -
|
149
|
+
- - ">="
|
150
150
|
- !ruby/object:Gem::Version
|
151
151
|
version: 1.9.3
|
152
152
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
153
153
|
requirements:
|
154
|
-
- -
|
154
|
+
- - ">="
|
155
155
|
- !ruby/object:Gem::Version
|
156
156
|
version: '0'
|
157
157
|
requirements: []
|
158
158
|
rubyforge_project:
|
159
|
-
rubygems_version: 2.0
|
159
|
+
rubygems_version: 2.2.0
|
160
160
|
signing_key:
|
161
161
|
specification_version: 4
|
162
162
|
summary: sms-logparser - Logparser for Simplex Media Server (SMS)
|