sms-logparser 0.9.6 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 79fdbf9f1e00638b7f470363cb1a436c49dbf5a6
4
- data.tar.gz: 59d6c29aa01ffbbb31c4b8924f4c8ccc6ac3faac
3
+ metadata.gz: fb123379f0d4fe32a268365460e4e916d02b962f
4
+ data.tar.gz: c2502c598e29d0d284ba10c7e2a3395ee3a2a717
5
5
  SHA512:
6
- metadata.gz: 1ff73068b045081b7a9ed360092c78a018ace25629eb4cf10f7fa76d531e64deea495da3ba308ec049face623b3972bc83d2614fcc26fecef3088218092b82a4
7
- data.tar.gz: 2ea9ba98dc42185dfe262d7ff9e4d4c577d6b49af911550b2edd2cb57402ef7a4ebababe829024085a2bd7aa17b21b017f32556c61bac469c0fee1e9d517862b
6
+ metadata.gz: f24537178e814531fd41c5cfe060e65f070df80eff68e2ac48b1fd1ebee1d0f269da03d46317aa120db64643488e0419f20c4933457d2f0c39212cf01491a50d
7
+ data.tar.gz: a91d0bdb92e8543ecd352798cddb5c1c961f8434c10b312538423bda419069935f8e7e256ab295405b4d29bc37724b19c86b220deea6bc83246f517ea2e51400
data/lib/sms-logparser.rb CHANGED
@@ -7,5 +7,5 @@ require "sms-logparser/version"
7
7
  require "sms-logparser/mysql"
8
8
  require "sms-logparser/parser"
9
9
  require "sms-logparser/api"
10
- require "sms-logparser/app_logger"
10
+ require "sms-logparser/loggster"
11
11
  require "sms-logparser/cli"
@@ -1,6 +1,7 @@
1
1
  module SmsLogparser
2
2
  class Api
3
3
  require 'uri'
4
+ require 'thread'
4
5
 
5
6
  def initialize(options)
6
7
  @options = options
@@ -12,6 +13,28 @@ module SmsLogparser
12
13
  end
13
14
 
14
15
  def send(data)
16
+ requests = build_urls(data)
17
+ return requests if @options[:simulate]
18
+
19
+ threads = requests.map do |request|
20
+ Thread.new do
21
+ begin
22
+ response = @connection.post(request[:uri])
23
+ request[:status] = response.status
24
+ rescue => e
25
+ raise RuntimeError, "Can't send request to #{request[:uri]}. #{e.message}", caller
26
+ end
27
+ unless @accepted_responses.include?(response.status)
28
+ msg = "Received HTTP status #{response.status} from API. Only accepting #{@accepted_responses.join(', ')}."
29
+ raise RuntimeError, msg, caller
30
+ end
31
+ end
32
+ end
33
+ threads.each {|thread| thread.join }
34
+ requests
35
+ end
36
+
37
+ def build_urls(data)
15
38
  requests = []
16
39
  path = @base_path + [data[:customer_id], data[:author_id], data[:project_id]].join('/')
17
40
  unless data[:file] =~ /.*\.m3u8$/
@@ -26,20 +49,6 @@ module SmsLogparser
26
49
  uri: [path, data[:visitor_type], 1].join('/')
27
50
  }
28
51
  end
29
- unless @options[:simulate]
30
- requests.each_with_index do |request, i|
31
- begin
32
- response = @connection.post(request[:uri])
33
- requests[i][:status] = response.status
34
- rescue => e
35
- raise RuntimeError, "Can't send request to #{request[:uri]}. #{e.message}", caller
36
- end
37
- unless @accepted_responses.include?(response.status)
38
- msg = "Received HTTP status #{response.status} from API. Only accepting #{@accepted_responses.join(', ')}."
39
- raise RuntimeError, msg, caller
40
- end
41
- end
42
- end
43
52
  requests
44
53
  end
45
54
 
@@ -48,15 +57,11 @@ module SmsLogparser
48
57
  def connection
49
58
  connection = Faraday.new(url: @url, request: {timeout: 5}) do |faraday|
50
59
  faraday.request :url_encoded
51
- if @options[:debug]
52
- faraday.use Faraday::Response::Logger, SmsLogparser::AppLogger.instance
53
- end
60
+ faraday.use Faraday::Response::Logger, SmsLogparser::Loggster.instance
54
61
  faraday.adapter :net_http_persistent
55
62
  end
56
63
  connection.headers[:user_agent] = "sms-logparser v#{SmsLogparser::VERSION}"
57
- if @options[:api_key]
58
- connection.headers['X-simplex-api-key'] = @options[:api_key]
59
- end
64
+ connection.headers['X-simplex-api-key'] = @options[:api_key] if @options[:api_key]
60
65
  connection
61
66
  end
62
67
 
@@ -9,8 +9,9 @@ module SmsLogparser
9
9
  aliases: %w(-c),
10
10
  desc: "Configuration file for default options"
11
11
 
12
- class_option :debug, type: :boolean, desc: "Show debug output"
13
- class_option :logfile, desc: "Path to the logfile (Default: STDOUT)"
12
+ class_option :severity, type: :string, aliases: %w(-S),
13
+ desc: "Log severity <debug|info|warn|error|fatal> (Default: warn)"
14
+ class_option :logfile, desc: "Path to the logfile (Default: STDOUT)", aliases: %w(-l)
14
15
  class_option :mysql_host, aliases: %w(-h), desc: "MySQL host"
15
16
  class_option :mysql_user, aliases: %w(-u), desc: "MySQL user (Default: root)"
16
17
  class_option :mysql_password, aliases: %w(-p), desc: "MySQL password"
@@ -29,7 +30,7 @@ module SmsLogparser
29
30
  option :simulate, type: :boolean, aliases: %w(-s),
30
31
  desc: "Dry run without submitting any data"
31
32
  option :verbose, type: :boolean, aliases: %w(-v), desc: "Verbose output"
32
- option :limit, type: :numeric, aliases: %w(-l), desc: "Limit the number of entries to query"
33
+ option :limit, type: :numeric, aliases: %w(-L), desc: "Limit the number of entries to query"
33
34
  option :accepted_api_responses, type: :array, aliases: %w(-r),
34
35
  desc: "API HTTP responses which are accepted (Default: only accept 200)."
35
36
  def parse
@@ -56,18 +57,18 @@ module SmsLogparser
56
57
  if data
57
58
  requests = api.send(data)
58
59
  state[:match_count] += 1
59
- verbose_parser_output(entry['ID'], data, requests) if options[:verbose]
60
+ verbose_parser_output(entry['ID'], data, requests)
60
61
  state[:last_event_id] = entry['ID']
61
62
  end
62
63
  end
63
64
  end
64
65
  end
65
66
  rescue SystemExit, Interrupt
66
- logger.error("Received ctrl-c. Stopping the parser run.")
67
+ logger.error("Received an interrupt. Stopping the parser run.")
67
68
  state[:status] = STATUS[:interrupted] if state
68
69
  rescue => e
69
- logger.error("#{e.message}. Aborting the parser run.")
70
- say(e.backtrace.join("\n"), :yellow) if options[:debug]
70
+ logger.error "Aborting the parser run."
71
+ logger.error e
71
72
  state[:status] = STATUS[:api_error] if state
72
73
  else
73
74
  state[:status] = STATUS[:ok]
@@ -79,10 +80,10 @@ module SmsLogparser
79
80
  mysql.write_parse_result(state) unless options[:simulate]
80
81
  end
81
82
  log_parse_results(state)
83
+ SmsLogparser::Loggster.instance.close
82
84
  end
83
85
  rescue => e
84
- logger.fatal(e.message)
85
- say(e.backtrace.join("\n"), :yellow) if options[:debug]
86
+ logger.fatal e
86
87
  end
87
88
  end
88
89
 
@@ -128,23 +129,26 @@ module SmsLogparser
128
129
  when 2
129
130
  logger.info("Recreated database tables.")
130
131
  end
132
+ SmsLogparser::Loggster.instance.close
131
133
  rescue => e
132
- logger.fatal(e.message)
133
- say(e.backtrace.join("\n"), :yellow) if options[:debug]
134
+ logger.fatal e
134
135
  end
135
136
 
136
137
  no_commands do
137
138
  def logger
138
- SmsLogparser::AppLogger.instance.set_log_device(options[:logfile])
139
+ SmsLogparser::Loggster.instance.set_severity options[:severity]
140
+ SmsLogparser::Loggster.instance.set_log_device options[:logfile]
139
141
  end
140
142
 
141
143
  def verbose_parser_output(entry_id, data, requests)
142
- logger.info {
144
+ logger.debug {
143
145
  "parsing data for #{entry_id} (#{data.map{|k,v| "#{k}=\"#{v || '-'}\""}.join(" ") || ''})"
144
146
  }
145
- logger.info {
146
- "parser urls for entry #{entry_id} (#{requests.map {|req| "#{req[:url]}#{req[:uri]}"}.join(" ")})"
147
- }
147
+ requests.each_with_index do |req, i|
148
+ logger.debug {
149
+ "URL #{i + 1} for entry #{entry_id} #{req[:url]}#{req[:uri]}"
150
+ }
151
+ end
148
152
  end
149
153
 
150
154
  def table_to_csv(table)
@@ -2,7 +2,7 @@ module SmsLogparser
2
2
  require 'logger'
3
3
  require 'singleton'
4
4
 
5
- class AppLogger < Logger
5
+ class Loggster < Logger
6
6
  include Singleton
7
7
 
8
8
  def initialize
@@ -20,5 +20,23 @@ module SmsLogparser
20
20
  self
21
21
  end
22
22
 
23
+ def set_severity(severity = "info")
24
+ self.sev_threshold = case severity
25
+ when "debug"
26
+ Logger::DEBUG
27
+ when "info"
28
+ Logger::INFO
29
+ when "warn"
30
+ Logger::WARN
31
+ when "error"
32
+ Logger::ERROR
33
+ when "fatal"
34
+ Logger::FATAL
35
+ else
36
+ Logger::INFO
37
+ end
38
+ self
39
+ end
40
+
23
41
  end
24
42
  end
@@ -6,7 +6,7 @@ module SmsLogparser
6
6
  @host_filter = options[:host_filter] || 'pcache%'
7
7
  @query_limit = options[:query_limit] || 1000
8
8
  @client = client
9
- @logger = SmsLogparser::AppLogger.instance
9
+ @logger = SmsLogparser::Loggster.instance
10
10
  end
11
11
 
12
12
  def client
@@ -116,7 +116,7 @@ module SmsLogparser
116
116
  ID > #{last_id}
117
117
  ORDER BY ID ASC
118
118
  LIMIT #{max_entries};}.gsub(/\s+/, " ").strip
119
- @logger.info("Querying for events... (#{query})") if @options[:debug]
119
+ @logger.debug { "Querying for events... (#{query})" }
120
120
  client.query(query)
121
121
  end
122
122
 
@@ -1,3 +1,3 @@
1
1
  module SmsLogparser
2
- VERSION = "0.9.6"
2
+ VERSION = "0.10.0"
3
3
  end
data/spec/cli_spec.rb CHANGED
@@ -1,5 +1,7 @@
1
1
  require 'spec_helper'
2
2
 
3
+ LOGFILE = '/tmp/sms-logparser.log'
4
+
3
5
  describe SmsLogparser::Cli do
4
6
  before do
5
7
  TestHelper.create_test_db
@@ -11,27 +13,26 @@ describe SmsLogparser::Cli do
11
13
 
12
14
  after do
13
15
  TestHelper.drop_test_db
16
+ FileUtils.rm(LOGFILE) if File.exists?(LOGFILE)
14
17
  end
15
18
 
16
19
  it "can create the parser_runs database table" do
17
20
  parser = TestHelper.sms_logparser
18
21
  parser.options[:force] = true
19
- out, err = capture_io do
20
- parser.setup
21
- end
22
- out.must_match /.*Created database table./
22
+ parser.options[:logfile] = LOGFILE
23
+ parser.setup
24
+ IO.read(LOGFILE).must_include 'Created database table.'
23
25
  end
24
26
 
25
27
  it "can parse a log database and find matches" do
26
28
  TestHelper.seed_db(10)
27
29
  parser = TestHelper.sms_logparser
28
30
  parser.options[:simulate] = true
31
+ parser.options[:logfile] = LOGFILE
29
32
  #parser.options[:api_base_url] = "http://localhost/creator/rest/"
30
- out, err = capture_io do
31
- parser.setup
32
- parser.parse
33
- end
34
- out.must_match /.*match_count=10.*/
33
+ parser.setup
34
+ parser.parse
35
+ IO.read(LOGFILE).must_include 'match_count="10"'
35
36
  end
36
37
 
37
38
  # it "skips over already parsed logs" do
@@ -51,10 +52,11 @@ describe SmsLogparser::Cli do
51
52
  parser = TestHelper.sms_logparser
52
53
  parser.options[:simulate] = true
53
54
  parser.options[:force] = true
55
+ parser.options[:logfile] = LOGFILE
56
+ parser.setup
57
+ parser.parse
54
58
 
55
59
  out, err = capture_io do
56
- parser.setup
57
- parser.parse
58
60
  parser.history
59
61
  end
60
62
  err.must_equal ""
metadata CHANGED
@@ -1,111 +1,111 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: sms-logparser
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.9.6
4
+ version: 0.10.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - niwo
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-04-11 00:00:00.000000000 Z
11
+ date: 2014-04-13 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - ~>
17
+ - - "~>"
18
18
  - !ruby/object:Gem::Version
19
19
  version: '1.3'
20
20
  type: :development
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - ~>
24
+ - - "~>"
25
25
  - !ruby/object:Gem::Version
26
26
  version: '1.3'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: rake
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
- - - '>='
31
+ - - ">="
32
32
  - !ruby/object:Gem::Version
33
33
  version: '0'
34
34
  type: :development
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - '>='
38
+ - - ">="
39
39
  - !ruby/object:Gem::Version
40
40
  version: '0'
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: webmock
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
- - - '>='
45
+ - - ">="
46
46
  - !ruby/object:Gem::Version
47
47
  version: '0'
48
48
  type: :development
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
- - - '>='
52
+ - - ">="
53
53
  - !ruby/object:Gem::Version
54
54
  version: '0'
55
55
  - !ruby/object:Gem::Dependency
56
56
  name: thor
57
57
  requirement: !ruby/object:Gem::Requirement
58
58
  requirements:
59
- - - ~>
59
+ - - "~>"
60
60
  - !ruby/object:Gem::Version
61
61
  version: 0.19.1
62
62
  type: :runtime
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
- - - ~>
66
+ - - "~>"
67
67
  - !ruby/object:Gem::Version
68
68
  version: 0.19.1
69
69
  - !ruby/object:Gem::Dependency
70
70
  name: mysql2
71
71
  requirement: !ruby/object:Gem::Requirement
72
72
  requirements:
73
- - - '>='
73
+ - - ">="
74
74
  - !ruby/object:Gem::Version
75
75
  version: '0'
76
76
  type: :runtime
77
77
  prerelease: false
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
80
- - - '>='
80
+ - - ">="
81
81
  - !ruby/object:Gem::Version
82
82
  version: '0'
83
83
  - !ruby/object:Gem::Dependency
84
84
  name: faraday
85
85
  requirement: !ruby/object:Gem::Requirement
86
86
  requirements:
87
- - - ~>
87
+ - - "~>"
88
88
  - !ruby/object:Gem::Version
89
89
  version: 0.9.0
90
90
  type: :runtime
91
91
  prerelease: false
92
92
  version_requirements: !ruby/object:Gem::Requirement
93
93
  requirements:
94
- - - ~>
94
+ - - "~>"
95
95
  - !ruby/object:Gem::Version
96
96
  version: 0.9.0
97
97
  - !ruby/object:Gem::Dependency
98
98
  name: net-http-persistent
99
99
  requirement: !ruby/object:Gem::Requirement
100
100
  requirements:
101
- - - '>='
101
+ - - ">="
102
102
  - !ruby/object:Gem::Version
103
103
  version: 2.9.4
104
104
  type: :runtime
105
105
  prerelease: false
106
106
  version_requirements: !ruby/object:Gem::Requirement
107
107
  requirements:
108
- - - '>='
108
+ - - ">="
109
109
  - !ruby/object:Gem::Version
110
110
  version: 2.9.4
111
111
  description: Reads access logs stored in a MySQL database (coming from the SWISS TXT
@@ -117,7 +117,7 @@ executables:
117
117
  extensions: []
118
118
  extra_rdoc_files: []
119
119
  files:
120
- - .gitignore
120
+ - ".gitignore"
121
121
  - Gemfile
122
122
  - LICENSE.txt
123
123
  - README.md
@@ -125,8 +125,8 @@ files:
125
125
  - bin/sms-logparser
126
126
  - lib/sms-logparser.rb
127
127
  - lib/sms-logparser/api.rb
128
- - lib/sms-logparser/app_logger.rb
129
128
  - lib/sms-logparser/cli.rb
129
+ - lib/sms-logparser/loggster.rb
130
130
  - lib/sms-logparser/mysql.rb
131
131
  - lib/sms-logparser/parser.rb
132
132
  - lib/sms-logparser/version.rb
@@ -146,17 +146,17 @@ require_paths:
146
146
  - lib
147
147
  required_ruby_version: !ruby/object:Gem::Requirement
148
148
  requirements:
149
- - - '>='
149
+ - - ">="
150
150
  - !ruby/object:Gem::Version
151
151
  version: 1.9.3
152
152
  required_rubygems_version: !ruby/object:Gem::Requirement
153
153
  requirements:
154
- - - '>='
154
+ - - ">="
155
155
  - !ruby/object:Gem::Version
156
156
  version: '0'
157
157
  requirements: []
158
158
  rubyforge_project:
159
- rubygems_version: 2.0.3
159
+ rubygems_version: 2.2.0
160
160
  signing_key:
161
161
  specification_version: 4
162
162
  summary: sms-logparser - Logparser for Simplex Media Server (SMS)