sms-logparser 0.11.1 → 0.12.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/sms-logparser.rb +1 -0
- data/lib/sms-logparser/api.rb +4 -1
- data/lib/sms-logparser/cli.rb +21 -0
- data/lib/sms-logparser/data_cache.rb +30 -0
- data/lib/sms-logparser/mysql.rb +9 -13
- data/lib/sms-logparser/version.rb +1 -1
- metadata +2 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: be3b90122d60da5a8342511ee4a4aa3879c549a7
|
4
|
+
data.tar.gz: 9928653741ef481378f0761ba5f5d5f25c83bc74
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: bc1cb0028bd7ac7392a7e504aebd6f471b8f636f0983872f7e2f7e23b9dba18eb11359f34714efaf2d49b33aefa849583821a8b003af0bfe82db52961a23830f
|
7
|
+
data.tar.gz: 139425fa7c82772ab61af0c2aefe88acec215177f559f770da307b8df00c4e848e4c834704b73d5c8ea80b47fbcc8fc1280b6d19f0498f519f80b9715151ea98
|
data/lib/sms-logparser.rb
CHANGED
data/lib/sms-logparser/api.rb
CHANGED
@@ -10,8 +10,11 @@ module SmsLogparser
|
|
10
10
|
@base_path = (@base_url.path << "/").squeeze('/')
|
11
11
|
@accepted_responses = parse_status_codes(options[:accepted_api_responses]) || [200]
|
12
12
|
@connection = connection
|
13
|
+
@data_cache = {}
|
13
14
|
end
|
14
15
|
|
16
|
+
|
17
|
+
|
15
18
|
def send(data)
|
16
19
|
requests = build_urls(data)
|
17
20
|
return requests if @options[:simulate]
|
@@ -55,7 +58,7 @@ module SmsLogparser
|
|
55
58
|
private
|
56
59
|
|
57
60
|
def connection
|
58
|
-
connection = Faraday.new(url: @url, request: {timeout:
|
61
|
+
connection = Faraday.new(url: @url, request: {timeout: 20}) do |faraday|
|
59
62
|
faraday.request :url_encoded
|
60
63
|
faraday.adapter :net_http_persistent
|
61
64
|
if @options[:severity] == "debug"
|
data/lib/sms-logparser/cli.rb
CHANGED
@@ -87,6 +87,27 @@ module SmsLogparser
|
|
87
87
|
end
|
88
88
|
end
|
89
89
|
|
90
|
+
desc "cached_pase", "Check the database for pcache logs and put them into the cache"
|
91
|
+
option :limit, type: :numeric, aliases: %w(-L), desc: "Limit the number of entries to query"
|
92
|
+
def cached_parse
|
93
|
+
cache = DataCache.new
|
94
|
+
mysql = Mysql.new(options)
|
95
|
+
say "Getting entries from database..."
|
96
|
+
mysql.get_entries(last_id: mysql.get_last_parse_id, limit: options[:limit]) do |entries|
|
97
|
+
entries.each do |entry|
|
98
|
+
Parser.extract_data_from_msg(entry['Message']) do |data|
|
99
|
+
if data
|
100
|
+
cache.add(data)
|
101
|
+
say "Cached data ", :magenta
|
102
|
+
say data
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
puts
|
108
|
+
puts cache.cache
|
109
|
+
end
|
110
|
+
|
90
111
|
desc "history", "List the last paser runs"
|
91
112
|
option :results, type: :numeric, default: 10, aliases: %w(-n),
|
92
113
|
desc: "Number of results to display"
|
@@ -0,0 +1,30 @@
|
|
1
|
+
module SmsLogparser
|
2
|
+
class DataCache
|
3
|
+
|
4
|
+
attr_reader :cache
|
5
|
+
|
6
|
+
def initialize
|
7
|
+
@cache = Hash.new
|
8
|
+
@wanted_keys = [:customer_id, :author_id, :project_id]
|
9
|
+
end
|
10
|
+
|
11
|
+
def add(data)
|
12
|
+
key = [data[:customer_id], data[:author_id], data[:project_id]].join('.')
|
13
|
+
@cache[key] = initialize_value(data) unless @cache.has_key?(key)
|
14
|
+
unless data[:file] =~ /.*\.m3u8$/
|
15
|
+
@cache[key][data[:traffic_type]] = @cache[key][data[:traffic_type]].to_i + data[:bytes].to_i
|
16
|
+
end
|
17
|
+
if data[:visitor_type]
|
18
|
+
@cache[key][data[:visitor_type]] = @cache[key][data[:visitor_type]].to_i + 1
|
19
|
+
end
|
20
|
+
@cache
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
|
25
|
+
def initialize_value(data)
|
26
|
+
data.select { |key,_| @wanted_keys.include? key }
|
27
|
+
end
|
28
|
+
|
29
|
+
end # class
|
30
|
+
end # module
|
data/lib/sms-logparser/mysql.rb
CHANGED
@@ -79,9 +79,9 @@ module SmsLogparser
|
|
79
79
|
|
80
80
|
def get_entries(options={})
|
81
81
|
last_id = options[:last_id] || get_last_parse_id
|
82
|
-
max_id
|
82
|
+
max_id = get_max_id(last_id, options[:limit])
|
83
83
|
while last_id < max_id
|
84
|
-
entries = select_entries(last_id,
|
84
|
+
entries = select_entries(last_id, max_id)
|
85
85
|
yield entries
|
86
86
|
entries = entries.to_a
|
87
87
|
last_id = entries.size > 0 ? entries[-1]['ID'] : max_id
|
@@ -97,24 +97,20 @@ module SmsLogparser
|
|
97
97
|
|
98
98
|
private
|
99
99
|
|
100
|
-
def
|
101
|
-
|
100
|
+
def get_max_id(last_id, user_limit = nil)
|
101
|
+
max_id = get_last_event_id
|
102
|
+
if user_limit && user_limit < (max_id - last_id)
|
102
103
|
max_id = last_id + user_limit
|
103
|
-
if @query_limit > user_limit
|
104
|
-
query_limit = user_limit
|
105
|
-
end
|
106
|
-
else
|
107
|
-
max_id = get_last_event_id
|
108
104
|
end
|
109
|
-
|
105
|
+
max_id
|
110
106
|
end
|
111
107
|
|
112
|
-
def select_entries(last_id,
|
108
|
+
def select_entries(last_id, max_id)
|
113
109
|
query = %Q{
|
114
110
|
SELECT ID, Message FROM SystemEvents
|
115
|
-
WHERE ID
|
111
|
+
WHERE ID BETWEEN #{last_id} AND #{max_id}
|
116
112
|
ORDER BY ID ASC
|
117
|
-
LIMIT #{
|
113
|
+
LIMIT #{@query_limit};
|
118
114
|
}.gsub(/\s+/, " ").strip
|
119
115
|
@logger.debug { "Querying for events... (#{query})" }
|
120
116
|
client.query(query)
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sms-logparser
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.12.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- niwo
|
@@ -126,6 +126,7 @@ files:
|
|
126
126
|
- lib/sms-logparser.rb
|
127
127
|
- lib/sms-logparser/api.rb
|
128
128
|
- lib/sms-logparser/cli.rb
|
129
|
+
- lib/sms-logparser/data_cache.rb
|
129
130
|
- lib/sms-logparser/loggster.rb
|
130
131
|
- lib/sms-logparser/mysql.rb
|
131
132
|
- lib/sms-logparser/parser.rb
|