sms-logparser 0.0.4 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 51e05056b98cf58379c37b19df329c948e2b8656
4
- data.tar.gz: bc696cef467afb68d9ee9b18e281ea8aa6b497b5
3
+ metadata.gz: 2e9efb4712d1d9d54c9904f57993ced13742bd51
4
+ data.tar.gz: 2fa877596b733aac7a13a27cd76ac22902978564
5
5
  SHA512:
6
- metadata.gz: 86d201529f5ac57d50082fb992d815e1969ae2a8dac652c11a017b27b0f8410a107b694829d55b9d5a3118d6ee5dac82797ee9d64132d87bc7067006f51aa0ae
7
- data.tar.gz: 808350b006292eae603d3a12a2274ff080640c65a65052e9b22f9797ad97ce222b548e6d0f6558f12dc707485ac9f148e6ee5be627b75290ede18ef67b6ee3b9
6
+ metadata.gz: ecb5588a86ae77f592f67117172b4b1d259fec82d7bd6e80305c70de60032db646c8751734eed245d5d90867a1e875bf08e366465a1db23dade2400df1744b61
7
+ data.tar.gz: 1202cb4ff8e9d3820682e5cb6bbc0b032c0a5168dfc32c963915b5055f5bfe7a344e64870eb10a1d972e3d0f413e236b5e243f1fcd7b06638b94ea7ab6e5f098
@@ -0,0 +1,26 @@
1
+ module SmsLogparser
2
+ class Api
3
+
4
+ def initialize(options)
5
+ @options = options
6
+ end
7
+
8
+ def send(data)
9
+ url = "#{@options[:api_base_path]}/"
10
+ url += "#{data[:customer_id]}/"
11
+ url += "#{data[:author_id]}/"
12
+ url += "#{data[:project_id]}/"
13
+ url += "#{data[:traffic_type]}/"
14
+ url += "#{data[:bytes]}"
15
+ unless @options[:simulate]
16
+ begin
17
+ RestClient.get(url)
18
+ rescue
19
+ raise "Can't send log to #{url}"
20
+ end
21
+ end
22
+ url
23
+ end
24
+
25
+ end # class
26
+ end # module
@@ -1,14 +1,6 @@
1
- module SmsLogparser
1
+ module SmsLogparser
2
2
  class Cli < Thor
3
3
 
4
- EXTENSION_MAP = {
5
- 'default' => 'TRAFFIC_PODCAST',
6
- 'mp4' => 'TRAFFIC_PODCAST',
7
- 'f4v' => 'TRAFFIC_PODCAST',
8
- 'flv' => 'TRAFFIC_PODCAST',
9
- 'ts' => 'TRAFFIC_MOBILE'
10
- }
11
-
12
4
  class_option :mysql_host, :default => 'localhost'
13
5
  class_option :mysql_user, :default => 'root'
14
6
  class_option :mysql_db, :default => 'Syslog'
@@ -25,130 +17,73 @@ module SmsLogparser
25
17
  def parse
26
18
  start_time = Time.now
27
19
  count = 0
28
- last_id = get_last_parse_id
29
- begin
30
- results = client.query(
31
- "SELECT * FROM SystemEvents\
32
- WHERE `FromHost` like 'pcache%'\
33
- AND ID > #{last_id} ORDER BY ID ASC"
34
- )
35
- rescue Mysql2::Error
36
- say "parser_table not found please create it with 'create_parser_table'", :red
37
- exit 1
38
- end
39
- results.each do |result|
40
- if result['Message'] =~ /\/content\/.*\.(f4v|flv|mp4|ts) .*/
41
- data = extract_data_from_msg(result['Message'])
42
- url = "#{options[:api_base_path]}/"
43
- url += "#{data[:customer_id]}/"
44
- url += "#{data[:author_id]}/"
45
- url += "#{data[:project_id]}/"
46
- url += "#{data[:traffic_type]}/"
47
- url += "#{data[:bytes]}"
48
- if options[:simulate]
49
- puts "Message ID: #{result['ID']}"
50
- puts "URL: #{url}"
51
- puts "Data: #{data}"
52
- puts "-----------------------"
53
- else
54
- begin
55
- RestClient.get(url)
56
- rescue
57
- say "Error: Can't send log to #{url}", :red
58
- say "Aborting.", :red
59
- break
60
- end
20
+ begin
21
+ mysql = Mysql.new(options)
22
+ entries = mysql.get_entries
23
+ api = Api.new(options)
24
+ last_id = mysql.get_last_parse_id
25
+ entries.each do |entry|
26
+ if Parser.match(entry)
27
+ data = Parser.extract_data_from_msg(entry['Message'])
28
+ url = api.send(data)
29
+ last_id = entry['ID']
30
+ count += 1
31
+ debug_parser_output(data, url, entry) if options[:simulate]
61
32
  end
62
- count += 1
63
33
  end
64
- last_id = result['ID']
34
+ mysql.write_parse_result(last_id, count) unless options[:simulate]
35
+ puts "Started: #{start_time.strftime('%d.%d.%Y %T')}"
36
+ puts "Runtime: #{(Time.now - start_time).round(2)}s"
37
+ puts "Matches: #{count}"
38
+ rescue => e
39
+ say "Error: #{e.message}", :red
65
40
  end
66
- write_parse_result(last_id, count) unless options[:simulate]
67
- puts "Started: #{start_time.strftime('%d.%d.%Y %T')}"
68
- puts "Runtime: #{(Time.now - start_time).round(2)}s"
69
- puts "Matches: #{count}"
70
41
  end
71
42
 
72
43
  desc "last_runs", "List the last paser runs"
73
44
  def last_runs
74
- begin
75
- runs = client.query(
76
- "SELECT * FROM SmsParserRuns ORDER BY ID ASC LIMIT 10"
77
- )
78
- rescue Mysql2::Error
79
- say "parser_table not found please create it with 'create_parser_table'", :red
80
- exit 1
81
- end
82
- if runs.size > 0
83
- table = [%w(RunAt #Events LastEventID)]
84
- runs.each do |run|
85
- table << [
86
- run['RunAt'],
87
- run['EventsFound'],
88
- run['LastEventID']
89
- ]
45
+ begin
46
+ runs = Mysql.new(options).last_runs
47
+ if runs.size > 0
48
+ table = [%w(RunAt #Events LastEventID)]
49
+ runs.each do |run|
50
+ table << [
51
+ run['RunAt'],
52
+ run['EventsFound'],
53
+ run['LastEventID']
54
+ ]
55
+ end
56
+ print_table table
57
+ else
58
+ say "No parser runs found in the database."
90
59
  end
91
- print_table table
92
- else
93
- say "No parser runs found in the database."
60
+ rescue => e
61
+ say "Error: #{e.message}", :red
94
62
  end
95
63
  end
96
64
 
97
- desc "create_parser_table", "Create the parser table to track the last logs parsed"
98
- def create_parser_table
99
- client.query(
100
- "CREATE TABLE IF NOT EXISTS\
101
- SmsParserRuns(\
102
- ID INT PRIMARY KEY AUTO_INCREMENT,\
103
- RunAt datetime DEFAULT NULL,\
104
- LastEventID INT DEFAULT NULL,\
105
- EventsFound INT DEFAULT 0,\
106
- INDEX `LastEventID_I1` (`LastEventID`)
107
- )"
108
- )
109
- say "OK", :green
65
+ desc "setup", "Create the parser table to track the last logs parsed"
66
+ def setup
67
+ begin
68
+ Mysql.new(options).create_parser_table
69
+ say "OK", :green
70
+ rescue => e
71
+ say "Error: #{e.message}", :red
72
+ end
110
73
  end
111
74
 
112
75
  no_commands do
113
- def get_last_parse_id
114
- last_parse = client.query(
115
- "SELECT LastEventID FROM SmsParserRuns ORDER BY ID DESC LIMIT 1"
116
- )
117
- last_parse.first ? last_parse.first['LastEventID'] : 0
118
- rescue Mysql2::Error
119
- say "parser_table not found please create it with 'create_parser_table'", :red
120
- exit 1
121
- end
122
-
123
- def write_parse_result(id, count)
124
- client.query("INSERT INTO SmsParserRuns(RunAt, LastEventID, EventsFound)\
125
- VALUES(\
126
- '#{Time.now.strftime("%Y-%m-%d %H:%M:%S")}',\
127
- #{id},\
128
- #{count}
129
- )"
130
- )
131
- end
132
-
133
- def extract_data_from_msg(msg)
134
- m = msg.match /\/content\/(\d+)\/(\d+)\/(\d+)\/.+\.(\S+)\s.*\"\s\d+\s(\d+)/
135
- data = {
136
- :customer_id => m[1],
137
- :author_id => m[2],
138
- :project_id => m[3],
139
- :ext => m[4],
140
- :traffic_type => (EXTENSION_MAP[m[4]] || EXTENSION_MAP['default']),
141
- :bytes => m[5]
142
- }
143
- end
144
-
145
- def client
146
- @client ||= Mysql2::Client.new(
147
- :host => options[:mysql_host],
148
- :username => options[:mysql_user],
149
- :database => options[:mysql_db]
150
- )
76
+ def debug_parser_output(data, url, entry)
77
+ puts
78
+ say "Message ID: ", :green
79
+ say entry['ID']
80
+ say "URL: ", :green
81
+ say url
82
+ say "Data: ", :green
83
+ say data
84
+ puts
151
85
  end
152
86
  end
87
+
153
88
  end
154
89
  end
@@ -0,0 +1,79 @@
1
+ module SmsLogparser
2
+ class Mysql
3
+
4
+ def initialize(options)
5
+ @options = options
6
+ end
7
+
8
+ def client
9
+ @client ||= Mysql2::Client.new(
10
+ :host => @options[:mysql_host],
11
+ :username => @options[:mysql_user],
12
+ :database => @options[:mysql_db]
13
+ )
14
+ end
15
+
16
+ def last_runs
17
+ begin
18
+ runs = client.query(
19
+ "SELECT * FROM SmsParserRuns ORDER BY ID ASC LIMIT 10"
20
+ )
21
+ rescue Mysql2::Error => e
22
+ raise e
23
+ end
24
+ end
25
+
26
+ def create_parser_table
27
+ begin
28
+ return client.query(
29
+ "CREATE TABLE IF NOT EXISTS\
30
+ SmsParserRuns(\
31
+ ID INT PRIMARY KEY AUTO_INCREMENT,\
32
+ RunAt datetime DEFAULT NULL,\
33
+ LastEventID INT DEFAULT NULL,\
34
+ EventsFound INT DEFAULT 0,\
35
+ INDEX `LastEventID_I1` (`LastEventID`)
36
+ )"
37
+ )
38
+ rescue Mysql2::Error => e
39
+ raise e
40
+ end
41
+ end
42
+
43
+ def get_entries(last_id = get_last_parse_id)
44
+ begin
45
+ return client.query(
46
+ "SELECT * FROM SystemEvents\
47
+ WHERE `FromHost` like 'pcache%'\
48
+ AND ID > #{last_id} ORDER BY ID ASC"
49
+ )
50
+ rescue Mysql2::Error => e
51
+ raise e
52
+ end
53
+ end
54
+
55
+ def write_parse_result(id, count)
56
+ client.query("INSERT INTO SmsParserRuns(RunAt, LastEventID, EventsFound)\
57
+ VALUES(\
58
+ '#{Time.now.strftime("%Y-%m-%d %H:%M:%S")}',\
59
+ #{id},\
60
+ #{count}
61
+ )"
62
+ )
63
+ end
64
+
65
+ def get_last_parse_id
66
+ id = 0
67
+ begin
68
+ last_parse = client.query(
69
+ "SELECT LastEventID FROM SmsParserRuns ORDER BY ID DESC LIMIT 1"
70
+ )
71
+ id = last_parse.first ? last_parse.first['LastEventID'] : 0
72
+ rescue Mysql2::Error => e
73
+ raise e
74
+ end
75
+ id
76
+ end
77
+
78
+ end # class
79
+ end # module
@@ -0,0 +1,34 @@
1
+ module SmsLogparser
2
+ class Parser
3
+
4
+ def self.extract_data_from_msg(msg)
5
+ m = msg.match /\/content\/(\d+)\/(\d+)\/(\d+)\/.+\.(\S+)\s.*\"\s\d+\s(\d+).+"(.*)"$/
6
+ raise "No match found." unless m
7
+ data = {
8
+ :customer_id => m[1],
9
+ :author_id => m[2],
10
+ :project_id => m[3],
11
+ :ext => m[4],
12
+ :traffic_type => Parser.get_traffic_type(m[4]),
13
+ :bytes => m[5],
14
+ :user_agent => m[6]
15
+ }
16
+ end
17
+
18
+ def self.match(entry)
19
+ entry['Message'] =~ /\/content\/.*\.(f4v|flv|mp4|mp3|ts) .*/
20
+ end
21
+
22
+ def self.get_traffic_type(user_agent)
23
+ case user_agent
24
+ when /.*(iTunes).*/
25
+ "TRAFFIC_PODCAST"
26
+ when /.*(IEMobile|Mobile Safari|iPhone|iPod|iPad).*/
27
+ "TRAFFIC_MOBILE"
28
+ else
29
+ "TRAFFIC_WEBCAST"
30
+ end
31
+ end
32
+
33
+ end # class
34
+ end # module
@@ -1,3 +1,3 @@
1
1
  module SmsLogparser
2
- VERSION = "0.0.4"
2
+ VERSION = "0.1.0"
3
3
  end
data/lib/sms-logparser.rb CHANGED
@@ -4,4 +4,7 @@ require 'mysql2'
4
4
  require 'rest_client'
5
5
 
6
6
  require "sms-logparser/version"
7
+ require "sms-logparser/mysql"
8
+ require "sms-logparser/parser"
9
+ require "sms-logparser/api"
7
10
  require "sms-logparser/cli"
@@ -1,6 +1,6 @@
1
1
  require 'spec_helper'
2
2
 
3
- describe SmsLogparser do
3
+ describe SmsLogparser::Cli do
4
4
  before do
5
5
  TestHelper.create_test_db
6
6
  TestHelper.create_sylog_db_table
@@ -12,7 +12,7 @@ describe SmsLogparser do
12
12
 
13
13
  it "can create the parser_runs database table" do
14
14
  out, err = capture_io do
15
- TestHelper.sms_logparser.create_parser_table
15
+ TestHelper.sms_logparser.setup
16
16
  end
17
17
  out.must_match /OK/
18
18
  end
@@ -22,7 +22,7 @@ describe SmsLogparser do
22
22
  parser = TestHelper.sms_logparser
23
23
  parser.options[:api_base_path] = 'http://devnull-as-a-service.com/dev/null/'
24
24
  out, err = capture_io do
25
- TestHelper.sms_logparser.create_parser_table
25
+ TestHelper.sms_logparser.setup
26
26
  parser.parse
27
27
  end
28
28
  out.must_match /.*Matches: 10$/
@@ -33,22 +33,22 @@ describe SmsLogparser do
33
33
  parser = TestHelper.sms_logparser
34
34
  parser.options[:api_base_path] = 'http://devnull-as-a-service.com/dev/null/'
35
35
  out, err = capture_io do
36
- TestHelper.sms_logparser.create_parser_table
36
+ TestHelper.sms_logparser.setup
37
37
  parser.parse
38
38
  parser.parse
39
39
  end
40
40
  out.must_match /.*Matches: 0$/
41
41
  end
42
42
 
43
- it "list parser runs" do
43
+ it "lists parser runs" do
44
44
  TestHelper.seed_db(1)
45
45
  parser = TestHelper.sms_logparser
46
46
  parser.options[:api_base_path] = 'http://devnull-as-a-service.com/dev/null/'
47
47
  out, err = capture_io do
48
- TestHelper.sms_logparser.create_parser_table
48
+ TestHelper.sms_logparser.setup
49
49
  parser.parse
50
50
  parser.last_runs
51
51
  end
52
52
  assert_equal(err, "")
53
53
  end
54
- end
54
+ end
data/spec/spec_helper.rb CHANGED
@@ -41,13 +41,13 @@ module TestHelper
41
41
  end
42
42
 
43
43
  def self.insert_logs(host = "pcache", message = "", number_of_inserts = 1)
44
- values = ''
44
+ values = []
45
45
  number_of_inserts.times do
46
- values += "('#{host}', '#{message}'), "
46
+ values << "('#{host}', '#{message}')"
47
47
  end
48
48
  self.client.query(
49
49
  "INSERT INTO #{@@mysql_db}.SystemEvents(FromHost, Message)\
50
- VALUES #{values.chomp(', ')}"
50
+ VALUES #{values.join(', ')}"
51
51
  )
52
52
  end
53
53
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: sms-logparser
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.4
4
+ version: 0.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - niwo
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-03-05 00:00:00.000000000 Z
11
+ date: 2014-03-13 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -95,10 +95,13 @@ files:
95
95
  - Rakefile
96
96
  - bin/sms-logparser
97
97
  - lib/sms-logparser.rb
98
+ - lib/sms-logparser/api.rb
98
99
  - lib/sms-logparser/cli.rb
100
+ - lib/sms-logparser/mysql.rb
101
+ - lib/sms-logparser/parser.rb
99
102
  - lib/sms-logparser/version.rb
100
103
  - sms-logparser.gemspec
101
- - spec/logparser_spec.rb
104
+ - spec/cli_spec.rb
102
105
  - spec/spec_helper.rb
103
106
  homepage: https://github.com/swisstxt/sms-logparser
104
107
  licenses:
@@ -125,5 +128,5 @@ signing_key:
125
128
  specification_version: 4
126
129
  summary: SMS Logparser
127
130
  test_files:
128
- - spec/logparser_spec.rb
131
+ - spec/cli_spec.rb
129
132
  - spec/spec_helper.rb