log_reader 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,20 @@
1
+ Copyright 2013 YOURNAME
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining
4
+ a copy of this software and associated documentation files (the
5
+ "Software"), to deal in the Software without restriction, including
6
+ without limitation the rights to use, copy, modify, merge, publish,
7
+ distribute, sublicense, and/or sell copies of the Software, and to
8
+ permit persons to whom the Software is furnished to do so, subject to
9
+ the following conditions:
10
+
11
+ The above copyright notice and this permission notice shall be
12
+ included in all copies or substantial portions of the Software.
13
+
14
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,3 @@
1
+ # LogReader
2
+
3
+ This project rocks and uses MIT-LICENSE.
@@ -0,0 +1,21 @@
1
+ begin
2
+ require 'bundler/setup'
3
+ rescue LoadError
4
+ puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
5
+ end
6
+
7
+ # require 'rdoc/task'
8
+ #
9
+ # RDoc::Task.new(:rdoc) do |rdoc|
10
+ # rdoc.rdoc_dir = 'rdoc'
11
+ # rdoc.title = 'LogReader'
12
+ # rdoc.options << '--line-numbers'
13
+ # rdoc.rdoc_files.include('README.rdoc')
14
+ # rdoc.rdoc_files.include('lib/**/*.rb')
15
+ # end
16
+
17
+
18
+
19
+
20
+ Bundler::GemHelper.install_tasks
21
+
@@ -0,0 +1,5 @@
1
+ # -*- encoding : utf-8 -*-
2
+ require 'log_reader/nginx_basic_logger'
3
+
4
+ module LogReader
5
+ end
@@ -0,0 +1,120 @@
1
+ # -*- enid : utf-8 -*-
2
+ module LogReader
3
+ # Nginx基本日志
4
+ class NginxBasicLogger
5
+
6
+ # 60.217.232.195 p11.hostname.net - [09/Sep/2013:23:07:40 +0800] "GET /deal/deal_image/0808/0163/ HTTP/1.1" 404 0 "http://www.hostname.com/hot/jipiao-quanguo" "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0"
7
+
8
+ # 127.0.0.1 store.tuan800.com - [09/Oct/2013:19:02:47 +0800] "GET /shop_1151398 HTTP/1.1" 404 22059 "-" "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:14.0) Gecko/20100101 Firefox/14.0.1" "-" "-"
9
+
10
+ #
11
+ # '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"'
12
+ # NGINX_BASIC_LOG_PATTERN = Regexp.compile(/\A(\S+) - (\S+) \[([^\]]+)\] "([^"]+)" (\S+) (\S+) "([^"]*?)" "([^"]*?)"( "([^"]*?)")?\Z/).freeze
13
+ #
14
+
15
+ # '$remote_addr $http_host - [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"'
16
+ # NGINX_BASIC_LOG_PATTERN = Regexp.compile('\A(\S+) (\S+) - \[([^\]]+)\] "([^"]+)" (\S+) (\S+) "([^"]*?)" "([^"]*?)"').freeze
17
+
18
+ # named regexp
19
+ NGINX_BASIC_LOG_PATTERN = Regexp.compile('\A(?<remote_addr>\S+) (?<remote_host>\S+) - \[(?<time_local>[^\]]+)\] "(?<request>[^"]+)" (?<status>\S+) (?<body_bytes_sent>\S+) "(?<http_referer>[^"]*?)" "(?<http_user_agent>[^"]*?)"').freeze
20
+
21
+ @@log_pattern = NGINX_BASIC_LOG_PATTERN
22
+
23
+ def initialize attrs={}
24
+ end
25
+
26
+ def text_to_sql_data text
27
+ _perform_to_sql_data do
28
+ "#{text}".split(/\n/)
29
+ end
30
+ end
31
+
32
+ # return:
33
+ # {
34
+ # :field_names => "(a, b, c, d, e, f)",
35
+ # :records => [
36
+ # ["()", "()", "()"],
37
+ # ["()", "()", "()"],
38
+ # ["()", "()", "()"],
39
+ # ...
40
+ # ["()", "()", "()"]
41
+ # ]
42
+ # }
43
+ def file_to_sql_data src
44
+ _perform_to_sql_data do
45
+ File.open(src, 'r')
46
+ end
47
+ end
48
+
49
+ private
50
+
51
+ def _perform_to_sql_data &block
52
+ result = {}
53
+
54
+ result[:field_names] = "(#{
55
+ [
56
+ 'remote_addr', 'remote_host', 'time_local', 'request_method', 'request_path', 'request_protocol',
57
+ 'status', 'body_bytes_sent', 'http_referer', 'http_user_agent', 'id'
58
+ ].join(',')
59
+ })"
60
+
61
+ records = []
62
+
63
+ error_lines = []
64
+
65
+ counter = 0
66
+ unit_data = []
67
+
68
+ block.call.each do |line|
69
+ if matched = @@log_pattern.match("#{line}")
70
+
71
+ counter += 1
72
+
73
+ if counter > 50000
74
+ counter = 0
75
+ records << unit_data
76
+ unit_data = []
77
+ end
78
+
79
+ remote_addr = matched['remote_addr']
80
+ remote_host = matched['remote_host']
81
+ time_local = matched['time_local']
82
+ request = matched['request']
83
+ status = matched['status']
84
+ body_bytes_sent = matched['body_bytes_sent']
85
+ http_referer = matched['http_referer']
86
+ http_user_agent = matched['http_user_agent']
87
+
88
+ # time_local = DateTime.strptime('09/Sep/2013:23:07:43 +0800', '%d/%b/%Y:%H:%M:%S %Z').to_s(:db)
89
+ time_local = DateTime.strptime(time_local, '%d/%b/%Y:%H:%M:%S %Z').strftime("%Y-%m-%d %H:%M:%S")
90
+
91
+ request_infos = request.to_s.split(/\s+/)
92
+
93
+ request_method = request_infos[0]
94
+ request_path = request_infos[1]
95
+ request_protocol = request_infos[2]
96
+
97
+ id = Digest::MD5.hexdigest(line)
98
+
99
+ unit_data << "(#{
100
+ [
101
+ "\"#{remote_addr}\"", "\"#{remote_host}\"", "\"#{time_local}\"", "\"#{request_method}\"", "\"#{request_path}\"", "\"#{request_protocol}\"",
102
+ "#{status}", "#{body_bytes_sent}", "\"#{http_referer}\"", "\"#{http_user_agent}\"", "\"#{id}\""
103
+ ].join(',')
104
+ })"
105
+ else
106
+ error_lines << line
107
+ end
108
+ end
109
+
110
+ # 不足一组的也要压入数组
111
+ records << unit_data
112
+
113
+ # puts error_lines[0..10]
114
+
115
+ result[:records] = records
116
+
117
+ result
118
+ end
119
+ end
120
+ end
@@ -0,0 +1,4 @@
1
+ # -*- encoding : utf-8 -*-
2
+ module LogReader
3
+ VERSION = "0.0.1"
4
+ end
@@ -0,0 +1,4 @@
1
+ # desc "Explaining what the task does"
2
+ # task :log_reader do
3
+ # # Task goes here
4
+ # end
metadata ADDED
@@ -0,0 +1,58 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: log_reader
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - happy
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2013-10-10 00:00:00.000000000 Z
13
+ dependencies: []
14
+ description: Description of LogReader.
15
+ email:
16
+ - andywang7259@gmail.com
17
+ executables: []
18
+ extensions: []
19
+ extra_rdoc_files: []
20
+ files:
21
+ - lib/log_reader/version.rb
22
+ - lib/log_reader/nginx_basic_logger.rb
23
+ - lib/log_reader.rb
24
+ - lib/tasks/log_reader_tasks.rake
25
+ - MIT-LICENSE
26
+ - Rakefile
27
+ - README.md
28
+ homepage: http://github.com/xiuxian123
29
+ licenses: []
30
+ post_install_message:
31
+ rdoc_options: []
32
+ require_paths:
33
+ - lib
34
+ required_ruby_version: !ruby/object:Gem::Requirement
35
+ none: false
36
+ requirements:
37
+ - - ">="
38
+ - !ruby/object:Gem::Version
39
+ version: '0'
40
+ segments:
41
+ - 0
42
+ hash: 3364914995248220073
43
+ required_rubygems_version: !ruby/object:Gem::Requirement
44
+ none: false
45
+ requirements:
46
+ - - ">="
47
+ - !ruby/object:Gem::Version
48
+ version: '0'
49
+ segments:
50
+ - 0
51
+ hash: 3364914995248220073
52
+ requirements: []
53
+ rubyforge_project:
54
+ rubygems_version: 1.8.25
55
+ signing_key:
56
+ specification_version: 3
57
+ summary: Summary of LogReader.
58
+ test_files: []