log_reader 0.0.1

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2013 YOURNAME
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,3 @@
+# LogReader
+
+This project rocks and uses MIT-LICENSE.

data/Rakefile

@@ -0,0 +1,21 @@
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+# require 'rdoc/task'
+# 
+# RDoc::Task.new(:rdoc) do |rdoc|
+#   rdoc.rdoc_dir = 'rdoc'
+#   rdoc.title    = 'LogReader'
+#   rdoc.options << '--line-numbers'
+#   rdoc.rdoc_files.include('README.rdoc')
+#   rdoc.rdoc_files.include('lib/**/*.rb')
+# end
+
+
+
+
+Bundler::GemHelper.install_tasks
+

data/lib/log_reader.rb

@@ -0,0 +1,5 @@
+# -*- encoding : utf-8 -*-
+require 'log_reader/nginx_basic_logger'
+
+module LogReader
+end

data/lib/log_reader/nginx_basic_logger.rb

@@ -0,0 +1,120 @@
+# -*- enid : utf-8 -*-
+module LogReader
+  # Nginx基本日志
+  class NginxBasicLogger
+
+    # 60.217.232.195 p11.hostname.net - [09/Sep/2013:23:07:40 +0800] "GET /deal/deal_image/0808/0163/ HTTP/1.1" 404 0 "http://www.hostname.com/hot/jipiao-quanguo" "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0"
+
+    # 127.0.0.1 store.tuan800.com - [09/Oct/2013:19:02:47 +0800] "GET /shop_1151398 HTTP/1.1" 404 22059 "-" "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:14.0) Gecko/20100101 Firefox/14.0.1" "-" "-"
+
+    #
+    # '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"'
+    # NGINX_BASIC_LOG_PATTERN = Regexp.compile(/\A(\S+) - (\S+) \[([^\]]+)\] "([^"]+)" (\S+) (\S+) "([^"]*?)" "([^"]*?)"( "([^"]*?)")?\Z/).freeze
+    #
+
+    # '$remote_addr $http_host - [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"'
+    # NGINX_BASIC_LOG_PATTERN = Regexp.compile('\A(\S+) (\S+) - \[([^\]]+)\] "([^"]+)" (\S+) (\S+) "([^"]*?)" "([^"]*?)"').freeze
+
+    # named regexp
+    NGINX_BASIC_LOG_PATTERN = Regexp.compile('\A(?<remote_addr>\S+) (?<remote_host>\S+) - \[(?<time_local>[^\]]+)\] "(?<request>[^"]+)" (?<status>\S+) (?<body_bytes_sent>\S+) "(?<http_referer>[^"]*?)" "(?<http_user_agent>[^"]*?)"').freeze
+
+    @@log_pattern = NGINX_BASIC_LOG_PATTERN
+
+    def initialize attrs={}
+    end
+
+    def text_to_sql_data text
+      _perform_to_sql_data do
+        "#{text}".split(/\n/)
+      end
+    end
+
+    # return:
+    #   {
+    #     :field_names => "(a, b, c, d, e, f)",
+    #     :records => [
+    #       ["()", "()", "()"],
+    #       ["()", "()", "()"],
+    #       ["()", "()", "()"],
+    #       ...
+    #       ["()", "()", "()"]
+    #     ]
+    #   }
+    def file_to_sql_data src
+      _perform_to_sql_data do
+        File.open(src, 'r')
+      end
+    end
+
+    private
+
+    def _perform_to_sql_data &block
+      result = {}
+
+      result[:field_names] = "(#{
+        [
+          'remote_addr', 'remote_host', 'time_local', 'request_method', 'request_path', 'request_protocol',
+          'status', 'body_bytes_sent', 'http_referer', 'http_user_agent', 'id'
+        ].join(',')
+      })"
+
+      records = []
+
+      error_lines = []
+
+      counter = 0
+      unit_data = []
+
+      block.call.each do |line|
+        if matched = @@log_pattern.match("#{line}")
+
+          counter += 1
+
+          if counter > 50000
+            counter = 0
+            records << unit_data
+            unit_data = []
+          end
+
+          remote_addr = matched['remote_addr']
+          remote_host = matched['remote_host']
+          time_local  = matched['time_local']
+          request     = matched['request']
+          status      = matched['status']
+          body_bytes_sent = matched['body_bytes_sent']
+          http_referer    = matched['http_referer']
+          http_user_agent = matched['http_user_agent']
+
+          # time_local = DateTime.strptime('09/Sep/2013:23:07:43 +0800', '%d/%b/%Y:%H:%M:%S %Z').to_s(:db)
+          time_local = DateTime.strptime(time_local, '%d/%b/%Y:%H:%M:%S %Z').strftime("%Y-%m-%d %H:%M:%S")
+
+          request_infos = request.to_s.split(/\s+/)
+
+          request_method   = request_infos[0]
+          request_path     = request_infos[1]
+          request_protocol = request_infos[2]
+
+          id = Digest::MD5.hexdigest(line)
+
+          unit_data << "(#{
+            [
+              "\"#{remote_addr}\"", "\"#{remote_host}\"", "\"#{time_local}\"", "\"#{request_method}\"", "\"#{request_path}\"", "\"#{request_protocol}\"",
+              "#{status}", "#{body_bytes_sent}", "\"#{http_referer}\"", "\"#{http_user_agent}\"", "\"#{id}\""
+            ].join(',')
+          })"
+        else
+          error_lines << line
+        end
+      end
+
+      # 不足一组的也要压入数组
+      records << unit_data
+
+      # puts error_lines[0..10]
+
+      result[:records] = records
+
+      result
+    end
+  end
+end

data/lib/log_reader/version.rb

@@ -0,0 +1,4 @@
+# -*- encoding : utf-8 -*-
+module LogReader
+  VERSION = "0.0.1"
+end

data/lib/tasks/log_reader_tasks.rake

@@ -0,0 +1,4 @@
+# desc "Explaining what the task does"
+# task :log_reader do
+#   # Task goes here
+# end

metadata

@@ -0,0 +1,58 @@
+--- !ruby/object:Gem::Specification
+name: log_reader
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- happy
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-10-10 00:00:00.000000000 Z
+dependencies: []
+description: Description of LogReader.
+email:
+- andywang7259@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/log_reader/version.rb
+- lib/log_reader/nginx_basic_logger.rb
+- lib/log_reader.rb
+- lib/tasks/log_reader_tasks.rake
+- MIT-LICENSE
+- Rakefile
+- README.md
+homepage: http://github.com/xiuxian123
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 3364914995248220073
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 3364914995248220073
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: Summary of LogReader.
+test_files: []