racknga 0.9.1 → 0.9.2

data/lib/racknga/middleware/log.rb

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2010  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2010-2011  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -20,9 +20,20 @@ require 'racknga/log_database'
 
 module Racknga
   module Middleware
+    # This is a middleware that puts access logs to groonga
+    # database. It may useful for OLAP (OnLine Analytical
+    # Processing).
+    #
+    # Usage:
+    #   use Racnkga::Middleware::Log, :database_path => "var/log/db"
+    #   run YourApplication
+    #
+    # @see Racknga::LogDatabase
     class Log
-      LOGGER_KEY = "racknga.logger"
+      LOGGER = "racknga.logger"
 
+      # @option options [String] :database_path the
+      # database path to be stored caches.
       def initialize(application, options={})
         @application = application
         @options = Utils.normalize_options(options || {})
@@ -32,8 +43,9 @@ module Racknga
         @logger = Logger.new(@database)
       end
 
+      # For Rack.
       def call(environment)
-        environment[LOGGER_KEY] = @logger
+        environment[LOGGER] = @logger
 
         start_time = Time.now
         status, headers, body = @application.call(environment)
@@ -45,10 +57,12 @@ module Racknga
         [status, headers, body]
       end
 
+      # ensures creating cache database.
       def ensure_database
         @database.ensure_database
       end
 
+      # close the cache database.
       def close_database
         @database.close_database
       end
@@ -77,6 +91,7 @@ module Racknga
                     :runtime => request_time)
       end
 
+      # @private
       class Logger
         def initialize(database)
           @database = database

data/lib/racknga/middleware/range.rb

@@ -20,11 +20,20 @@ require 'time'
 
 module Racknga
   module Middleware
+    # This is a middleware that provides HTTP range request
+    # (partial request) support. For example, HTTP range
+    # request is used for playing a video on the way.
+    #
+    # Usage:
+    #   require "racknga"
+    #   use Racknga::Middleware::Range
+    #   run YourApplication
     class Range
       def initialize(application)
         @application = application
       end
 
+      # For Rack.
       def call(environment)
         status, headers, body = @application.call(environment)
         return [status, headers, body] if status != 200
@@ -109,6 +118,7 @@ module Racknga
         nil
       end
 
+      # @private
       class RangeStream
         def initialize(body, first_byte, length)
           @body = body

data/lib/racknga/nginx_access_log_parser.rb

@@ -0,0 +1,139 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2011  Ryo Onodera <onodera@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "racknga/reverse_line_reader"
+
+module Racknga
+  # Supported formats:
+  #  * combined (default format)
+  #  * combined_with_time (custom format for Passenger)
+  #
+  # Configurations in nginx:
+  #  * combined
+  #    log_format combined '$remote_addr - $remote_user [$time_local]  '
+  #                        '"$request" $status $body_bytes_sent '
+  #                        '"$http_referer" "$http_user_agent"';
+  #    access_log log/access.log combined
+  #
+  #  * combined_with_time
+  #    log_format combined_with_time '$remote_addr - $remote_user '
+  #                                  '[$time_local, $upstream_http_x_runtime, $request_time]  '
+  #                                  '"$request" $status $body_bytes_sent '
+  #                                  '"$http_referer" "$http_user_agent"';
+  #    access_log log/access.log combined_with_time
+  class NginxAccessLogParser
+    include Enumerable
+
+    class FormatError < StandardError
+    end
+
+    def initialize(line_reader)
+      @line_reader = line_reader
+    end
+
+    def each
+      @line_reader.each do |line|
+        line.force_encoding("UTF-8")
+        yield(parse_line(line)) if line.valid_encoding?
+      end
+    end
+
+    REMOTE_ADDRESS = '(?:\d{1,3}\.){3}\d{1,3}'
+    REMOTE_USER = '[^ ]+'
+    TIME_LOCAL = '[^ ]+ \+\d{4}'
+    RUNTIME = '(?:[\d.]+|-)'
+    REQUEST_TIME = '[\d.]+'
+    REQUEST = '.*?'
+    STATUS = '\d{3}'
+    BODY_BYTES_SENT = '\d+'
+    HTTP_REFERER = '.*?'
+    HTTP_USER_AGENT = '(?:\\"|[^\"])*?' # '
+    LOG_FORMAT =
+      /\A(#{REMOTE_ADDRESS}) - (#{REMOTE_USER}) \[(#{TIME_LOCAL})(?:, (#{RUNTIME}), (#{REQUEST_TIME}))?\]  "(#{REQUEST})" (#{STATUS}) (#{BODY_BYTES_SENT}) "(#{HTTP_REFERER})" "(#{HTTP_USER_AGENT})"\n\z/
+    def parse_line(line)
+      if line =~ LOG_FORMAT
+        last_match = Regexp.last_match
+        options = {}
+        options[:remote_address] = last_match[1]
+        options[:remote_user] = last_match[2]
+        parse_time_local(last_match[3], options)
+        options[:runtime] = last_match[4].to_f
+        options[:request_time] = last_match[5].to_f
+        options[:request] = last_match[6]
+        options[:status] = last_match[7].to_i
+        options[:body_bytes_sent] = last_match[8].to_i
+        options[:http_referer] = last_match[9]
+        options[:http_user_agent] = last_match[10]
+        LogEntry.new(options)
+      else
+        raise FormatError.new("ill-formatted log entry: #{line.inspect} !~ #{LOG_FORMAT}")
+      end
+    end
+
+    def parse_time_local(token, options)
+      day, month, year, hour, minute, second, _time_zone = token.split(/[\/: ]/)
+      options[:time_local] = Time.local(year, month, day, hour, minute, second)
+    end
+  end
+
+  class ReversedNginxAccessLogParser < NginxAccessLogParser
+    def initialize(line_reader)
+      @line_reader = ReverseLineReader.new(line_reader)
+    end
+  end
+
+  class LogEntry
+    ATTRIBUTES = [
+      :remote_address,
+      :remote_user,
+      :time_local,
+      :runtime,
+      :request_time,
+      :request,
+      :status,
+      :body_bytes_sent,
+      :http_referer,
+      :http_user_agent,
+    ]
+
+    attr_reader(*ATTRIBUTES)
+    def initialize(options=nil)
+      options ||= {}
+      @remote_address = options[:remote_address]
+      @remote_user = options[:remote_user]
+      @time_local = options[:time_local] || Time.at(0)
+      @runtime = options[:runtime] || 0.0
+      @request_time = options[:request_time] || 0.0
+      @request = options[:request]
+      @status = options[:status]
+      @body_bytes_sent = options[:body_bytes_sent]
+      @http_referer = options[:http_referer]
+      @http_user_agent = options[:http_user_agent]
+    end
+
+    def attributes
+      ATTRIBUTES.collect do |attribute|
+        __send__(attribute)
+      end
+    end
+
+    def ==(other)
+      attributes == other.attributes
+    end
+  end
+end

data/lib/racknga/reverse_line_reader.rb

@@ -0,0 +1,73 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2011  Ryo Onodera <onodera@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Racknga
+  class ReverseLineReader
+    def initialize(io)
+      @io = io
+      @io.seek(0, IO::SEEK_END)
+      @buffer = ""
+      @data = ""
+    end
+
+    def each
+      separator = $/
+      separator_length = separator.length
+      while read_to_buffer
+        loop do
+          index = @buffer.rindex(separator, @buffer.length - 1 - separator_length)
+          break if index.nil? or index.zero?
+          last_line = @buffer.slice!((index + separator_length)..-1)
+          yield(last_line)
+        end
+      end
+      yield(@buffer) unless @buffer.empty?
+    end
+
+    private
+    BYTES_PER_READ = 4096
+    def read
+      position = @io.pos
+      if position < BYTES_PER_READ
+        bytes_per_read = position
+      else
+        bytes_per_read = BYTES_PER_READ
+      end
+
+      if bytes_per_read.zero?
+        @data.replace("")
+      else
+        @io.seek(-bytes_per_read, IO::SEEK_CUR)
+        @io.read(bytes_per_read, @data)
+        @io.seek(-bytes_per_read, IO::SEEK_CUR)
+      end
+
+      @data
+    end
+
+    def read_to_buffer
+      data = read.force_encoding("BINARY")
+      if data.empty?
+        false
+      else
+        @buffer.insert(0, data)
+        true
+      end
+    end
+  end
+end

data/lib/racknga/version.rb

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2010  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2010-2011  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -17,5 +17,5 @@
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
 module Racknga
-  VERSION = '0.9.1'
+  VERSION = '0.9.2'
 end

data/lib/racknga/will_paginate.rb

@@ -19,6 +19,7 @@
 require "groonga"
 
 module Groonga
+  # @deprecated since 0.9.2. Use Kaminari instead of will_paginate.
   module WillPaginateAPI
     def total_pages
       n_pages

data/lib/racknga.rb

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2010  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2010-2011  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -20,7 +20,9 @@ require 'rack'
 
 require 'racknga/version'
 require 'racknga/utils'
+require "racknga/nginx_access_log_parser"
 require 'racknga/middleware/deflater'
 require 'racknga/middleware/exception_notifier'
 require 'racknga/middleware/jsonp'
 require 'racknga/middleware/range'
+require "racknga/middleware/instance_name"