ruby-nuggets 0.9.1 → 0.9.2

data/lib/nuggets/i18n.rb

@@ -0,0 +1,165 @@
+# encoding: utf-8
+
+#--
+###############################################################################
+#                                                                             #
+# A component of ruby-nuggets, some extensions to the Ruby programming        #
+# language.                                                                   #
+#                                                                             #
+# Copyright (C) 2007-2011 Jens Wille                                          #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@gmail.com>                                       #
+#                                                                             #
+# ruby-nuggets is free software; you can redistribute it and/or modify it     #
+# under the terms of the GNU Affero General Public License as published by    #
+# the Free Software Foundation; either version 3 of the License, or (at your  #
+# option) any later version.                                                  #
+#                                                                             #
+# ruby-nuggets is distributed in the hope that it will be useful, but WITHOUT #
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or       #
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License #
+# for more details.                                                           #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with ruby-nuggets. If not, see <http://www.gnu.org/licenses/>.        #
+#                                                                             #
+###############################################################################
+#++
+
+module Nuggets
+  module I18n
+
+    DIACRITICS = {
+      'À' => 'A',  # LATIN CAPITAL LETTER A WITH GRAVE
+      'Á' => 'A',  # LATIN CAPITAL LETTER A WITH ACUTE
+      'Â' => 'A',  # LATIN CAPITAL LETTER A WITH CIRCUMFLEX
+      'Ã' => 'A',  # LATIN CAPITAL LETTER A WITH TILDE
+      'Ä' => 'AE', # LATIN CAPITAL LETTER A WITH DIAERESIS
+      'Å' => 'A',  # LATIN CAPITAL LETTER A WITH RING ABOVE
+      'Æ' => 'AE', # LATIN CAPITAL LETTER AE
+      'Ç' => 'C',  # LATIN CAPITAL LETTER C WITH CEDILLA
+      'È' => 'E',  # LATIN CAPITAL LETTER E WITH GRAVE
+      'É' => 'E',  # LATIN CAPITAL LETTER E WITH ACUTE
+      'Ê' => 'E',  # LATIN CAPITAL LETTER E WITH CIRCUMFLEX
+      'Ë' => 'E',  # LATIN CAPITAL LETTER E WITH DIAERESIS
+      'Ì' => 'I',  # LATIN CAPITAL LETTER I WITH GRAVE
+      'Í' => 'I',  # LATIN CAPITAL LETTER I WITH ACUTE
+      'Î' => 'I',  # LATIN CAPITAL LETTER I WITH CIRCUMFLEX
+      'Ï' => 'I',  # LATIN CAPITAL LETTER I WITH DIAERESIS
+      'Ð' => 'DH', # LATIN CAPITAL LETTER ETH
+      'Ñ' => 'N',  # LATIN CAPITAL LETTER N WITH TILDE
+      'Ò' => 'O',  # LATIN CAPITAL LETTER O WITH GRAVE
+      'Ó' => 'O',  # LATIN CAPITAL LETTER O WITH ACUTE
+      'Ô' => 'O',  # LATIN CAPITAL LETTER O WITH CIRCUMFLEX
+      'Õ' => 'O',  # LATIN CAPITAL LETTER O WITH TILDE
+      'Ö' => 'OE', # LATIN CAPITAL LETTER O WITH DIAERESIS
+      'Ø' => 'O',  # LATIN CAPITAL LETTER O WITH STROKE
+      'Ù' => 'U',  # LATIN CAPITAL LETTER U WITH GRAVE
+      'Ú' => 'U',  # LATIN CAPITAL LETTER U WITH ACUTE
+      'Û' => 'U',  # LATIN CAPITAL LETTER U WITH CIRCUMFLEX
+      'Ü' => 'UE', # LATIN CAPITAL LETTER U WITH DIAERESIS
+      'Ý' => 'Y',  # LATIN CAPITAL LETTER Y WITH ACUTE
+      'Þ' => 'TH', # LATIN CAPITAL LETTER THORN
+      'ß' => 'ss', # LATIN SMALL LETTER SHARP S
+      'à' => 'a',  # LATIN SMALL LETTER A WITH GRAVE
+      'á' => 'a',  # LATIN SMALL LETTER A WITH ACUTE
+      'â' => 'a',  # LATIN SMALL LETTER A WITH CIRCUMFLEX
+      'ã' => 'a',  # LATIN SMALL LETTER A WITH TILDE
+      'ä' => 'ae', # LATIN SMALL LETTER A WITH DIAERESIS
+      'å' => 'a',  # LATIN SMALL LETTER A WITH RING ABOVE
+      'æ' => 'ae', # LATIN SMALL LETTER AE
+      'ç' => 'c',  # LATIN SMALL LETTER C WITH CEDILLA
+      'è' => 'e',  # LATIN SMALL LETTER E WITH GRAVE
+      'é' => 'e',  # LATIN SMALL LETTER E WITH ACUTE
+      'ê' => 'e',  # LATIN SMALL LETTER E WITH CIRCUMFLEX
+      'ë' => 'e',  # LATIN SMALL LETTER E WITH DIAERESIS
+      'ì' => 'i',  # LATIN SMALL LETTER I WITH GRAVE
+      'í' => 'i',  # LATIN SMALL LETTER I WITH ACUTE
+      'î' => 'i',  # LATIN SMALL LETTER I WITH CIRCUMFLEX
+      'ï' => 'i',  # LATIN SMALL LETTER I WITH DIAERESIS
+      'ð' => 'dh', # LATIN SMALL LETTER ETH
+      'ñ' => 'n',  # LATIN SMALL LETTER N WITH TILDE
+      'ò' => 'o',  # LATIN SMALL LETTER O WITH GRAVE
+      'ó' => 'o',  # LATIN SMALL LETTER O WITH ACUTE
+      'ô' => 'o',  # LATIN SMALL LETTER O WITH CIRCUMFLEX
+      'õ' => 'o',  # LATIN SMALL LETTER O WITH TILDE
+      'ö' => 'oe', # LATIN SMALL LETTER O WITH DIAERESIS
+      'ø' => 'o',  # LATIN SMALL LETTER O WITH STROKE
+      'ù' => 'u',  # LATIN SMALL LETTER U WITH GRAVE
+      'ú' => 'u',  # LATIN SMALL LETTER U WITH ACUTE
+      'û' => 'u',  # LATIN SMALL LETTER U WITH CIRCUMFLEX
+      'ü' => 'ue', # LATIN SMALL LETTER U WITH DIAERESIS
+      'ý' => 'y',  # LATIN SMALL LETTER Y WITH ACUTE
+      'þ' => 'th', # LATIN SMALL LETTER THORN
+      'ÿ' => 'y'   # LATIN SMALL LETTER Y WITH DIAERESIS
+    }
+
+    def self.args_for_map_diacritics
+      @args_for_map_diacritics ||= begin
+        map = ::Hash.new { |h, k| h[k] = [] }
+
+        DIACRITICS.each { |a| a.each { |i| map[i].concat(a) } }
+        map.each { |k, v| v.uniq!; map[k] = "(#{::Regexp.union(*v).source})" }
+
+        [::Regexp.union(*map.keys.sort_by { |k| -k.length }), map.method(:[])]
+      end
+    end
+
+  end
+end
+
+class String
+
+  # call-seq:
+  #   str.replace_diacritics => new_str
+  #
+  # Substitutes any diacritics in _str_ with their replacements as per
+  # Nuggets::I18n::DIACRITICS.
+  def replace_diacritics
+    (_dup = dup).replace_diacritics! || _dup
+  end
+
+  # call-seq:
+  #   str.replace_diacritics! => str or +nil+
+  #
+  # Destructive version of #replace_diacritics.
+  def replace_diacritics!
+    diacritics = ::Nuggets::I18n::DIACRITICS
+
+    gsub!(/#{::Regexp.union(*diacritics.keys)}/) { |m|
+      s = diacritics[m]
+
+      # Try to adjust case:
+      #   'Äh' => 'AEh' => 'Aeh'
+      #
+      # But:
+      #   'SÖS' => 'SOES' (not 'SOeS'!)
+      if s.length > 1
+        t = $'[0..0]
+        s[1..-1] = s[1..-1].downcase if t == t.downcase
+      end
+
+      s
+    }
+  end
+
+  def map_diacritics
+    (_dup = dup).map_diacritics! || _dup
+  end
+
+  def map_diacritics!
+    re, block = ::Nuggets::I18n.args_for_map_diacritics
+    gsub!(re, &block)
+  end
+
+end
+
+if $0 == __FILE__
+  s = 'Äh, Rüby iß sö cüül, nö? SÖS!'
+  p s
+  p s.replace_diacritics
+
+  s.replace_diacritics!
+  p s
+end

data/lib/nuggets/lazy_attr.rb

@@ -0,0 +1,45 @@
+#--
+###############################################################################
+#                                                                             #
+# A component of ruby-nuggets, some extensions to the Ruby programming        #
+# language.                                                                   #
+#                                                                             #
+# Copyright (C) 2007-2012 Jens Wille                                          #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@gmail.com>                                       #
+#                                                                             #
+# ruby-nuggets is free software; you can redistribute it and/or modify it     #
+# under the terms of the GNU Affero General Public License as published by    #
+# the Free Software Foundation; either version 3 of the License, or (at your  #
+# option) any later version.                                                  #
+#                                                                             #
+# ruby-nuggets is distributed in the hope that it will be useful, but WITHOUT #
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or       #
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License #
+# for more details.                                                           #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with ruby-nuggets. If not, see <http://www.gnu.org/licenses/>.        #
+#                                                                             #
+###############################################################################
+#++
+
+module Nuggets
+
+  module LazyAttr
+
+    private
+
+    def lazy_attr(attr, freeze = true)
+      class << self; self; end.class_eval { attr_reader attr }
+
+      value = instance_variable_get(name = "@#{attr}") ||
+              instance_variable_set(name, yield)
+
+      freeze ? value.freeze : value
+    end
+
+  end
+
+end

data/lib/nuggets/log_parser/apache.rb

@@ -0,0 +1,102 @@
+#--
+###############################################################################
+#                                                                             #
+# A component of ruby-nuggets, some extensions to the Ruby programming        #
+# language.                                                                   #
+#                                                                             #
+# Copyright (C) 2007-2012 Jens Wille                                          #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@gmail.com>                                       #
+#                                                                             #
+# ruby-nuggets is free software; you can redistribute it and/or modify it     #
+# under the terms of the GNU Affero General Public License as published by    #
+# the Free Software Foundation; either version 3 of the License, or (at your  #
+# option) any later version.                                                  #
+#                                                                             #
+# ruby-nuggets is distributed in the hope that it will be useful, but WITHOUT #
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or       #
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License #
+# for more details.                                                           #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with ruby-nuggets. If not, see <http://www.gnu.org/licenses/>.        #
+#                                                                             #
+###############################################################################
+#++
+
+require 'nuggets/log_parser'
+
+module Nuggets
+  module LogParser
+    module Apache
+
+      extend self
+
+      DEFAULT_RE = %r{(.*?)}
+
+      DIRECTIVES = {
+        'h' => [:ip,        %r{(\d+(?:\.\d+){3}|[\w.-]+)}],
+        'l' => [:auth,      DEFAULT_RE],
+        'u' => [:username,  DEFAULT_RE],
+        't' => [:datetime,  %r{\[(.*?)\]}],
+        'r' => [:request,   DEFAULT_RE],
+        'R' => [:request,   %r{(.*?)(?:"|\z)}],
+        's' => [:status,    %r{(\d+)}],
+        'b' => [:bytecount, %r{(-|\d+)}],
+        'v' => [:domain,    DEFAULT_RE],
+        'i' => [nil,        DEFAULT_RE],
+      }
+
+      DIRECTIVES_RE = %r{%.*?(?:\{(.*?)\})?([#{DIRECTIVES.keys.join}])([\s\\"]*)}
+
+      ORDER = []
+
+      class << self
+
+        def register(name, format)
+          base = const_set(name, Module.new)
+
+          re, items = parse_format(format)
+          base.const_set(:RE,    re)
+          base.const_set(:ITEMS, items)
+
+          ORDER << base
+          LogParser.register(base, self)
+        end
+
+        def parse_format(format)
+          re, items = '\A', []
+
+          format.scan(DIRECTIVES_RE) { |h, c, e|
+            i, r = DIRECTIVES[c]
+            re << r.source << e.gsub(/\s/, '\\s')
+            items << i ||= h.downcase.tr('-', '_').to_sym
+          }
+
+          [Regexp.new(re), items]
+        end
+
+        def detect_type(line)
+          ORDER.find { |type| line =~ type::RE }
+        end
+
+      end
+
+      [ [:Combined, '%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"'],
+        [:Common,   '%h %l %u %t "%r" %>s %b'],
+        [:Minimal,  '%h %l %u %t "%R']
+      ].each { |name, format| register(name, format) }
+
+      def parse_line(line, entry = {})
+        if md = self::RE.match(line)
+          self::ITEMS.each_with_index { |k, i| entry[k] = md[i + 1] }
+          yield if block_given?
+        end
+
+        entry
+      end
+
+    end
+  end
+end

data/lib/nuggets/log_parser/rails.rb

@@ -0,0 +1,220 @@
+#--
+###############################################################################
+#                                                                             #
+# A component of ruby-nuggets, some extensions to the Ruby programming        #
+# language.                                                                   #
+#                                                                             #
+# Copyright (C) 2007-2012 Jens Wille                                          #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@gmail.com>                                       #
+#                                                                             #
+# ruby-nuggets is free software; you can redistribute it and/or modify it     #
+# under the terms of the GNU Affero General Public License as published by    #
+# the Free Software Foundation; either version 3 of the License, or (at your  #
+# option) any later version.                                                  #
+#                                                                             #
+# ruby-nuggets is distributed in the hope that it will be useful, but WITHOUT #
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or       #
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License #
+# for more details.                                                           #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with ruby-nuggets. If not, see <http://www.gnu.org/licenses/>.        #
+#                                                                             #
+###############################################################################
+#++
+
+require 'nuggets/log_parser'
+
+module Rails
+  module LogParser
+    module Rails
+
+      LogParser.register(self)
+
+      # Log line prefix
+      PREFIX_RE = %r{
+        \A
+        (?:
+          \[
+            (\d+)                #  pid
+            :
+            (.*?)                #  host
+          \]
+          \s
+        )?
+        \s*
+      }x
+
+      # Log entry separator
+      SEPARATOR_RE = %r{
+        \s+\|\s+
+      }x
+
+      # Log line patterns
+      ITEMS = [
+        [:processing, {
+          :re => %r{
+            #{PREFIX_RE}         #  pid, host
+            Processing\s+
+            (\w+)                #  controller
+            \#
+            (\w+)                #  action
+            \s+
+            \(
+              for\s+
+              (.*?)              #  ip
+              \s+at\s+
+              (.*?)              #  datetime
+            \)
+            \s+
+            \[
+              (\w+)              #  request_method
+            \]
+          }xo,
+          :keys => [:controller, :action, :ip, :datetime, :request_method]
+        }],
+        [:session_id, {
+          :re => %r{
+            #{PREFIX_RE}         #  pid, host
+            Session\sID:\s+
+            (\S+)                #  sid
+          }xo,
+          :keys => [:sid]
+        }],
+        [:parameters, {
+          :re => %r{
+            #{PREFIX_RE}         #  pid, host
+            Parameters:\s+
+            (\{.*\})             #  params
+          }xo,  #}
+          :proc => lambda { |entry, md|
+            entry[:params_hash] = md[3].hash
+            entry[:params] = begin
+              eval("$SAFE = 3\n#{md[3].gsub(/#<.*?>/, '%q{\&}')}", nil, __FILE__, __LINE__)  # !!!
+            rescue SyntaxError, SecurityError
+              {}
+            end
+          }
+        }],
+        [:client_info, {
+          :re => %r{
+            #{PREFIX_RE}         #  pid, host
+            Client\sinfo:\s+
+            UA\s+=\s+
+            (.*?)                #  user_agent
+            #{SEPARATOR_RE}
+            LANG\s+=\s+
+            (.*)                 #  accept_language
+          }xo,
+          :keys => [:user_agent, :accept_language]
+        }],
+        [:referer, {
+          :re => %r{
+            #{PREFIX_RE}         #  pid, host
+            Referer:\s+
+            (.*)                 #  referer
+          }xo,
+          :keys => [:referer]
+        }],
+        [:meta, {
+          :re => %r{
+            #{PREFIX_RE}         #  pid, host
+            Meta:\s+
+            User\s+=\s+
+            (.*?)                #  user_id
+            #{SEPARATOR_RE}
+            Institution\s+=\s+
+            (.*)                 #  institution_id
+          }xo,
+          :keys => [:user_id, :institution_id]
+        }],
+        [:stats, {
+          :re => %r{
+            #{PREFIX_RE}         #  pid, host
+            Stats:\s+
+            (.*)                 # flags
+          }xo,
+          :proc => lambda { |entry, md|
+            entry[:flags] = md[3].split(SEPARATOR_RE)
+          }
+        }],
+        [:oauth, {
+          :re => %r{
+            #{PREFIX_RE}         #  pid, host
+            OAuth:\s+
+            Token\s+=\s+
+            (.*?)#(.*?)          #  token_type, token
+            #{SEPARATOR_RE}
+            User\s+=\s+
+            (.*?)                #  user_id
+            #{SEPARATOR_RE}
+            Client\s+=\s+
+            (.*)                 #  client_id
+          }xo,
+          :keys => [:token_type, :token, :user_id, :client_id]
+        }],
+        [:benchmark, {
+          :re => %r{
+            #{PREFIX_RE}         #  pid, host
+            Completed\sin\s+
+            (\S+)                #  runtime
+            .*?
+            (?:                  #- OPTIONAL
+              #{SEPARATOR_RE}
+              Rendering:\s+
+              (\S+)              #  rendering_runtime
+              .*?
+            )?
+            (?:                  #- OPTIONAL
+              #{SEPARATOR_RE}
+              DB:\s+
+              (\S+)              #  db_runtime
+              .*?
+            )?
+            (?:                  #- OPTIONAL
+              #{SEPARATOR_RE}
+              Mem:\s+
+              \S+
+              .*?
+            )?
+            #{SEPARATOR_RE}
+            (.*?)                #  status
+            \s+
+            \[
+              (.*)               #  request_uri
+            \]
+          }xo,
+          :keys => [:runtime, :rendering_runtime, :db_runtime, :status, :request_uri]
+        }]
+      ]
+
+      ITEMS.each { |_, item|
+        item[:proc] ||= lambda { |entry, md|
+          item[:keys].each_with_index { |k, i|
+            entry[k] = md[i + 3]  # 1 = pid, 2 = host
+          }
+        }
+      }
+
+      def parse_line(line, entry = {})
+        ITEMS.each { |key, item|
+          if md = item[:re].match(line)
+            if key == :processing
+              yield if block_given?
+              entry[:pid], entry[:host] = md[1], md[2]
+            end
+
+            item[:proc][entry, md]
+
+            break
+          end
+        }
+
+        entry
+      end
+
+    end
+  end
+end

data/lib/nuggets/log_parser.rb

@@ -0,0 +1,71 @@
+#--
+###############################################################################
+#                                                                             #
+# A component of ruby-nuggets, some extensions to the Ruby programming        #
+# language.                                                                   #
+#                                                                             #
+# Copyright (C) 2007-2012 Jens Wille                                          #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@gmail.com>                                       #
+#                                                                             #
+# ruby-nuggets is free software; you can redistribute it and/or modify it     #
+# under the terms of the GNU Affero General Public License as published by    #
+# the Free Software Foundation; either version 3 of the License, or (at your  #
+# option) any later version.                                                  #
+#                                                                             #
+# ruby-nuggets is distributed in the hope that it will be useful, but WITHOUT #
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or       #
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License #
+# for more details.                                                           #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with ruby-nuggets. If not, see <http://www.gnu.org/licenses/>.        #
+#                                                                             #
+###############################################################################
+#++
+
+require 'zlib'
+
+module Nuggets
+  module LogParser
+
+    extend self
+
+    GZ_EXT_RE = %r{\.gz\z}
+
+    def self.register(base, *modules)
+      base.send(:include, *modules << self)
+      base.extend(base)
+    end
+
+    def parse(input)
+      entry = {}
+
+      input.each { |line| parse_line(line, entry) {
+        unless entry.empty?
+          yield entry.dup
+          entry.clear
+        end
+      } }
+
+      yield entry unless entry.empty?
+    end
+
+    def parse_line(line, entry = {})
+      # Yield when entry complete. Preferrably return +entry+.
+      raise NotImplementedError, 'must be implemented by type'
+    end
+
+    def parse_file(file, &block)
+      block ||= (entries = []; lambda { |entry| entries << entry })
+
+      (file =~ GZ_EXT_RE ? Zlib::GzipReader : File).open(file) { |f|
+        block.arity == 1 ? parse(f, &block) : block[f, method(:parse)]
+      }
+
+      entries
+    end
+
+  end
+end