td-logger 0.1.2 → 0.2.0

data/ChangeLog

@@ -1,4 +1,12 @@
 
+== 2011-08-28 version 0.2.0
+
+* Rewritten
+* Supported access logging: td_access_log or TreasureData.access_log
+* Supported model change logging: td_enable_model_tracer
+* Supported other logging: TreasureData.log
+
+
 == 2011-08-21 version 0.1.1
 
 * Updated library dependencies

data/README.rdoc

@@ -13,44 +13,85 @@ edit +environment.rb+ and add to the initalizer block:
 
 And then add +config/treasure_data.yml+ file as following:
 
-    # logging to Treasure Data directry
+    # logging to Treasure Data directly
     development:
       apikey: "YOUR_API_KEY"
       database: myapp
+      access_log_table: access
       auto_create_table: true
     
     # logging via td-agent
     production:
       agent: "localhost:24224"
-      tag: myapp
+      tag: td.myapp
+      access_log_table: access
     
     # disable logging
     test:
 
-=== Logging actions
+=== Logging access logs
 
-Add 'add_td_tracer' line to your controller classes:
+Following information will be logged automatically:
+
+* controller name ('controller' column)
+* action name ('action' column)
+* remote host ip address ('remote_addr' column)
+* http status code ('status' column)
+* http referer ('refer' column)
+
+To add information to the log, call 'td_access_log' method in a Controller:
 
     class YourController < ApplicationController
       def myaction
-        # ...
+        td_access_log[:column] = "value"
+      end
+    end
+
+Or call 'TreasureData.access_log' method outside of a Controller:
+
+    class MyHelperModule
+      def mymethod
+        TreasureData.access_log[:column] = "value"
       end
-      add_td_tracer :myaction, 'mytable'
     end
 
-It logs {"controller":"your","action":"myaction"}. Additionally, routing parameters are included automatically.
 
-You can add environment variables by adding 'extra' option as follows:
+=== Logging model changes
+
+Add 'td_enable_model_tracer' line to your Model class:
+
+    class MyModel < ActiveRecord::Base
+      td_enable_model_tracer :mytable
+    end
+
+It logs all columns when the model is changed. Use :only option to limit columns to log:
+
+    class MyModel < ActiveRecord::Base
+      td_enable_model_tracer :mytable, :only => [:id, :mycol]
+    end
 
-    # logs client address on the 'addr' column
-    add_td_tracer :myaction, 'mytable', :extra=>{:REMOTE_ADDR=>:addr}
+Alternatively, use :except option to except columns to log:
 
-    # logs referer and path info
-    add_td_tracer :myaction, 'mytable', :extra=>{:HTTP_REFERER=>:referer, :PATH_INFO=>:path}
+    class MyModel < ActiveRecord::Base
+      td_enable_model_tracer :mytable, :except => [:created_at]
+    end
 
 Add 'static' option to add constant key-value pairs:
 
-    add_td_tracer :myaction, 'mytable', :static=>{:version=>1}
+    class MyModel < ActiveRecord::Base
+      td_enable_model_tracer :mytable, :except => [:created_at], :static => {'model'=>'my'}
+    end
+
+
+=== Logging other logs
+
+You can log anytime using 'TreasureData.log' method:
+
+    class MyClass
+      def mymethod
+        TreasureData.log('mytable', {:col1=>"val1", :col2=>"val2"}
+      end
+    end
 
 
 == Copyright

data/lib/td/logger.rb

@@ -1,266 +1,32 @@
+require 'fluent/logger'
 
 module TreasureData
-module Logger
 
-
-class Config
-  def initialize(conf)
-    if agent = conf['agent']
-      host, port = agent.split(':',2)
-      port = (port || 24224).to_i
-      @agent_host = host
-      @agent_port = port
-
-      @tag = conf['tag']
-      @tag ||= conf['database']
-      raise "'tag' nor 'database' options are not set" unless @tag
-
-    else
-      @apikey = conf['apikey']
-      raise "'apikey' option is not set" unless @apikey
-
-      @database = conf['database']
-      raise "'database' option is not set" unless @database
-
-      @auto_create_table = !!conf['auto_create_table']
-    end
-  end
-
-  attr_reader :agent_host, :agent_port, :tag
-  attr_reader :apikey, :database, :auto_create_table
-
-  def agent_mode?
-    @agent_host != nil
-  end
+def self.open(apikey, database, auto_create_table=false)
+  require 'td/logger/tdlog'
+  TreasureData::Logger::TreasureDataLogger.open(apikey, database, auto_create_table)
 end
 
-
+def self.open_agent(tag, agent_host, agent_port)
+  Fluent::Logger::FluentLogger.open(tag, agent_host, agent_port)
 end
-end
-
-
-module TreasureData
-
 
 def self.log(tag, record)
   record['time'] ||= Time.now.to_i
   Fluent::Logger.post(tag, record)
 end
 
-
-module Logger
-module RailsAgent
-
-CONFIG_SAMPLE = <<EOF
-defaults: &defaults
-  apikey: "YOUR_API_KEY"
-  database: myapp
-  table: access
-
-test:
-  <<: *defaults
-
-development:
-  <<: *defaults
-
-production:
-  <<: *defaults
-EOF
-
-CONFIG_PATH = 'config/treasure_data.yml'
-
-def self.init(rails)
-  c = read_config(rails)
-  return unless c
-
-  require 'fluent/logger'
-  if c.agent_mode?
-    Fluent::Logger::FluentLogger.open(c.tag, c.agent_host, c.agent_port)
-  else
-    require 'td/logger/tdlog'
-    TreasureDataLogger.open(c.apikey, c.database, c.auto_create_table)
-  end
-
-  rails.middleware.use Middleware
-  ActionController::Base.class_eval do
-    include ::TreasureData::Logger::RailsAgent::ControllerLogger
-  end
-end
-
-def self.read_config(rails)
-  logger = Rails.logger || ::Logger.new(STDOUT)
-  begin
-    yaml = YAML.load_file("#{RAILS_ROOT}/#{CONFIG_PATH}")
-  rescue
-    logger.warn "Can't load #{CONFIG_PATH} file."
-    logger.warn "  #{$!}"
-    logger.warn "Put the following file:"
-    logger.warn sample
-    return
-  end
-
-  conf = yaml[RAILS_ENV]
-  unless conf
-    logger.warn "#{CONFIG_PATH} doesn't include setting for current environment (#{RAILS_ENV})."
-    logger.warn "Disabling Treasure Data logger."
-    return
-  end
-
-  begin
-    return Config.new(conf)
-  rescue
-    logger.warn "#{CONFIG_PATH}: #{$!}."
-    logger.warn "Disabling Treasure Data logger."
-    return
-  end
-end
-
-class Middleware
-  def initialize(app, options={})
-    @app = app
-  end
-
-  def call(env)
-    r = @app.call(env)
-
-    if m = env['treasure_data.log_method']
-      m.call(env)
-    end
-
-    r
-  end
-
-  def self.set_log_method(env, method)
-    env['treasure_data.log_method'] = method
-  end
 end
 
-module ControllerLogger
-  def self.included(mod)
-    mod.extend(ModuleMethods)
-  end
-
-  class ActionLogger
-    PARAM_KEY = if defined? Rails
-        if Rails.respond_to?(:version) && Rails.version =~ /^3/
-          # Rails 3
-          'action_dispatch.request.path_parameters'
-        else
-          # Rails 2
-          'action_controller.request.path_parameters'
-        end
-      else
-        # Rack default
-        'rack.routing_args'
-      end
-
-    def initialize(method, tag, options)
-      @method = method
-      @tag = tag
 
-      @only = nil
-      @except = nil
-      @extra = nil
-      @static = {}
-
-      if o = options[:only_params]
-        @only = case o
-          when Array
-            o
-          else
-            [o]
-          end.map {|e| e.to_s }
-      end
-
-      if o = options[:except_params]
-        @except = case o
-          when Array
-            o
-          else
-            [o]
-          end.map {|e| e.to_s }
-      end
-
-      if o = options[:extra]
-        @extra = case o
-          when Hash
-            m = {}
-            o.each_pair {|k,v| m[k.to_s] = v.to_s }
-            m
-          when Array
-            o.map {|e|
-              case e
-              when Hash
-                m = {}
-                e.each_pair {|k,v| m[k.to_s] = v.to_s }
-                m
-              else
-                {e.to_s => e.to_s}
-              end
-            }.inject({}) {|r,e| r.merge!(e) }
-          else
-            {o.to_s => o.to_s}
-          end
-      end
-
-      if o = options[:static]
-        o.each_pair {|k,v| @static[k] = v }
-      end
-    end
-
-    def call(env)
-      m = env[PARAM_KEY].dup || {}
-
-      if @only
-        m.reject! {|k,v| !@only.include?(k) }
-      end
-      if @except
-        m.reject! {|k,v| @except.include?(k) }
-      end
-      if @extra
-        @extra.each_pair {|k,v| m[v] = env[k] }
-      end
-
-      m.merge!(@static)
-
-      ::TreasureData.log(@tag, m)
-    end
-  end
-
-  module ModuleMethods
-    def add_td_tracer(method, tag, options={})
-      al = ActionLogger.new(method, tag, options)
-      module_eval <<-EOF
-        def #{method}_td_action_tracer_(*args, &block)
-          ::TreasureData::Logger::RailsAgent::Middleware.set_log_method(request.env, method(:#{method}_td_action_trace_))
-          #{method}_td_action_tracer_orig_(*args, &block)
-        end
-      EOF
-      module_eval do
-        define_method(:"#{method}_td_action_trace_", &al.method(:call))
-      end
-      alias_method "#{method}_td_action_tracer_orig_", method
-      alias_method method, "#{method}_td_action_tracer_"
-    end
+class Time
+  def to_msgpack(out = '')
+    to_i.to_msgpack(out)
   end
 end
 
-end
-end
-
-end
 
 if defined? Rails
-  if Rails.respond_to?(:version) && Rails.version =~ /^3/
-    module TreasureData
-      class Railtie < Rails::Railtie
-        initializer "treasure_data_agent.start_plugin" do |app|
-          TreasureData::Logger::RailsAgent.init(app.config)
-        end
-      end
-    end
-  else
-    TreasureData::Logger::RailsAgent.init(Rails.configuration)
-  end
+  require 'td/logger/agent/rails'
 end
 

data/lib/td/logger/agent/access_log.rb

@@ -0,0 +1,58 @@
+
+module TreasureData
+module Logger
+module Agent
+
+  ACCESS_LOG_PARAM_ENV =
+        if defined? Rails
+          if Rails.respond_to?(:version) && Rails.version =~ /^3/
+            # Rails 3
+            'action_dispatch.request.path_parameters'
+          else
+            # Rails 2
+            'action_controller.request.path_parameters'
+          end
+        else
+          # Rack default
+          'rack.routing_args'
+        end
+
+  ACCESS_LOG_PRESET_PARAM_KEYS = {
+    'controller' => 'controller',
+    'action' => 'action',
+  }
+
+  ACCESS_LOG_PRESET_ENV_KEYS = {
+    'remote_addr' => 'REMOTE_ADDR',
+    'referer' => 'HTTP_REFERER',
+  }
+
+  def self.enable_access_log(tag)
+    Middleware.before do |env|
+      data = {}
+      Thread.current['td.access_log'] = data
+      env['td.access_log'] = data
+    end
+
+    Middleware.after do |env,result|
+      data = env['td.access_log'] || {}
+
+      ACCESS_LOG_PRESET_ENV_KEYS.each_pair {|key,val|
+        data[key] ||= env[val] if env[val]
+      }
+
+      m = env[ACCESS_LOG_PARAM_ENV]
+      ACCESS_LOG_PRESET_PARAM_KEYS.each_pair {|key,val|
+        data[key] ||= m[val] if m[val]
+      }
+
+      # result code
+      data['status'] ||= result[0].to_i
+
+      TreasureData.log(tag, data)
+    end
+  end
+
+end
+end
+end

data/lib/td/logger/agent/middleware.rb

@@ -0,0 +1,41 @@
+
+module TreasureData
+module Logger
+module Agent
+
+
+class Middleware
+  @@before = []
+  @@after = []
+
+  def self.before(&block)
+    @@before << block
+  end
+
+  def self.after(&block)
+    @@after << block
+  end
+
+  def initialize(app, options={})
+    @app = app
+  end
+
+  def call(env)
+    @@before.each {|m|
+      m.call(env)
+    }
+
+    result = @app.call(env)
+
+    @@after.each {|m|
+      m.call(env, result)
+    }
+
+    result
+  end
+end
+
+
+end
+end
+end

data/lib/td/logger/agent/rails.rb

@@ -0,0 +1,134 @@
+module TreasureData
+module Logger
+module Agent
+  require 'td/logger/agent/middleware'
+  require 'td/logger/agent/access_log'
+  require 'td/logger/agent/rails/controller'
+  require 'td/logger/agent/rails/model'
+
+  module Rails
+
+    CONFIG_PATH = 'config/treasure_data.yml'
+
+    CONFIG_SAMPLE = <<EOF
+# logging to Treasure Data directly
+development:
+  apikey: "YOUR_API_KEY"
+  database: myapp
+  access_log_table: access
+  auto_create_table: true
+
+# logging via td-agent
+production:
+  agent: "localhost:24224"
+  tag: td.myapp
+  access_log_table: access
+
+# disable logging
+test:
+EOF
+
+    class Config
+      def initialize(conf)
+        if agent = conf['agent']
+          host, port = agent.split(':',2)
+          port = (port || 24224).to_i
+          @agent_host = host
+          @agent_port = port
+
+          @tag = conf['tag']
+          @tag ||= conf['database']
+          raise "'tag' nor 'database' options are not set" unless @tag
+
+        else
+          @apikey = conf['apikey']
+          raise "'apikey' option is not set" unless @apikey
+
+          @database = conf['database']
+          raise "'database' option is not set" unless @database
+
+          @auto_create_table = !!conf['auto_create_table']
+        end
+
+        @access_log_table = conf['access_log_table']
+      end
+
+      attr_reader :agent_host, :agent_port, :tag
+      attr_reader :apikey, :database, :auto_create_table
+      attr_reader :access_log_table
+
+      def agent_mode?
+        @agent_host != nil
+      end
+
+      def access_log_enabled?
+        !@access_log_table.nil? && !@access_log_table.empty?
+      end
+    end
+
+    def self.read_config(rails)
+      logger = ::Rails.logger || ::Logger.new(STDOUT)
+      begin
+        yaml = YAML.load_file("#{RAILS_ROOT}/#{CONFIG_PATH}")
+      rescue
+        logger.warn "Can't load #{CONFIG_PATH} file."
+        logger.warn "  #{$!}"
+        logger.warn "Put the following file:"
+        logger.warn sample
+        return
+      end
+
+      conf = yaml[RAILS_ENV]
+      unless conf
+        logger.warn "#{CONFIG_PATH} doesn't include setting for current environment (#{RAILS_ENV})."
+        logger.warn "Disabling Treasure Data logger."
+        return
+      end
+
+      begin
+        return Config.new(conf)
+      rescue
+        logger.warn "#{CONFIG_PATH}: #{$!}."
+        logger.warn "Disabling Treasure Data logger."
+        return
+      end
+    end
+
+    def self.init(rails)
+      c = read_config(rails)
+      return unless c
+
+      if c.agent_mode?
+        ::TreasureData.open_agent(c.tag, c.agent_host, c.agent_port)
+      else
+        ::TreasureData.open(c.apikey, c.database, c.auto_create_table)
+      end
+
+      rails.middleware.use Agent::Middleware
+
+      if c.access_log_enabled?
+        Agent.enable_access_log(c.access_log_table)
+      end
+      Agent::Rails.init_controller
+      Agent::Rails.init_model
+    end
+
+  end
+end
+end
+end
+
+if defined? Rails
+  if Rails.respond_to?(:version) && Rails.version =~ /^3/
+    module TreasureData
+      class Railtie < Rails::Railtie
+        initializer "treasure_data_agent.start_plugin" do |app|
+          TreasureData::Logger::Agent::Rails.init(app.config)
+        end
+      end
+    end
+  else
+    TreasureData::Logger::Agent::Rails.init(Rails.configuration)
+  end
+end
+

data/lib/td/logger/agent/rails/controller.rb

@@ -0,0 +1,32 @@
+module TreasureData
+module Logger
+module Agent
+  module Rails
+
+    def self.init_controller
+      ms = ControllerMethods
+      modms = ControllerModuleMethods
+      ActionController::Base.class_eval do
+        include ms
+        extend modms
+      end
+    end
+
+    module ControllerMethods
+      def td_access_log
+        request.env['td.access_log'] ||= {}
+      end
+    end
+
+    module ControllerModuleMethods
+    end
+
+  end
+end
+end
+
+def self.access_log
+  Thread.current['td.access_log']
+end
+
+end

data/lib/td/logger/agent/rails/model.rb

@@ -0,0 +1,79 @@
+module TreasureData
+module Logger
+module Agent
+  module Rails
+
+    def self.init_model
+      ms = ModelMethods
+      modms = ModelModuleMethods
+      require 'active_record' unless defined?(ActiveRecord)
+      ActiveRecord::Base.class_eval do
+        include ms
+        extend modms
+      end
+    end
+
+    module ModelMethods
+    end
+
+    module ModelModuleMethods
+      def td_enable_model_tracer(tag, options={})
+        only = nil
+        except = nil
+        static = {}
+
+        if o = options[:only]
+          only = case o
+            when Array
+              o
+            else
+              [o]
+            end.map {|e| e.to_s }
+        end
+
+        if o = options[:except]
+          except = case o
+            when Array
+              o
+            else
+              [o]
+            end.map {|e| e.to_s }
+        end
+
+        if o = options[:static]
+          o.each_pair {|k,v|
+            static[k.to_s] = v
+          }
+        end
+
+        if defined?(after_commit)
+          # Rails 3
+          m = :after_commit
+        else
+          # Rails 2
+          m = :after_save
+        end
+
+        __send__(m) do |record|
+          data = {}
+          record.attribute_names.each {|name|
+            name = name.to_s
+            if (!only || only.include?(name)) && (!except || !except.include?(name))
+              data[name] = record.read_attribute(name)
+            end
+          }
+          static.each_pair {|k,v|
+            data[k] = v
+          }
+          if time = data['updated_at'] && time.is_a?(Time)
+            data['time'] = time.to_i
+            data.delete('updated_at')
+          end
+          TreasureData.log(tag, data)
+        end
+      end
+    end
+  end
+end
+end
+end

data/lib/td/logger/tdlog.rb

@@ -2,7 +2,7 @@
 module TreasureData
 module Logger
 
-# TODO shutdown handler
+# TODO shutdown handler  (deadlock)
 
 class TreasureDataLogger < Fluent::Logger::LoggerBase
   def initialize(apikey, tag, auto_create_table)

data/lib/td/logger/version.rb

@@ -1,7 +1,7 @@
 module TreasureData
 module Logger
 
-VERSION = '0.1.2'
+VERSION = '0.2.0'
 
 end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: td-logger
 version: !ruby/object:Gem::Version 
-  hash: 31
+  hash: 23
   prerelease: false
   segments: 
   - 0
-  - 1
   - 2
-  version: 0.1.2
+  - 0
+  version: 0.2.0
 platform: ruby
 authors: 
 - Sadayuki Furuhashi
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-08-21 00:00:00 +09:00
+date: 2011-08-28 00:00:00 +09:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -78,6 +78,11 @@ extra_rdoc_files:
 files: 
 - lib/td-logger.rb
 - lib/td/logger.rb
+- lib/td/logger/agent/access_log.rb
+- lib/td/logger/agent/middleware.rb
+- lib/td/logger/agent/rails.rb
+- lib/td/logger/agent/rails/controller.rb
+- lib/td/logger/agent/rails/model.rb
 - lib/td/logger/tdlog.rb
 - lib/td/logger/version.rb
 - ChangeLog