td-logger 0.2.5 → 0.2.6

data/ChangeLog

@@ -1,4 +1,14 @@
 
+== 2011-10-06 version 0.2.6
+
+* Divided 'uri' column into 'path' and 'host' columns in access logs
+* Access log merges params which are not filtered by config.filter_parameters
+* Access logger doesn't overwrite exist keys even if its value is null
+* Access log uses 'agent' column instead of 'ua' which is same as
+  '$ td import --format=apache'
+* Run IO threads lazily for Unicorn and Passenger
+
+
 == 2011-09-30 version 0.2.5
 
 * Use HTTP_X_FORWARDED_FOR instead of REMOTE_ADDR if it's available

data/lib/td/logger.rb

@@ -12,7 +12,6 @@ def self.open_agent(tag, agent_host, agent_port)
 end
 
 def self.log(tag, record)
-  record['time'] ||= Time.now.to_i
   Fluent::Logger.post(tag, record)
 end
 

data/lib/td/logger/agent/access_log.rb

@@ -22,7 +22,15 @@ module Agent
     :action => :action,
   }
 
-  def self.enable_access_log(tag)
+  def self.enable_access_log(config)
+    tag = config.access_log_table
+
+    if config.rails_config.respond_to?(:filter_parameters)
+      filter_parameters = config.rails_config.filter_parameters
+    else
+      filter_parameters = []
+    end
+
     Middleware.before do |env|
       record = {}
       Thread.current['td.access_log'] = record
@@ -39,29 +47,78 @@ module Agent
       # ignore OPTIONS request
       if req.request_method != "OPTIONS"
         record = env['td.access_log'] || {}
-        access_time = env['td.access_time']
 
-        # add 'elapsed' column
-        if access_time
-          elapsed = Time.now - access_time
-          record[:elapsed] = elapsed
-          # set 'time' column to access time
+        # 'elapsed' column
+        if access_time = env['td.access_time']
+          unless record.has_key?(:elapsed)
+            record[:elapsed] = Time.now - access_time
+          end
+
+          # always overwrite 'time' column by access time
           record[:time] = access_time
         end
 
-        record[:method] ||= req.request_method
-        record[:ip] ||= (env['action_dispatch.remote_ip'] || req.ip).to_s
-        record[:uri] ||= env['REQUEST_URI'].to_s if env['REQUEST_URI']
-        record[:referer] ||= env['HTTP_REFERER'].to_s if env['HTTP_REFERER']
-        record[:ua] ||= env['HTTP_USER_AGENT'].to_s if env['HTTP_USER_AGENT']
-
-        m = env[ACCESS_LOG_PARAM_ENV]
-        ACCESS_LOG_PRESET_PARAM_KEYS.each_pair {|key,val|
-          record[key] ||= m[val] if m[val]
+        # merge params
+        req.params.each_pair {|key,val|
+          key = key.to_sym
+          unless record.has_key?(key) || filter_parameters.include?(key)
+            record[key] = val
+          end
         }
 
-        # result code
-        record[:status] ||= result[0].to_i
+        # 'method' column
+        if !record.has_key?(:method)
+          record[:method] = req.request_method
+        end
+
+        # 'ip' column
+        unless record.has_key?(:ip)
+          record[:ip] = (env['action_dispatch.remote_ip'] || req.ip).to_s
+        end
+
+        # 'path' column
+        #   requested path before '?'
+        unless record.has_key?(:path)
+          if path = env['REQUEST_URI']
+            if m = /(?:\w{1,10}\:\/\/[^\/]+)?([^\?]*)/.match(path)
+              record[:path] = m[1]
+            end
+          end
+        end
+
+        # 'host' column
+        #   Rack#host_with_port consideres HTTP_X_FORWARDED_HOST
+        unless record.has_key?(:host)
+          record[:host] = req.host_with_port
+        end
+
+        # 'referer' column
+        unless record.has_key?(:referer)
+          if referer = env['HTTP_REFERER']
+            record[:referer] = referer.to_s
+          end
+        end
+
+        # 'agent' column
+        unless record.has_key?(:agent)
+          if agent = env['HTTP_USER_AGENT']
+            record[:agent] = agent
+          end
+        end
+
+        # 'status' column
+        unless record.has_key?(:status)
+          record[:status] = result[0].to_i
+        end
+
+        # 'controller' and 'action' columns
+        if m = env[ACCESS_LOG_PARAM_ENV]
+          ACCESS_LOG_PRESET_PARAM_KEYS.each_pair {|key,val|
+            unless record.has_key?(key)
+              record[key] = m[val] if m[val]
+            end
+          }
+        end
 
         TreasureData.log(tag, record)
       end

data/lib/td/logger/agent/rails.rb

@@ -24,7 +24,9 @@ test:
 EOF
 
     class Config
-      def initialize(conf)
+      def initialize(conf, rails_config)
+        @rails_config = rails_config
+
         if agent = conf['agent']
           host, port = agent.split(':',2)
           port = (port || 24224).to_i
@@ -48,6 +50,7 @@ EOF
         @access_log_table = conf['access_log_table']
       end
 
+      attr_reader :rails_config
       attr_reader :agent_host, :agent_port, :tag
       attr_reader :apikey, :database, :auto_create_table
       attr_reader :access_log_table
@@ -79,7 +82,7 @@ EOF
           'database' => ENV['TREASURE_DATA_DB'] || "rails_#{::Rails.env}",
           'access_log_table' => ENV['TREASURE_DATA_TABLE'] || 'web_access',
           'auto_create_table' => true
-        })
+        }, rails)
       end
 
       begin
@@ -102,7 +105,7 @@ EOF
       end
 
       begin
-        return Config.new(conf)
+        return Config.new(conf, rails)
       rescue
         logger.warn "#{CONFIG_PATH}: #{$!}."
         logger.warn "Disabling Treasure Data logger."
@@ -128,7 +131,7 @@ EOF
       rails.middleware.use Agent::Middleware
 
       if c.access_log_enabled?
-        Agent.enable_access_log(c.access_log_table)
+        Agent.enable_access_log(c)
       end
       Agent::Rails.init_controller
       Agent::Rails.init_model

data/lib/td/logger/tdlog.rb

@@ -56,7 +56,10 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
     @finish = false
     @next_time = Time.now.to_i + @flush_interval
     @error_count = 0
-    @upload_thread = Thread.new(&method(:upload_main))
+
+    # start thread when the first post() is called for
+    # Unicorn and Passenger.
+    @upload_thread = nil
   end
 
   attr_accessor :logger
@@ -68,7 +71,7 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
         @flush_now = true
         @cond.signal
       }
-      @upload_thread.join
+      @upload_thread.join if @upload_thread
 
       @map.each {|(db,table),buffer|
         upload(db, table, buffer)
@@ -80,11 +83,11 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
   end
 
   def post(tag, record)
+    record[:time] ||= Time.now.to_i
+
     tag = "#{@tag}.#{tag}"
     db, table = tag.split('.')[-2, 2]
 
-    record['time'] ||= Time.now.to_i
-
     key = [db, table]
     @mutex.synchronize do
       buffer = (@map[key] ||= '')
@@ -95,6 +98,11 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
         @map.delete(key)
         @cond.signal
       end
+
+      # stat upload thread if it's not run
+      unless @upload_thread
+        @upload_thread = Thread.new(&method(:upload_main))
+      end
     end
 
     nil

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: td-logger
 version: !ruby/object:Gem::Version 
-  hash: 29
+  hash: 27
   prerelease: false
   segments: 
   - 0
   - 2
-  - 5
-  version: 0.2.5
+  - 6
+  version: 0.2.6
 platform: ruby
 authors: 
 - Sadayuki Furuhashi
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-09-30 00:00:00 +09:00
+date: 2011-10-06 00:00:00 +09:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency