td-logger 0.1.0

data/ChangeLog

@@ -0,0 +1,5 @@
+
+Release 0.1.0 - 2011/08/21
+
+  * First release
+

data/README.rdoc

@@ -0,0 +1,60 @@
+= Treasure Data logging library for Rails
+
+== Getting Started
+
+Add the following line to your Gemfile:
+
+    gem 'td-logger'
+
+For Rails 2.x (not tested) without Bundler,
+edit +environment.rb+ and add to the initalizer block:
+
+    config.gem "td-logger"
+
+And then add +config/treasure_data.yml+ file as following:
+
+    # logging to Treasure Data directry
+    development:
+      apikey: "YOUR_API_KEY"
+      database: myapp
+      auto_create_table: true
+    
+    # logging via td-agent
+    production:
+      agent: "localhost:24224"
+      tag: myapp
+    
+    # disable logging
+    test:
+
+=== Logging actions
+
+Add 'add_td_tracer' line to your controller classes:
+
+    class YourController < ApplicationController
+      def myaction
+        # ...
+      end
+      add_td_tracer :myaction, 'mytable'
+    end
+
+It logs {"controller":"your","action":"myaction"}. Additionally, routing parameters are included automatically.
+
+You can add environment variables by adding 'extra' option as follows:
+
+    # logs client address on the 'addr' column
+    add_td_tracer :myaction, 'mytable', :extra=>{:REMOTE_ADDR=>:addr}
+
+    # logs referer and path info
+    add_td_tracer :myaction, 'mytable', :extra=>{:HTTP_REFERER=>:referer, :PATH_INFO=>:path}
+
+Add 'static' option to add constant key-value pairs:
+
+    add_td_tracer :myaction, 'mytable', :static=>{:version=>1}
+
+
+== Copyright
+
+Copyright:: Copyright (c) 2011 Treasure Data Inc.
+License::   Apache License, Version 2.0
+

data/lib/td/logger/tdlog.rb

@@ -0,0 +1,187 @@
+
+module TreasureData
+module Logger
+
+# TODO shutdown handler
+
+class TreasureDataLogger < Fluent::Logger::LoggerBase
+  def initialize(apikey, tag, auto_create_table)
+    require 'thread'
+    require 'stringio'
+    require 'zlib'
+    require 'msgpack'
+    require 'time'
+    require 'net/http'
+    require 'cgi'
+    require 'logger'
+
+    @tag = tag
+    @auto_create_table = auto_create_table
+    @logger = ::Logger.new(STDOUT)
+
+    # TODO
+    gem 'td'
+    require 'td/client'
+    @client = TreasureData::Client.new(apikey)
+
+    @mutex = Mutex.new
+    @cond = ConditionVariable.new
+    @map = {}  # (db,table) => buffer:String
+    @queue = []
+
+    @chunk_limit = 8*1024*1024
+    @flush_interval = 60
+    @retry_wait = 1.0
+    @retry_limit = 8
+
+    @finish = false
+    @next_time = Time.now.to_i + @flush_interval
+    @error_count = 0
+    @upload_thread = Thread.new(&method(:upload_main))
+  end
+
+  attr_accessor :logger
+
+  def close
+    @finish = true
+    @mutex.synchronize {
+      @flush_now = true
+      @cond.signal
+    }
+  end
+
+  def post(tag, record)
+    tag = "#{@tag}.#{tag}"
+    db, table = tag.split('.')[-2, 2]
+
+    record['time'] ||= Time.now.to_i
+
+    key = [db, table]
+    @mutex.synchronize do
+      buffer = (@map[key] ||= '')
+      record.to_msgpack(buffer)
+
+      if buffer.size > @chunk_limit
+        @queue << [db, table, buffer]
+        @map.delete(key)
+        @cond.signal
+      end
+    end
+
+    nil
+  end
+
+  def upload_main
+  @mutex.lock
+  until @finish
+    now = Time.now.to_i
+
+    if @next_time <= now || (@flush_now && @error_count == 0)
+      @mutex.unlock
+      begin
+        try_flush
+      ensure
+        @mutex.lock
+      end
+      @flush_now = false
+    end
+
+    if @error_count == 0
+      next_wait = @flush_interval
+    else
+      next_wait = @retry_wait * (2 ** (@error_count-1))
+    end
+    @next_time = next_wait + now
+
+    cond_wait(next_wait)
+  end
+
+  rescue
+    @logger.error "Unexpected error: #{$!}"
+    $!.backtrace.each {|bt|
+      @logger.info bt
+    }
+  ensure
+    @mutex.unlock
+  end
+
+  private
+  def try_flush
+    @mutex.synchronize do
+      if @queue.empty?
+        @map.reject! {|(db,table),buffer|
+          @queue << [db, table, buffer]
+        }
+      end
+    end
+
+    until @queue.empty?
+      tuple = @queue.first
+
+      begin
+        upload(*tuple)
+        @queue.shift
+        @error_count = 0
+      rescue
+        if @error_count < @retry_limit
+          @logger.error "Failed to upload logs to Treasure Data, retrying: #{$!}"
+          @error_count += 1
+        else
+          @logger.error "Failed to upload logs to Treasure Data, trashed: #{$!}"
+          $!.backtrace.each {|bt|
+            @logger.info bt
+          }
+          @error_count = 0
+          @queue.clear
+        end
+        return
+      end
+    end
+  end
+
+  def upload(db, table, buffer)
+    out = StringIO.new
+    Zlib::GzipWriter.wrap(out) {|gz| gz.write buffer }
+    stream = StringIO.new(out.string)
+
+    begin
+      @client.import(db, table, "msgpack.gz", stream, stream.size)
+    rescue TreasureData::NotFoundError
+      unless @auto_create_table
+        raise $!
+      end
+      @logger.info "Creating table #{db}.#{table} on TreasureData"
+      begin
+        @client.create_log_table(db, table)
+      rescue TreasureData::NotFoundError
+        @client.create_database(db)
+        @client.create_log_table(db, table)
+      end
+      retry
+    end
+  end
+
+  def e(s)
+    CGI.escape(s.to_s)
+  end
+
+  if ConditionVariable.new.method(:wait).arity == 1
+    #$log.warn "WARNING: Running on Ruby 1.8. Ruby 1.9 is recommended."
+    require 'timeout'
+    def cond_wait(sec)
+      Timeout.timeout(sec) {
+        @cond.wait(@mutex)
+      }
+    rescue Timeout::Error
+    end
+  else
+    def cond_wait(sec)
+      @cond.wait(@mutex, sec)
+    end
+  end
+end
+
+
+end
+end
+

data/lib/td/logger/version.rb

@@ -0,0 +1,7 @@
+module TreasureData
+module Logger
+
+VERSION = '0.1.0'
+
+end
+end

data/lib/td/logger.rb

@@ -0,0 +1,266 @@
+
+module TreasureData
+module Logger
+
+
+class Config
+  def initialize(conf)
+    if agent = conf['agent']
+      host, port = agent.split(':',2)
+      port = (port || 24224).to_i
+      @agent_host = host
+      @agent_port = port
+
+      @tag = conf['tag']
+      @tag ||= conf['database']
+      raise "'tag' nor 'database' options are not set" unless @tag
+
+    else
+      @apikey = conf['apikey']
+      raise "'apikey' option is not set" unless @apikey
+
+      @database = conf['database']
+      raise "'database' option is not set" unless @database
+
+      @auto_create_table = !!conf['auto_create_table']
+    end
+  end
+
+  attr_reader :agent_host, :agent_port, :tag
+  attr_reader :apikey, :database, :auto_create_table
+
+  def agent_mode?
+    @agent_host != nil
+  end
+end
+
+
+end
+end
+
+
+module TreasureData
+
+
+def self.log(tag, record)
+  record['time'] ||= Time.now.to_i
+  Fluent::Logger.post(tag, record)
+end
+
+
+module Logger
+module RailsAgent
+
+CONFIG_SAMPLE = <<EOF
+defaults: &defaults
+  apikey: "YOUR_API_KEY"
+  database: myapp
+  table: access
+
+test:
+  <<: *defaults
+
+development:
+  <<: *defaults
+
+production:
+  <<: *defaults
+EOF
+
+CONFIG_PATH = 'config/treasure_data.yml'
+
+def self.init(rails)
+  c = read_config(rails)
+  return unless c
+
+  require 'fluent/logger'
+  if c.agent_mode?
+    Fluent::Logger::FluentLogger.open(c.tag, c.agent_host, c.agent_port)
+  else
+    require 'td/logger/tdlog'
+    TreasureDataLogger.open(c.apikey, c.database, c.auto_create_table)
+  end
+
+  rails.middleware.use Middleware
+  ActionController::Base.class_eval do
+    include ::TreasureData::Logger::RailsAgent::ControllerLogger
+  end
+end
+
+def self.read_config(rails)
+  logger = Rails.logger || ::Logger.new(STDOUT)
+  begin
+    yaml = YAML.load_file("#{RAILS_ROOT}/#{CONFIG_PATH}")
+  rescue
+    logger.warn "Can't load #{CONFIG_PATH} file."
+    logger.warn "  #{$!}"
+    logger.warn "Put the following file:"
+    logger.warn sample
+    return
+  end
+
+  conf = yaml[RAILS_ENV]
+  unless conf
+    logger.warn "#{CONFIG_PATH} doesn't include setting for current environment (#{RAILS_ENV})."
+    logger.warn "Disabling Treasure Data logger."
+    return
+  end
+
+  begin
+    return Config.new(conf)
+  rescue
+    logger.warn "#{CONFIG_PATH}: #{$!}."
+    logger.warn "Disabling Treasure Data logger."
+    return
+  end
+end
+
+class Middleware
+  def initialize(app, options={})
+    @app = app
+  end
+
+  def call(env)
+    r = @app.call(env)
+
+    if m = env['treasure_data.log_method']
+      m.call(env)
+    end
+
+    r
+  end
+
+  def self.set_log_method(env, method)
+    env['treasure_data.log_method'] = method
+  end
+end
+
+module ControllerLogger
+  def self.included(mod)
+    mod.extend(ModuleMethods)
+  end
+
+  class ActionLogger
+    PARAM_KEY = if defined? Rails
+        if Rails.respond_to?(:version) && Rails.version =~ /^3/
+          # Rails 3
+          'action_dispatch.request.path_parameters'
+        else
+          # Rails 2
+          'action_controller.request.path_parameters'
+        end
+      else
+        # Rack default
+        'rack.routing_args'
+      end
+
+    def initialize(method, tag, options)
+      @method = method
+      @tag = tag
+
+      @only = nil
+      @except = nil
+      @extra = nil
+      @static = {}
+
+      if o = options[:only_params]
+        @only = case o
+          when Array
+            o
+          else
+            [o]
+          end.map {|e| e.to_s }
+      end
+
+      if o = options[:except_params]
+        @except = case o
+          when Array
+            o
+          else
+            [o]
+          end.map {|e| e.to_s }
+      end
+
+      if o = options[:extra]
+        @extra = case o
+          when Hash
+            m = {}
+            o.each_pair {|k,v| m[k.to_s] = v.to_s }
+            m
+          when Array
+            o.map {|e|
+              case e
+              when Hash
+                m = {}
+                e.each_pair {|k,v| m[k.to_s] = v.to_s }
+                m
+              else
+                {e.to_s => e.to_s}
+              end
+            }.inject({}) {|r,e| r.merge!(e) }
+          else
+            {o.to_s => o.to_s}
+          end
+      end
+
+      if o = options[:static]
+        o.each_pair {|k,v| @static[k] = v }
+      end
+    end
+
+    def call(env)
+      m = env[PARAM_KEY].dup || {}
+
+      if @only
+        m.reject! {|k,v| !@only.include?(k) }
+      end
+      if @except
+        m.reject! {|k,v| @except.include?(k) }
+      end
+      if @extra
+        @extra.each_pair {|k,v| m[v] = env[k] }
+      end
+
+      m.merge!(@static)
+
+      ::TreasureData.log(@tag, m)
+    end
+  end
+
+  module ModuleMethods
+    def add_td_tracer(method, tag, options={})
+      al = ActionLogger.new(method, tag, options)
+      module_eval <<-EOF
+        def #{method}_td_action_tracer_(*args, &block)
+          ::TreasureData::Logger::RailsAgent::Middleware.set_log_method(request.env, method(:#{method}_td_action_trace_))
+          #{method}_td_action_tracer_orig_(*args, &block)
+        end
+      EOF
+      module_eval do
+        define_method(:"#{method}_td_action_trace_", &al.method(:call))
+      end
+      alias_method "#{method}_td_action_tracer_orig_", method
+      alias_method method, "#{method}_td_action_tracer_"
+    end
+  end
+end
+
+end
+end
+
+end
+
+if defined? Rails
+  if Rails.respond_to?(:version) && Rails.version =~ /^3/
+    module TreasureData
+      class Railtie < Rails::Railtie
+        initializer "treasure_data_agent.start_plugin" do |app|
+          TreasureData::Logger::RailsAgent.init(app.config)
+        end
+      end
+    end
+  else
+    TreasureData::Logger::RailsAgent.init(Rails.configuration)
+  end
+end
+

metadata

@@ -0,0 +1,119 @@
+--- !ruby/object:Gem::Specification 
+name: td-logger
+version: !ruby/object:Gem::Version 
+  hash: 27
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Sadayuki Furuhashi
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-08-21 00:00:00 +09:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: msgpack
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 7
+        segments: 
+        - 0
+        - 4
+        - 4
+        version: 0.4.4
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: td
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 9
+        segments: 
+        - 0
+        - 7
+        - 5
+        version: 0.7.5
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: fluent-logger
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 19
+        segments: 
+        - 0
+        - 3
+        - 0
+        version: 0.3.0
+  type: :runtime
+  version_requirements: *id003
+description: 
+email: 
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- ChangeLog
+- README.rdoc
+files: 
+- lib/td/logger.rb
+- lib/td/logger/tdlog.rb
+- lib/td/logger/version.rb
+- ChangeLog
+- README.rdoc
+has_rdoc: true
+homepage: 
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Treasure Data command line tool
+test_files: []
+