td-logger 0.3.7 → 0.3.8

data/ChangeLog

@@ -1,4 +1,10 @@
 
+== 2012-02-24 version 0.3.8
+
+* Supports for EngineYard Add-on users
+* Limit length of upload queue upto 50
+
+
 == 2011-11-11 version 0.3.7
 
 * Use bundler for packaging

data/README.rdoc

@@ -1,5 +1,14 @@
 = Treasure Data logging library for Rails
 
+== About
+
+This gem is a *logging* *library* *for* *Treasure* *Data*. The events logged by this module will be uploaded into the cloud. There're two ways to upload:
+
+* 1) direct upload from applications: app -> cloud (used by default)
+* 2) in-direct upload from td-agent: app -> td-agent -> cloud
+
+(1) is more easy to configure and extra setup, but it requires extra memory in the application processes. (2) requires the extra installation of the daemons into your cluster, but lowers the logging impact to your applications. You can choose whether to use, by config file described laster.
+
 == Getting Started
 
 Add the following line to your Gemfile:
@@ -11,18 +20,17 @@ edit +environment.rb+ and add to the initalizer block:
 
     config.gem "td-logger"
 
-And then add +config/treasure_data.yml+ file as following:
+And then add +config/treasure_data.yml+ file as followings. This is a template, which is doing the direct upload from the application processes.
 
-    # logging to Treasure Data directly
     development:
       apikey: "YOUR_API_KEY"
-      database: myapp
+      database: rails_development
       debug_mode: true  # enable debug mode
     
-    # logging via td-agent (fluent)
     production:
-      agent: "localhost:24224"
-      tag: td.myapp
+      apikey: "YOUR_API_KEY"
+      database: rails_production
+      debug_mode: false
     
     # disable logging
     test:
@@ -56,9 +64,38 @@ In rails application, you can use 'TD.event.attribute' to set static attribute t
       end
     end
 
+== Configuration file
+
+=== Direct Upload from Application Buffer
+
+This configuration enables the application to upload the events, directly from the application processes (app -> cloud).
+
+    production:
+      apikey: "YOUR_API_KEY"
+      database: rails_production
+      debug_mode: false
+
+=== In-Direct Upload from td-agent
+
+When 'agent' and 'tag' options are specified, the events are logged into the local td-agent daemon. Later on, the daemon uploads the event periodically to the cloud (app -> td-agent -> cloud).
+
+This requires the additional setup to your infrastructure, but lowers the memory and performance impact to your application processes. For the installation process, please look at our {knowledge base}[http://help.treasure-data.com/kb/installing-td-agent-daemon] page.
+
+NOTE: This configuration is not supported on PaaP platforms.
+
+    production:
+      agent: "localhost:24224"
+      tag: td.myapp
+      debug_mode: false
+
+== Further Readings
+
+If you have any problem, please ask us from the support site.
+
+* {Support Site}[http://help.treasure-data.com/]
+* {Knowledge Base}[http://help.treasure-data.com/kb]
 
 == Copyright
 
 Copyright:: Copyright (c) 2011 Treasure Data Inc.
 License::   Apache License, Version 2.0
-

data/VERSION

@@ -1 +1 @@
-0.3.7
+0.3.8

data/lib/td/logger/agent/rails/config.rb

@@ -50,7 +50,13 @@ module Agent::Rails
       if File.exist?("#{::Rails.root}/#{CONFIG_PATH}")
         load_file(logger)
       else
-        load_env(logger)
+        if File.exist?("#{::Rails.root}/#{CONFIG_PATH_EY_DEPLOY}")
+          load_file_ey(logger, "#{::Rails.root}/#{CONFIG_PATH_EY_DEPLOY}")
+        elsif File.exist?("#{::Rails.root}/#{CONFIG_PATH_EY_LOCAL}")
+          load_file_ey(logger, "#{::Rails.root}/#{CONFIG_PATH_EY_LOCAL}")
+        else
+          load_env(logger)
+        end
       end
     end
 
@@ -84,6 +90,35 @@ module Agent::Rails
       end
     end
 
+    def self.load_file_ey(logger, path)
+      require 'yaml'
+      require 'erb'
+
+      begin
+        src = File.read(path)
+        yaml = ERB.new(src).result
+        env_conf = YAML.load(yaml)
+      rescue
+        logger.warn "WARNING: Can't load #{path} file: #{$!}"
+        logger.warn "WARNING: Disabling Treasure Data event logger."
+        return nil
+      end
+
+      apikey = env_conf['td']['TREASURE_DATA_API_KEY'] if env_conf.is_a?(Hash)
+      unless apikey
+        logger.warn "WARNING: #{path} does not exist."
+        logger.warn "WARNING: Disabling Treasure Data event logger."
+        return nil
+      end
+
+      return Config.new({
+        'apikey' => apikey,
+        'database' => ENV['TREASURE_DATA_DB'] || "rails_#{::Rails.env}",
+        'access_log_table' => ENV['TREASURE_DATA_ACCESS_LOG_TABLE'],
+        'auto_create_table' => true
+      })
+    end
+
     def self.load_env(logger)
       apikey = ENV['TREASURE_DATA_API_KEY'] || ENV['TD_API_KEY']
 

data/lib/td/logger/agent/rails.rb

@@ -3,6 +3,8 @@ module Logger
 module Agent::Rails
 
   CONFIG_PATH = 'config/treasure_data.yml'
+  CONFIG_PATH_EY_LOCAL = 'config/ey_services_config_local.yml'
+  CONFIG_PATH_EY_DEPLOY = 'config/ey_services_config_deploy.yml'
 
   require 'td/logger/agent/rack'
   require 'td/logger/agent/rails/config'

data/lib/td/logger/td_logger.rb

@@ -64,6 +64,8 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
     @queue = []
 
     @chunk_limit = 8*1024*1024
+    @queue_limit = 50
+
     @flush_interval = 10
     @max_flush_interval = 300
     @retry_wait = 1.0
@@ -179,6 +181,11 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
     key = [db, table]
 
     @mutex.synchronize do
+      if @queue.length > @queue_limit
+        @logger.error("TreasureDataLogger: queue length exceeds limit. can't add new event log: #{msg.inspect}")
+        return false
+      end
+
       buffer = (@map[key] ||= '')
 
       buffer << data

data/lib/td/logger/version.rb

@@ -1,7 +1,7 @@
 module TreasureData
 module Logger
 
-VERSION = '0.3.7'
+VERSION = '0.3.8'
 
 end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: td-logger
 version: !ruby/object:Gem::Version
-  version: 0.3.7
+  version: 0.3.8
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-11-10 00:00:00.000000000Z
+date: 2012-02-24 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
-  requirement: &70274289643260 !ruby/object:Gem::Requirement
+  requirement: &20688200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 0.4.4
   type: :runtime
   prerelease: false
-  version_requirements: *70274289643260
+  version_requirements: *20688200
 - !ruby/object:Gem::Dependency
   name: td-client
-  requirement: &70274289642660 !ruby/object:Gem::Requirement
+  requirement: &20686380 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 0.8.4
   type: :runtime
   prerelease: false
-  version_requirements: *70274289642660
+  version_requirements: *20686380
 - !ruby/object:Gem::Dependency
   name: fluent-logger
-  requirement: &70274289642080 !ruby/object:Gem::Requirement
+  requirement: &20685280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 0.4.1
   type: :runtime
   prerelease: false
-  version_requirements: *70274289642080
+  version_requirements: *20685280
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70274289641520 !ruby/object:Gem::Requirement
+  requirement: &20684180 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: 0.9.2
   type: :development
   prerelease: false
-  version_requirements: *70274289641520
+  version_requirements: *20684180
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70274289640940 !ruby/object:Gem::Requirement
+  requirement: &20630960 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,7 +65,7 @@ dependencies:
         version: 2.7.0
   type: :development
   prerelease: false
-  version_requirements: *70274289640940
+  version_requirements: *20630960
 description: Treasure Data logging library for Rails
 email: 
 executables: []
@@ -105,26 +105,16 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -1557178488144164455
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -1557178488144164455
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.10
 signing_key: 
 specification_version: 3
 summary: Treasure Data logging library for Rails
-test_files:
-- spec/event_spec.rb
-- spec/rails_config_spec.rb
-- spec/spec_helper.rb
-- spec/td_logger_spec.rb
+test_files: []