td-logger 0.3.24 → 0.3.25

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    NzExZTFkZTM1MDlkZDQzYjU3YzI1NGU2NTQwYTA1NzJhMGM2MTYyZg==
-  data.tar.gz: !binary |-
-    MWM3YWYyMmVmNzQyMmIzMTU4YjUyNWFlYWJlNTdlMTFiN2I3OGRhZQ==
+SHA1:
+  metadata.gz: 95a36a6c7cb7b469128717a154baa2a4c4ea556a
+  data.tar.gz: 91edda08bbf14f498394ae9ddf8154b7be25b3ee
 SHA512:
-  metadata.gz: !binary |-
-    MTBiMTQ2ZDEwMTJkY2ZlN2YyMjc0OTExZDYyNWY3NmZlZTMxNTBiYjhjM2Ni
-    MzAzZjM1ZmE5ZGY4ZTcxNTkxOTQwMTJlOWU4YzIyZThjOWExMzQxOTJiYmQ1
-    NzdiZjk4ZDM4MWYwZTE3NTA3Y2QwMWQ3ZjM5OGM5NjQyNzc4NDc=
-  data.tar.gz: !binary |-
-    NzQ4MGUwYWFjOGQxOTgzYzczMThlNTA3NGI1MWRiNWExMThkNDdiZTNiNjlh
-    NzNiYWIyOWFlN2Q4YTU5NDc1Njg1MzBmOTBhMGZjMTI0NzhmY2M3YWVjZmNm
-    ZTYzNWVjM2Y4NDkxZDk2NWZhNDg3MGM1MWEzZDc4OTdhMDM3YjE=
+  metadata.gz: d2344c5248eaaee17ad48f126b64083bc984fb347c2a0544480f90ec74251aa4fedb5a5fe5b5a724bbf2915df8c6898cb2753421552ace7431cbafdb30e3d85a
+  data.tar.gz: a4fa052a3af4160b1c531648a3b7ac63bb4992bdde711966c1d396ff5ff823276f0a59ae9c5b68fdcaef8141873b64227b5132964aa94a649342dca09e387182

data/.coveralls.yml

@@ -0,0 +1 @@
+service_name: travis-ci

data/.gitignore

@@ -1,2 +1,4 @@
 Gemfile.lock
 pkg/*
+spec/tmp/
+coverage/

data/.travis.yml

@@ -3,10 +3,14 @@ language: ruby
 rvm:
   - 1.9.3
   - 2.0.0
-  - 2.1.0
-  - 2.1.1
+  - 2.1
+  - 2.2
+  - 2.3.0
   - ruby-head
   - rbx
+  - jruby-1.7.20
+
+sudo: false
 
 branches:
   only:
@@ -21,3 +25,7 @@ matrix:
   allow_failures:
     - rvm: ruby-head
     - rvm: rbx
+    - rvm: jruby-1.7.20
+
+notifications:
+  webhooks: http://td-beda.herokuapp.com/travisci_callback

data/ChangeLog

@@ -1,3 +1,9 @@
+== 2016-01-19 version 0.3.25
+
+* can use unique_key when post td
+* can set the timing of buffering
+* can configurate buffering valiables
+
 == 2014-11-18 version 0.3.24
 
 * Pass all option hash's arguments to the td-client library to support

data/README.rdoc

@@ -1,11 +1,14 @@
 = Treasure Data logging library for Rails
 
+{<img src="https://travis-ci.org/treasure-data/td-logger-ruby.svg?branch=master" alt="Build Status" />}[https://travis-ci.org/treasure-data/td-logger-ruby]
+{<img src="https://coveralls.io/repos/treasure-data/td-logger-ruby/badge.svg?branch=master" alt="Coverage Status" />}[https://coveralls.io/r/treasure-data/td-logger-ruby?branch=master]
+
 == About
 
 This gem is a *logging* *library* *for* *Treasure* *Data*. The events logged by this module will be uploaded into the cloud. There're two ways to upload:
 
 * *direct* *upload* *from* *applications*: app -> cloud. This option is easier to configure and setup, but it requires extra memory in the application processes. The data is logged directly in the Treasure Data Cloud through the REST APIs by leveraging the {Treasure Data Ruby Client (td-client-ruby)}[https://github.com/treasure-data/td-client-ruby] library.
-* *in-direct* *upload* *from* *td-agent*: app -> td-agent -> cloud. This option requires extra steps to install the daemons into your cluster, but lowers the logging impact on your applications. The data is sent to a td-agent running either locally or remotely by leveraging the {Fluentd Ruby Logger (fluent-logger-ruby)}[https://github.com/fluent/fluent-logger-ruby/] library.
+* *indirect* *upload* *from* *td-agent*: app -> td-agent -> cloud. This option requires extra steps to install the daemons into your cluster, but lowers the logging impact on your applications. The data is sent to a td-agent running either locally or remotely by leveraging the {Fluentd Ruby Logger (fluent-logger-ruby)}[https://github.com/fluent/fluent-logger-ruby/] library.
 
 The actual upload method can be selected in the config file as described below.
 
@@ -41,7 +44,7 @@ This configuration enables the application to upload the events directly from th
 
 If you would wish to keep your API key as an environment variable, you can use this setting:
 
-      apikey: <% ENV['TD_API_KEY'] %>
+      apikey: <%= ENV['TD_API_KEY'] %>
 
 Alternatively, one can initialize the Ruby Logger module for direct upload straight from within the application:
 

data/VERSION

@@ -1 +1 @@
-0.3.24
+0.3.25

data/appveyor.yml

@@ -0,0 +1,17 @@
+---
+version: "{build}"
+clone_depth: 10
+install:
+  - SET PATH=C:\Ruby%ruby_version%\bin;%PATH%
+  - ruby --version
+  - gem --version
+  - gem install bundler --quiet --no-ri --no-rdoc
+  - bundler --version
+  - bundle install
+build_script:
+  - bundle exec rake build
+test_script:
+  - bundle exec rake spec
+environment:
+  matrix:
+    - ruby_version: "22"

data/lib/td/logger/td_logger.rb

@@ -46,11 +46,11 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
     @map = {}  # (db,table) => buffer:String
     @queue = []
 
-    @chunk_limit = 8 * 1024 * 1024
+    @chunk_limit = options[:chunk_limit] || 8 * 1024 * 1024
     @queue_limit = 50
 
-    @flush_interval = 2
-    @max_flush_interval = 300
+    @flush_interval = options[:flush_interval] || 2
+    @max_flush_interval = options[:max_flush_interval] || 300
     @retry_wait = 1.0
     @retry_limit = 12
 
@@ -62,6 +62,8 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
     # Unicorn and Passenger.
     @upload_thread = nil
 
+    @use_unique_key = options[:use_unique_key]
+
     # The calling order of finalizer registered by define_finalizer is indeterminate,
     # so we should use at_exit instead for memory safety.
     at_exit { close }
@@ -211,6 +213,8 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
   end
 
   def add(db, table, msg)
+    # NOTE: TreasureData::API is defined at td-client-ruby gem
+    #       https://github.com/treasure-data/td-client-ruby/blob/master/lib/td/client/api.rb
     begin
       TreasureData::API.validate_database_name(db)
     rescue
@@ -329,12 +333,14 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
   end
 
   def upload(db, table, data)
+    unique_key = @use_unique_key ? generate_unique_key : nil
+
     begin
       stream = StringIO.new(data)
 
-      @logger.debug "Uploading event logs to #{db}.#{table} table on Treasure Data (#{stream.size} bytes)"
+      @logger.info "Uploading event logs to #{db}.#{table} table on Treasure Data (#{stream.size} bytes)"
 
-      @client.import(db, table, "msgpack.gz", stream, stream.size)
+      @client.import(db, table, "msgpack.gz", stream, stream.size, unique_key)
     rescue TreasureData::NotFoundError
       unless @auto_create_table
         raise $!
@@ -350,6 +356,15 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
     end
   end
 
+  # NOTE fluentd unique_id and fluent-plugin-td unique_str in reference.
+  #      https://github.com/fluent/fluentd/blob/v0.12.15/lib/fluent/plugin/buf_memory.rb#L22
+  #      https://github.com/treasure-data/fluent-plugin-td/blob/v0.10.27/lib/fluent/plugin/out_tdlog.rb#L225
+  def generate_unique_key(now = Time.now)
+    u1 = ((now.to_i*1000*1000+now.usec) << 12 | rand(0xfff))
+    uid = [u1 >> 32, u1 & 0xffffffff, rand(0xffffffff), rand(0xffffffff)].pack('NNNN')
+    uid.unpack('C*').map { |x| "%02x" % x }.join
+  end
+
   require 'thread'  # ConditionVariable
   if ConditionVariable.new.method(:wait).arity == 1
     # "WARNING: Running on Ruby 1.8. Ruby 1.9 is recommended."

data/lib/td/logger/version.rb

@@ -1,7 +1,7 @@
 module TreasureData
 module Logger
 
-VERSION = '0.3.24'
+VERSION = '0.3.25'
 
 end
 end

data/spec/spec_helper.rb

@@ -1,6 +1,24 @@
+if defined?(:RUBY_ENGINE) && RUBY_ENGINE == 'ruby'
+  # SimpleCov officially supports MRI 1.9+ only for now
+  # https://github.com/colszowka/simplecov#ruby-version-compatibility
+
+  require 'simplecov'
+  unless ENV['APPVEYOR']
+    require 'coveralls'
+
+    SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter[
+      SimpleCov::Formatter::HTMLFormatter,
+      Coveralls::SimpleCov::Formatter
+    ]
+  end
+  SimpleCov.start("test_frameworks")
+end
+
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 
+REPO_ROOT = Pathname.new(File.expand_path("../", __FILE__))
+
 require 'td-logger'
 
 RSpec.configure do |config|

data/spec/td/logger/agent/config_spec.rb

@@ -0,0 +1,260 @@
+require 'spec_helper'
+require 'fileutils'
+require 'logger'
+
+TMP_DIR = REPO_ROOT.join("tmp")
+FileUtils.rm_rf(TMP_DIR)
+
+class Rails
+  def self.configuration
+    self.new
+  end
+
+  def self.logger
+    Logger.new(STDOUT)
+  end
+
+  def self.root
+    TMP_DIR
+  end
+
+  def self.env
+    "test"
+  end
+
+  def middleware
+    self
+  end
+
+  def use(mod)
+  end
+end
+
+class ActionController
+  class Base
+  end
+end
+
+class ActiveSupport
+  class TimeWithZone
+  end
+end
+
+require 'td/logger/agent/rails' # NOTE: should be require after Rails goodies mocked
+
+
+describe TreasureData::Logger::Agent::Rails::Config do
+  before(:each) do
+    FileUtils.rm_rf(TMP_DIR)
+    FileUtils.mkdir_p(TMP_DIR)
+    ENV.delete('TREASURE_DATA_API_KEY')
+    ENV.delete('TREASURE_DATA_DB')
+  end
+
+  context 'init' do
+    it 'load_env' do
+      ENV['TREASURE_DATA_API_KEY'] = 'test1'
+      ENV['TREASURE_DATA_DB'] = 'db1'
+      c = TreasureData::Logger::Agent::Rails::Config.init
+      expect(c.disabled).to eq(false)
+      expect(c.agent_mode?).to eq(false)
+      expect(c.apikey).to eq('test1')
+      expect(c.database).to eq('db1')
+      expect(c.auto_create_table).to eq(true)
+    end
+
+    it 'load_file' do
+      FileUtils.mkdir_p("#{TMP_DIR}/config")
+      File.open("#{TMP_DIR}/config/treasure_data.yml", "w") {|f|
+        f.write <<EOF
+test:
+  apikey: test2
+  database: db2
+  debug_mode: true
+EOF
+      }
+      c = TreasureData::Logger::Agent::Rails::Config.init
+      expect(c.disabled).to eq(false)
+      expect(c.agent_mode?).to eq(false)
+      expect(c.apikey).to eq('test2')
+      expect(c.database).to eq('db2')
+      expect(c.auto_create_table).to eq(true)
+      expect(c.debug_mode).to eq(true)
+      expect(c.access_log_enabled?).to eq false
+    end
+
+    it 'load_file without test' do
+      FileUtils.mkdir_p("#{TMP_DIR}/config")
+      File.open("#{TMP_DIR}/config/treasure_data.yml", "w") {|f|
+        f.write <<EOF
+development:
+  apikey: test2
+  database: db2
+  debug_mode: true
+EOF
+      }
+      c = TreasureData::Logger::Agent::Rails::Config.init
+      expect(c.disabled).to eq(true)
+      expect(c.access_log_enabled?).to eq false
+    end
+
+    it 'prefer file than env' do
+      ENV['TREASURE_DATA_API_KEY'] = 'test3'
+      ENV['TREASURE_DATA_DB'] = 'db3'
+
+      FileUtils.mkdir_p("#{TMP_DIR}/config")
+      File.open("#{TMP_DIR}/config/treasure_data.yml", "w") {|f|
+        f.write <<EOF
+test:
+  apikey: test4
+  database: db4
+  auto_create_table: false
+EOF
+      }
+      c = TreasureData::Logger::Agent::Rails::Config.init
+      expect(c.disabled).to eq(false)
+      expect(c.agent_mode?).to eq(false)
+      expect(c.apikey).to eq('test4')
+      expect(c.database).to eq('db4')
+      expect(c.auto_create_table).to eq(false)
+      expect(c.debug_mode).to eq(false)
+      expect(c.access_log_enabled?).to eq false
+    end
+
+    it 'agent mode' do
+      FileUtils.mkdir_p("#{TMP_DIR}/config")
+      File.open("#{TMP_DIR}/config/treasure_data.yml", "w") {|f|
+        f.write <<EOF
+test:
+  agent: localhost
+  tag: td.db5
+EOF
+      }
+      c = TreasureData::Logger::Agent::Rails::Config.init
+      expect(c.disabled).to eq(false)
+      expect(c.agent_mode?).to eq(true)
+      expect(c.tag).to eq('td.db5')
+      expect(c.agent_host).to eq('localhost')
+      expect(c.agent_port).to eq(24224)
+      expect(c.access_log_enabled?).to eq false
+    end
+
+    it 'test mode' do
+      FileUtils.mkdir_p("#{TMP_DIR}/config")
+      File.open("#{TMP_DIR}/config/treasure_data.yml", "w") {|f|
+        f.write <<EOF
+test:
+  agent: localhost
+  tag: td.db5
+  test_mode: true
+EOF
+      }
+      c = TreasureData::Logger::Agent::Rails::Config.init
+      expect(c.disabled).to eq(false)
+      expect(c.test_mode?).to eq(true)
+      expect(c.access_log_enabled?).to eq false
+    end
+
+    it "access_log_enabled" do
+      FileUtils.mkdir_p("#{TMP_DIR}/config")
+      File.open("#{TMP_DIR}/config/treasure_data.yml", "w") {|f|
+        f.write <<EOF
+test:
+  agent: localhost
+  tag: foo.bar
+  access_log_table: log
+EOF
+      }
+      c = TreasureData::Logger::Agent::Rails::Config.init
+      expect(c.disabled).to eq(false)
+      expect(c.access_log_enabled?).to eq true
+    end
+
+    describe "load_file_ey" do
+      context "config_path does not exists" do
+        it "load from CONFIG_PATH_EY_DEPLOY" do
+          FileUtils.mkdir_p("#{TMP_DIR}/config")
+          File.unlink("#{TMP_DIR}/config/treasure_data.yml") rescue nil
+          File.open(TreasureData::Logger::Agent::Rails::CONFIG_PATH_EY_DEPLOY, "w") {|f|
+            f.write <<EOF
+td:
+  TREASURE_DATA_API_KEY: foo
+EOF
+          }
+          c = TreasureData::Logger::Agent::Rails::Config.init
+          expect(c.disabled).to eq(false)
+        end
+
+        it "disabled true if td.TREASURE_DATA_API_KEY missing" do
+          FileUtils.mkdir_p("#{TMP_DIR}/config")
+          File.unlink("#{TMP_DIR}/config/treasure_data.yml") rescue nil
+          File.open(TreasureData::Logger::Agent::Rails::CONFIG_PATH_EY_DEPLOY, "w") {|f|
+            f.write <<EOF
+EOF
+          }
+          c = TreasureData::Logger::Agent::Rails::Config.init
+          expect(c.disabled).to eq(true)
+        end
+      end
+
+      context "config_path and CONFIG_PATH_EY_DEPLOY does not exists" do
+        it "load from CONFIG_PATH_EY_LOCAL" do
+          FileUtils.mkdir_p("#{TMP_DIR}/config")
+          File.unlink("#{TMP_DIR}/config/treasure_data.yml") rescue nil
+          File.unlink(TreasureData::Logger::Agent::Rails::CONFIG_PATH_EY_DEPLOY) rescue nil
+          File.open(TreasureData::Logger::Agent::Rails::CONFIG_PATH_EY_LOCAL, "w") {|f|
+            f.write <<EOF
+td:
+  TREASURE_DATA_API_KEY: foo
+EOF
+          }
+          c = TreasureData::Logger::Agent::Rails::Config.init
+          expect(c.disabled).to eq(false)
+        end
+
+        it "disabled true if td.TREASURE_DATA_API_KEY missing" do
+          FileUtils.mkdir_p("#{TMP_DIR}/config")
+          File.unlink("#{TMP_DIR}/config/treasure_data.yml") rescue nil
+          File.open(TreasureData::Logger::Agent::Rails::CONFIG_PATH_EY_LOCAL, "w") {|f|
+            f.write <<EOF
+EOF
+          }
+          c = TreasureData::Logger::Agent::Rails::Config.init
+          expect(c.disabled).to eq(true)
+        end
+      end
+
+      context "config_path, CONFIG_PATH_EY_DEPLOY, CONFIG_PATH_EY_LOCAL are all exists" do
+        it "load from CONFIG_PATH_EY_LOCAL" do
+          FileUtils.mkdir_p("#{TMP_DIR}/config")
+          File.open("#{TMP_DIR}/config/treasure_data.yml", "w") {|f|
+            f.write <<EOF
+test:
+  agent: localhost
+  tag: foo.bar
+  test_mode: false
+EOF
+          }
+          File.open(TreasureData::Logger::Agent::Rails::CONFIG_PATH_EY_DEPLOY, "w") {|f|
+            f.write <<EOF
+test:
+  agent: localhost
+  tag: foo.bar
+  test_mode: true
+EOF
+          }
+          File.open(TreasureData::Logger::Agent::Rails::CONFIG_PATH_EY_LOCAL, "w") {|f|
+            f.write <<EOF
+test:
+  agent: localhost
+  tag: foo.bar
+  test_mode: true
+EOF
+          }
+          c = TreasureData::Logger::Agent::Rails::Config.init
+          expect(c.test_mode).to eq(false)
+        end
+      end
+    end
+  end
+end