td-logger 0.2.8 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/ChangeLog CHANGED
@@ -1,4 +1,13 @@
1
1
 
2
+ == 2011-11-05 version 0.3.0
3
+
4
+ * Added new API: TD.event
5
+ * Added event presets: TD.event.register, login, pay
6
+ * Added feature to add set attributes to all events: TD.event.attribute
7
+ * Stop to log access logs
8
+ * Use NullLogger when logger is disabled
9
+
10
+
2
11
  == 2011-10-17 version 0.2.8
3
12
 
4
13
  * Access logger checks to_msgpack for each values and nested values
data/README.rdoc CHANGED
@@ -29,67 +29,32 @@ And then add +config/treasure_data.yml+ file as following:
29
29
  # disable logging
30
30
  test:
31
31
 
32
- === Logging access logs
32
+ == Logging events
33
33
 
34
- Following information will be logged automatically:
34
+ You can log anytime using 'TD.event.post' method:
35
35
 
36
- * controller name ('controller' column)
37
- * action name ('action' column)
38
- * remote host ip address ('remote_addr' column)
39
- * http status code ('status' column)
40
- * http referer ('refer' column)
41
-
42
- To add information to the log, call 'td_access_log' method in a Controller:
43
-
44
- class YourController < ApplicationController
45
- def myaction
46
- td_access_log[:column] = "value"
47
- end
48
- end
49
-
50
- Or call 'TreasureData.access_log' method outside of a Controller:
51
-
52
- class MyHelperModule
36
+ class MyClass
53
37
  def mymethod
54
- TreasureData.access_log[:column] = "value"
38
+ TD.event.post('table_name', {:foo=>:bar})
55
39
  end
56
40
  end
57
41
 
42
+ == Rails extension
58
43
 
59
- === Logging model changes
60
-
61
- Add 'td_enable_model_tracer' line to your Model class:
62
-
63
- class MyModel < ActiveRecord::Base
64
- td_enable_model_tracer :mytable
65
- end
66
-
67
- It logs all columns when the model is changed. Use :only option to limit columns to log:
68
-
69
- class MyModel < ActiveRecord::Base
70
- td_enable_model_tracer :mytable, :only => [:id, :mycol]
71
- end
72
-
73
- Alternatively, use :except option to except columns to log:
74
-
75
- class MyModel < ActiveRecord::Base
76
- td_enable_model_tracer :mytable, :except => [:created_at]
77
- end
78
-
79
- Add 'static' option to add constant key-value pairs:
44
+ In rails application, you can use 'TD.event.attribute' to set static attribute to all events:
80
45
 
81
- class MyModel < ActiveRecord::Base
82
- td_enable_model_tracer :mytable, :except => [:created_at], :static => {'model'=>'my'}
46
+ class ApplicationController
47
+ def authenticate
48
+ # set 'uid' attribute to succeeding event logs
49
+ TD.event.attribute[:uid] = 198
50
+ end
83
51
  end
84
52
 
85
-
86
- === Logging other logs
87
-
88
- You can log anytime using 'TreasureData.log' method:
89
-
90
- class MyClass
53
+ class MyController < ApplicationController
91
54
  def mymethod
92
- TreasureData.log('mytable', {:col1=>"val1", :col2=>"val2"}
55
+ authenticate()
56
+ # this event includes 'uid' attribute
57
+ TD.event.post('table_name', {:foo=>:bar})
93
58
  end
94
59
  end
95
60
 
@@ -0,0 +1,39 @@
1
+ module TreasureData
2
+ module Logger
3
+ module Agent::Rack
4
+
5
+ class Hook
6
+ @@before = []
7
+ @@after = []
8
+
9
+ def self.before(&block)
10
+ @@before << block
11
+ end
12
+
13
+ def self.after(&block)
14
+ @@after << block
15
+ end
16
+
17
+ def initialize(app, options={})
18
+ @app = app
19
+ end
20
+
21
+ def call(env)
22
+ @@before.each {|m|
23
+ m.call(env)
24
+ }
25
+
26
+ result = @app.call(env)
27
+
28
+ @@after.each {|m|
29
+ m.call(env, result)
30
+ }
31
+
32
+ result
33
+ end
34
+ end
35
+
36
+ end
37
+ end
38
+ end
39
+
@@ -0,0 +1,105 @@
1
+ module TreasureData
2
+ module Logger
3
+ module Agent::Rails
4
+
5
+ class Config
6
+ def initialize(conf)
7
+ if agent = conf['agent']
8
+ host, port = agent.split(':',2)
9
+ port = (port || 24224).to_i
10
+ @agent_host = host
11
+ @agent_port = port
12
+
13
+ @tag = conf['tag']
14
+ @tag ||= conf['database']
15
+ raise "'tag' nor 'database' options are not set" unless @tag
16
+
17
+ else
18
+ @apikey = conf['apikey']
19
+ raise "'apikey' option is not set" unless @apikey
20
+
21
+ @database = conf['database']
22
+ raise "'database' option is not set" unless @database
23
+
24
+ if conf.has_key?('auto_create_table')
25
+ @auto_create_table = !!conf['auto_create_table']
26
+ else
27
+ @auto_create_table = true
28
+ end
29
+ end
30
+
31
+ @access_log_table = conf['access_log_table']
32
+ end
33
+
34
+ attr_reader :agent_host, :agent_port, :tag
35
+ attr_reader :apikey, :database, :auto_create_table
36
+ attr_reader :access_log_table
37
+
38
+ def agent_mode?
39
+ @agent_host != nil
40
+ end
41
+
42
+ def access_log_enabled?
43
+ !@access_log_table.nil? && !@access_log_table.empty?
44
+ end
45
+
46
+ def self.init
47
+ logger = ::Rails.logger || ::Logger.new(STDERR)
48
+ if File.exist?("#{::Rails.root}/#{CONFIG_PATH}")
49
+ load_file(logger)
50
+ else
51
+ load_env(logger)
52
+ end
53
+ end
54
+
55
+ def self.load_file(logger)
56
+ require 'yaml'
57
+ require 'erb'
58
+
59
+ begin
60
+ src = File.read("#{::Rails.root}/#{CONFIG_PATH}")
61
+ yaml = ERB.new(src).result
62
+ env_conf = YAML.load(yaml)
63
+ rescue
64
+ logger.warn "Can't load #{CONFIG_PATH} file: #{$!}"
65
+ logger.warn "Disabling Treasure Data event logger."
66
+ return nil
67
+ end
68
+
69
+ conf = env_conf[::Rails.env]
70
+ unless conf
71
+ logger.warn "#{CONFIG_PATH} doesn't include setting for current environment (#{::Rails.env})."
72
+ logger.warn "Disabling Treasure Data event logger."
73
+ return nil
74
+ end
75
+
76
+ begin
77
+ return Config.new(conf)
78
+ rescue
79
+ logger.warn "#{CONFIG_PATH}: #{$!}."
80
+ logger.warn "Disabling Treasure Data event logger."
81
+ return nil
82
+ end
83
+ end
84
+
85
+ def self.load_env(logger)
86
+ apikey = ENV['TREASURE_DATA_API_KEY'] || ENV['TD_API_KEY']
87
+
88
+ unless apikey
89
+ logger.warn "#{CONFIG_PATH} does not exist."
90
+ logger.warn "Disabling Treasure Data event logger."
91
+ return nil
92
+ end
93
+
94
+ return Config.new({
95
+ 'apikey' => apikey,
96
+ 'database' => ENV['TREASURE_DATA_DB'] || "rails_#{::Rails.env}",
97
+ 'access_log_table' => ENV['TREASURE_DATA_ACCESS_LOG_TABLE'],
98
+ 'auto_create_table' => true
99
+ })
100
+ end
101
+ end
102
+
103
+ end
104
+ end
105
+ end
@@ -1,35 +1,33 @@
1
1
  module TreasureData
2
2
  module Logger
3
- module Agent
4
- module Rails
3
+ module Agent::Rails
4
+ module ControllerExtension
5
5
 
6
- def self.init_controller
7
- ActionController::Base.send(:include, ControllerExtension)
6
+ def self.init
7
+ ::ActionController::Base.send(:include, self)
8
8
  end
9
9
 
10
- module ControllerExtension
11
- if defined?(ActiveSupport::Concern)
12
- # Rails 2
13
- extend ActiveSupport::Concern
14
- else
15
- def self.included(mod)
16
- im = InstanceMethods
17
- cm = ClassMethods
18
- mod.class_eval do
19
- include im
20
- extend cm
21
- end
10
+ if defined?(ActiveSupport::Concern)
11
+ extend ActiveSupport::Concern
12
+ else
13
+ # Rails 2
14
+ def self.included(mod)
15
+ im = InstanceMethods
16
+ cm = ClassMethods
17
+ mod.class_eval do
18
+ include im
19
+ extend cm
22
20
  end
23
21
  end
22
+ end
24
23
 
25
- module InstanceMethods
26
- def td_access_log
27
- request.env['td.access_log'] ||= {}
28
- end
24
+ module InstanceMethods
25
+ def event
26
+ TreasureData::Logger.event
29
27
  end
28
+ end
30
29
 
31
- module ClassMethods
32
- end
30
+ module ClassMethods
33
31
  end
34
32
 
35
33
  end
@@ -1,93 +1,92 @@
1
1
  module TreasureData
2
2
  module Logger
3
- module Agent
4
- module Rails
3
+ module Agent::Rails
4
+ module ModelExtension
5
5
 
6
- def self.init_model
7
- unless defined?(::ActiveRecord)
8
- # disable model extension if Rails > 3 and
9
- # ActiveRecord is not loaded (other ORM is used)
10
- if ::Rails.respond_to?(:version) && ::Rails.version =~ /^3/
11
- return
12
- end
13
- require 'active_record'
6
+ def self.init
7
+ # disable model extension ActiveRecord is not loaded
8
+ # on Rails > 3 (other ORM is used)
9
+ if !defined?(::ActiveRecord) &&
10
+ ::Rails.respond_to?(:version) && ::Rails.version =~ /^3/
11
+ return
14
12
  end
15
- ::ActiveRecord::Base.send(:include, ModelExtension)
13
+ require 'active_record'
14
+ ::ActiveRecord::Base.send(:include, self)
16
15
  end
17
16
 
18
- module ModelExtension
19
- if defined?(ActiveSupport::Concern)
20
- # Rails 2
21
- extend ActiveSupport::Concern
22
- else
23
- def self.included(mod)
24
- im = InstanceMethods
25
- cm = ClassMethods
26
- mod.class_eval do
27
- include im
28
- extend cm
29
- end
17
+ if defined?(ActiveSupport::Concern)
18
+ extend ActiveSupport::Concern
19
+ else
20
+ # Rails 2
21
+ def self.included(mod)
22
+ im = InstanceMethods
23
+ cm = ClassMethods
24
+ mod.class_eval do
25
+ include im
26
+ extend cm
30
27
  end
31
28
  end
29
+ end
32
30
 
33
- module InstanceMethods
34
- end
31
+ module InstanceMethods
32
+ end
35
33
 
36
- module ClassMethods
37
- def td_enable_model_tracer(tag, options={})
38
- only = nil
39
- except = nil
40
- static = {}
34
+ module ClassMethods
35
+ def td_enable_model_tracer(tag, options={})
36
+ only = nil
37
+ except = nil
38
+ static = {}
41
39
 
42
- if o = options[:only]
43
- only = case o
40
+ if o = options[:only]
41
+ only = case o
42
+ when Array
43
+ o
44
+ else
45
+ [o]
46
+ end.map {|e| e.to_s }
47
+ end
48
+
49
+ if o = options[:except]
50
+ except = case o
44
51
  when Array
45
52
  o
46
53
  else
47
54
  [o]
48
55
  end.map {|e| e.to_s }
49
- end
50
-
51
- if o = options[:except]
52
- except = case o
53
- when Array
54
- o
55
- else
56
- [o]
57
- end.map {|e| e.to_s }
58
- end
56
+ end
59
57
 
60
- if o = options[:static]
61
- o.each_pair {|k,v|
62
- static[k.to_s] = v
63
- }
64
- end
58
+ if o = options[:static]
59
+ o.each_pair {|k,v|
60
+ static[k.to_s] = v
61
+ }
62
+ end
65
63
 
66
- if defined?(after_commit)
67
- # Rails 3
68
- m = :after_commit
69
- else
70
- # Rails 2
71
- m = :after_save
72
- end
64
+ if defined?(after_commit)
65
+ # Rails 3
66
+ m = :after_commit
67
+ else
68
+ # Rails 2
69
+ m = :after_save
70
+ end
73
71
 
74
- __send__(m) do |record|
75
- data = {}
76
- record.attribute_names.each {|name|
77
- name = name.to_s
78
- if (!only || only.include?(name)) && (!except || !except.include?(name))
79
- data[name] = record.read_attribute(name)
80
- end
81
- }
82
- static.each_pair {|k,v|
83
- data[k] = v
84
- }
85
- if time = data['updated_at'] && time.is_a?(Time)
86
- data['time'] = time.to_i
87
- data.delete('updated_at')
72
+ __send__(m) do |record|
73
+ data = {}
74
+ record.attribute_names.each {|name|
75
+ name = name.to_s
76
+ if (!only || only.include?(name)) && (!except || !except.include?(name))
77
+ data[name] = record.read_attribute(name)
88
78
  end
89
- TreasureData.log(tag, data)
79
+ }
80
+ static.each_pair {|k,v|
81
+ data[k] = v
82
+ }
83
+ time = data['updated_at']
84
+ if time.is_a?(Time)
85
+ data.delete('updated_at')
86
+ else
87
+ time = Time.now
90
88
  end
89
+ TreasureData::Logger.post(tag, data, time)
91
90
  end
92
91
  end
93
92
  end
@@ -1,158 +1,50 @@
1
1
  module TreasureData
2
2
  module Logger
3
- module Agent
4
- module Rails
3
+ module Agent::Rails
5
4
 
6
- CONFIG_PATH = 'config/treasure_data.yml'
5
+ CONFIG_PATH = 'config/treasure_data.yml'
7
6
 
8
- CONFIG_SAMPLE = <<EOF
9
- # logging to Treasure Data directly
10
- development:
11
- apikey: "YOUR_API_KEY"
12
- database: myapp
13
- access_log_table: access
14
- auto_create_table: true
7
+ require 'td/logger/agent/rack'
8
+ require 'td/logger/agent/rails/config'
9
+ require 'td/logger/agent/rails/controller'
10
+ #require 'td/logger/agent/rails/model'
15
11
 
16
- # logging via td-agent
17
- production:
18
- agent: "localhost:24224"
19
- tag: td.myapp
20
- access_log_table: access
21
-
22
- # disable logging
23
- test:
24
- EOF
25
-
26
- class Config
27
- def initialize(conf, rails_config)
28
- @rails_config = rails_config
29
-
30
- if agent = conf['agent']
31
- host, port = agent.split(':',2)
32
- port = (port || 24224).to_i
33
- @agent_host = host
34
- @agent_port = port
35
-
36
- @tag = conf['tag']
37
- @tag ||= conf['database']
38
- raise "'tag' nor 'database' options are not set" unless @tag
39
-
40
- else
41
- @apikey = conf['apikey']
42
- raise "'apikey' option is not set" unless @apikey
43
-
44
- @database = conf['database']
45
- raise "'database' option is not set" unless @database
46
-
47
- @auto_create_table = !!conf['auto_create_table']
48
- end
49
-
50
- @access_log_table = conf['access_log_table']
51
- end
52
-
53
- attr_reader :rails_config
54
- attr_reader :agent_host, :agent_port, :tag
55
- attr_reader :apikey, :database, :auto_create_table
56
- attr_reader :access_log_table
57
-
58
- def agent_mode?
59
- @agent_host != nil
60
- end
61
-
62
- def access_log_enabled?
63
- !@access_log_table.nil? && !@access_log_table.empty?
64
- end
12
+ def self.init(rails)
13
+ c = Config.init
14
+ unless c
15
+ ::TreasureData::Logger.open_null
16
+ return false
65
17
  end
66
18
 
67
- def self.read_config(rails)
68
- require 'yaml'
69
- require 'erb'
70
- logger = ::Rails.logger || ::Logger.new(STDERR)
71
-
72
- unless File.exist?(CONFIG_PATH)
73
- apikey = ENV['TREASURE_DATA_API_KEY'] || ENV['TD_API_KEY']
74
- unless apikey
75
- logger.warn "TREASURE_DATA_API_KEY environment variable is not set"
76
- logger.warn "#{CONFIG_PATH} does not exist."
77
- logger.warn "Disabling Treasure Data logger."
78
- return
79
- end
80
- return Config.new({
81
- 'apikey' => apikey,
82
- 'database' => ENV['TREASURE_DATA_DB'] || "rails_#{::Rails.env}",
83
- 'access_log_table' => ENV['TREASURE_DATA_TABLE'] || 'web_access',
84
- 'auto_create_table' => true
85
- }, rails)
86
- end
87
-
88
- begin
89
- src = File.read("#{::Rails.root}/#{CONFIG_PATH}")
90
- yaml = ERB.new(src).result
91
- env_conf = YAML.load(yaml)
92
- rescue
93
- logger.warn "Can't load #{CONFIG_PATH} file: #{$!}"
94
- logger.warn "Disabling Treasure Data logger."
95
- logger.warn "Example:"
96
- logger.warn CONFIG_SAMPLE
97
- return
98
- end
99
-
100
- conf = env_conf[::Rails.env]
101
- unless conf
102
- logger.warn "#{CONFIG_PATH} doesn't include setting for current environment (#{::Rails.env})."
103
- logger.warn "Disabling Treasure Data logger."
104
- return
105
- end
106
-
107
- begin
108
- return Config.new(conf, rails)
109
- rescue
110
- logger.warn "#{CONFIG_PATH}: #{$!}."
111
- logger.warn "Disabling Treasure Data logger."
112
- return
113
- end
19
+ if c.agent_mode?
20
+ ::TreasureData::Logger.open_agent(c.tag, :host=>c.agent_host, :port=>c.agent_port)
21
+ else
22
+ ::TreasureData::Logger.open(c.database, :apikey=>c.apikey, :auto_create_table=>c.auto_create_table)
114
23
  end
115
24
 
116
- def self.init(rails)
117
- require 'td/logger/agent/middleware'
118
- require 'td/logger/agent/access_log'
119
- require 'td/logger/agent/rails/controller'
120
- require 'td/logger/agent/rails/model'
121
-
122
- c = read_config(rails)
123
- return unless c
25
+ rails.middleware.use Agent::Rack::Hook
124
26
 
125
- if c.agent_mode?
126
- ::TreasureData.open_agent(c.tag, c.agent_host, c.agent_port)
127
- else
128
- ::TreasureData.open(c.apikey, c.database, c.auto_create_table)
129
- end
130
-
131
- rails.middleware.use Agent::Middleware
132
-
133
- if c.access_log_enabled?
134
- Agent.enable_access_log(c)
135
- end
136
- Agent::Rails.init_controller
137
- Agent::Rails.init_model
27
+ Agent::Rack::Hook.before do |env|
28
+ TreasureData::Logger.event.attribute.clear
138
29
  end
139
30
 
31
+ Agent::Rails::ControllerExtension.init
32
+ #Agent::Rails::AccessLogger.init(c.access_log_table) if c.access_log_enabled?
33
+ #Agent::Rails::ModelExtension.init
34
+
35
+ true
140
36
  end
141
- end
142
- end
143
- end
144
37
 
145
- if defined? ::Rails
146
38
  if ::Rails.respond_to?(:version) && ::Rails.version =~ /^3/
147
- module TreasureData
148
- class Railtie < ::Rails::Railtie
149
- initializer "treasure_data_agent.start_plugin" do |app|
150
- TreasureData::Logger::Agent::Rails.init(app.config)
151
- end
39
+ class Railtie < ::Rails::Railtie
40
+ initializer "treasure_data_logger.start_plugin" do |app|
41
+ TreasureData::Logger::Agent::Rails.init(app.config)
152
42
  end
153
43
  end
154
44
  else
155
45
  TreasureData::Logger::Agent::Rails.init(::Rails.configuration)
156
46
  end
157
- end
158
47
 
48
+ end
49
+ end
50
+ end
@@ -0,0 +1,61 @@
1
+ module TreasureData
2
+ module Logger
3
+
4
+ module Event
5
+ def self.use(mod)
6
+ send(:include, mod)
7
+ end
8
+ end
9
+
10
+ module EventPreset
11
+ def action(name, record, uid=TD.event.attribute[:uid])
12
+ unless uid
13
+ raise ArgumentError, "wrong number of arguments (2 for 3): :uid attribute is required"
14
+ end
15
+ post(name, record.merge({:uid=>uid}))
16
+ end
17
+
18
+ def register(uid=TD.event.attribute[:uid])
19
+ unless uid
20
+ raise ArgumentError, "wrong number of arguments (0 for 1): :uid attribute is required"
21
+ end
22
+ action(:register, {}, uid)
23
+ end
24
+
25
+ def login(uid=TD.event.attribute[:uid])
26
+ unless uid
27
+ raise ArgumentError, "wrong number of arguments (0 for 1): :uid attribute is required"
28
+ end
29
+ action(:login, {}, uid)
30
+ end
31
+
32
+ def pay(category, sub_category, name, price, count, uid=TD.event.attribute[:uid])
33
+ unless uid
34
+ raise ArgumentError, "wrong number of arguments (3 for 4): :uid attribute is required"
35
+ end
36
+ action(:pay, {:category=>category, :sub_category=>sub_category, :name=>name, :price=>price, :count=>count}, uid)
37
+ end
38
+ end
39
+
40
+ Event.use EventPreset
41
+
42
+ class EventCollector
43
+ def initialize
44
+ @attribute = {}
45
+ end
46
+
47
+ attr_accessor :attribute
48
+
49
+ def post(action, record, time=nil)
50
+ TreasureData::Logger.post(action, @attribute.merge(record), time)
51
+ end
52
+
53
+ include Event
54
+ end
55
+
56
+ def self.event
57
+ Thread.current[:td_event_collector] ||= EventCollector.new
58
+ end
59
+
60
+ end
61
+ end
@@ -1,4 +1,3 @@
1
-
2
1
  module TreasureData
3
2
  module Logger
4
3
 
@@ -23,20 +22,31 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
23
22
  end
24
23
  extend Finalizable
25
24
 
26
- def initialize(apikey, tag, auto_create_table)
25
+ def initialize(tag_prefix, options={})
26
+ defaults = {
27
+ :auto_create_table => false,
28
+ }
29
+ options = defaults.merge!(options)
30
+
31
+ @tag_prefix = tag_prefix
32
+ @auto_create_table = !!options[:auto_create_table]
33
+
34
+ apikey = options[:apikey]
35
+ unless apikey
36
+ raise ArgumentError, ":apikey options is required"
37
+ end
38
+
27
39
  require 'thread'
28
40
  require 'stringio'
29
41
  require 'zlib'
30
42
  require 'msgpack'
43
+ require 'json'
31
44
  require 'time'
32
45
  require 'net/http'
33
46
  require 'cgi'
34
47
  require 'logger'
35
48
  require 'td/client'
36
49
 
37
- @tag = tag
38
- @auto_create_table = auto_create_table
39
-
40
50
  @logger = ::Logger.new(STDERR)
41
51
  @logger.level = ::Logger::INFO
42
52
 
@@ -82,30 +92,14 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
82
92
  end
83
93
  end
84
94
 
85
- def post(tag, record)
86
- record[:time] ||= Time.now.to_i
95
+ def post(tag, record, time=nil)
96
+ time ||= Time.now
97
+ record[:time] ||= time.to_i
87
98
 
88
- tag = "#{@tag}.#{tag}"
99
+ tag = "#{@tag_prefix}.#{tag}" if @tag_prefix
89
100
  db, table = tag.split('.')[-2, 2]
90
101
 
91
- key = [db, table]
92
- @mutex.synchronize do
93
- buffer = (@map[key] ||= '')
94
- record.to_msgpack(buffer)
95
-
96
- if buffer.size > @chunk_limit
97
- @queue << [db, table, buffer]
98
- @map.delete(key)
99
- @cond.signal
100
- end
101
-
102
- # stat upload thread if it's not run
103
- unless @upload_thread
104
- @upload_thread = Thread.new(&method(:upload_main))
105
- end
106
- end
107
-
108
- nil
102
+ add(db, table, record)
109
103
  end
110
104
 
111
105
  def upload_main
@@ -146,6 +140,44 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
146
140
  end
147
141
 
148
142
  private
143
+ def add(db, table, msg)
144
+ begin
145
+ data = msg.to_msgpack
146
+ rescue
147
+ @logger.error("TreasureDataLogger: Can't convert to msgpack: #{msg.inspect}: #{$!}")
148
+ return false
149
+ end
150
+
151
+ key = [db, table]
152
+
153
+ @mutex.synchronize do
154
+ buffer = (@map[key] ||= '')
155
+
156
+ buffer << data
157
+
158
+ if buffer.size > @chunk_limit
159
+ @queue << [db, table, buffer]
160
+ @map.delete(key)
161
+ @cond.signal
162
+ end
163
+
164
+ # stat upload thread if it's not run
165
+ unless @upload_thread
166
+ @upload_thread = Thread.new(&method(:upload_main))
167
+ end
168
+ end
169
+
170
+ true
171
+ end
172
+
173
+ def to_msgpack(msg)
174
+ begin
175
+ msg.to_msgpack
176
+ rescue NoMethodError
177
+ JSON.load(JSON.dump(msg)).to_msgpack
178
+ end
179
+ end
180
+
149
181
  def try_flush
150
182
  @mutex.synchronize do
151
183
  if @queue.empty?
@@ -167,10 +199,10 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
167
199
  flushed = true
168
200
  rescue
169
201
  if @error_count < @retry_limit
170
- @logger.error "Failed to import logs to Treasure Data, retrying: #{$!}"
202
+ @logger.error "Failed to upload event logs to Treasure Data, retrying: #{$!}"
171
203
  @error_count += 1
172
204
  else
173
- @logger.error "Failed to import logs to Treasure Data, trashed: #{$!}"
205
+ @logger.error "Failed to upload event logs to Treasure Data, trashed: #{$!}"
174
206
  $!.backtrace.each {|bt|
175
207
  @logger.info bt
176
208
  }
@@ -185,7 +217,7 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
185
217
  end
186
218
 
187
219
  def upload(db, table, buffer)
188
- @logger.debug "Importing logs to #{db}.#{table} table on TreasureData"
220
+ @logger.debug "Uploading event logs to #{db}.#{table} table on Treasure Data"
189
221
  begin
190
222
  out = StringIO.new
191
223
  Zlib::GzipWriter.wrap(out) {|gz| gz.write buffer }
@@ -196,7 +228,7 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
196
228
  unless @auto_create_table
197
229
  raise $!
198
230
  end
199
- @logger.info "Creating table #{db}.#{table} on TreasureData"
231
+ @logger.info "Creating table #{db}.#{table} on Treasure Data"
200
232
  begin
201
233
  @client.create_log_table(db, table)
202
234
  rescue TreasureData::NotFoundError
@@ -212,7 +244,7 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
212
244
  end
213
245
 
214
246
  if ConditionVariable.new.method(:wait).arity == 1
215
- #$log.warn "WARNING: Running on Ruby 1.8. Ruby 1.9 is recommended."
247
+ # "WARNING: Running on Ruby 1.8. Ruby 1.9 is recommended."
216
248
  require 'timeout'
217
249
  def cond_wait(sec)
218
250
  Timeout.timeout(sec) {
@@ -1,7 +1,7 @@
1
1
  module TreasureData
2
2
  module Logger
3
3
 
4
- VERSION = '0.2.8'
4
+ VERSION = '0.3.0'
5
5
 
6
6
  end
7
7
  end
data/lib/td/logger.rb CHANGED
@@ -1,31 +1,77 @@
1
1
  require 'fluent/logger'
2
2
 
3
3
  module TreasureData
4
+ module Logger
5
+ autoload :TreasureDataLogger, 'td/logger/td_logger'
4
6
 
5
- def self.open(apikey, database, auto_create_table=false)
6
- require 'td/logger/tdlog'
7
- TreasureData::Logger::TreasureDataLogger.open(apikey, database, auto_create_table)
8
- end
7
+ @@logger = nil
9
8
 
10
- def self.open_agent(tag, agent_host, agent_port)
11
- Fluent::Logger::FluentLogger.open(tag, agent_host, agent_port)
12
- end
9
+ def self.open(database, options={})
10
+ @@logger = TreasureData::Logger::TreasureDataLogger.new(database, options)
11
+ end
13
12
 
14
- def self.log(tag, record)
15
- Fluent::Logger.post(tag, record)
16
- end
13
+ def self.open_agent(tag, options={})
14
+ @@logger = Fluent::Logger::FluentLogger.new(tag, options)
15
+ end
16
+
17
+ def self.open_null
18
+ @@logger = Fluent::Logger::NullLogger.new
19
+ end
17
20
 
21
+ def self.post(tag, record, time=nil)
22
+ @@logger.post(tag, record, time)
23
+ end
24
+ end
18
25
  end
19
26
 
20
27
 
21
- class Time
22
- def to_msgpack(out = '')
23
- to_i.to_msgpack(out)
28
+ # shortcut methods
29
+ module TreasureData
30
+ require 'td/logger/event'
31
+
32
+ def self.open(database, options={})
33
+ TreasureData::Logger.open(database, options)
34
+ end
35
+
36
+ def self.open_agent(tag, options={})
37
+ TreasureData::Logger.open_agent(tag, options)
38
+ end
39
+
40
+ def self.open_null
41
+ TreasureData::Logger.open_null
42
+ end
43
+
44
+ def self.post(tag, record, time=nil)
45
+ TreasureData::Logger.post(tag, record, time)
46
+ end
47
+
48
+ def self.event
49
+ TreasureData::Logger.event
50
+ end
51
+
52
+ Event = TreasureData::Logger::Event
53
+
54
+ # backward compatibility
55
+ def self.log(*args) # :nodoc:
56
+ TreasureData::Logger.post(*args)
24
57
  end
25
58
  end
26
59
 
60
+ # shortcut constants
61
+ TD = TreasureData
62
+
63
+ # implement Time#to_msgpack
64
+ unless Time.now.respond_to?(:to_msgpack)
65
+ class Time
66
+ def to_msgpack(out='')
67
+ strftime("%Y-%m-%d %H:%M:%S %z").to_msgpack(out)
68
+ end
69
+ end
70
+ end
27
71
 
28
- if defined? Rails
29
- require 'td/logger/agent/rails'
72
+ module TreasureData::Logger::Agent
73
+ if defined? ::Rails
74
+ require 'td/logger/agent/rails'
75
+ end
30
76
  end
31
77
 
metadata CHANGED
@@ -1,125 +1,91 @@
1
- --- !ruby/object:Gem::Specification
1
+ --- !ruby/object:Gem::Specification
2
2
  name: td-logger
3
- version: !ruby/object:Gem::Version
4
- hash: 7
5
- prerelease: false
6
- segments:
7
- - 0
8
- - 2
9
- - 8
10
- version: 0.2.8
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.3.0
5
+ prerelease:
11
6
  platform: ruby
12
- authors:
7
+ authors:
13
8
  - Sadayuki Furuhashi
14
9
  autorequire:
15
10
  bindir: bin
16
11
  cert_chain: []
17
-
18
- date: 2011-10-18 00:00:00 +09:00
19
- default_executable:
20
- dependencies:
21
- - !ruby/object:Gem::Dependency
12
+ date: 2011-11-04 00:00:00.000000000Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
22
15
  name: msgpack
23
- prerelease: false
24
- requirement: &id001 !ruby/object:Gem::Requirement
16
+ requirement: &70213731736700 !ruby/object:Gem::Requirement
25
17
  none: false
26
- requirements:
18
+ requirements:
27
19
  - - ~>
28
- - !ruby/object:Gem::Version
29
- hash: 7
30
- segments:
31
- - 0
32
- - 4
33
- - 4
20
+ - !ruby/object:Gem::Version
34
21
  version: 0.4.4
35
22
  type: :runtime
36
- version_requirements: *id001
37
- - !ruby/object:Gem::Dependency
38
- name: td-client
39
23
  prerelease: false
40
- requirement: &id002 !ruby/object:Gem::Requirement
24
+ version_requirements: *70213731736700
25
+ - !ruby/object:Gem::Dependency
26
+ name: td-client
27
+ requirement: &70213731736040 !ruby/object:Gem::Requirement
41
28
  none: false
42
- requirements:
29
+ requirements:
43
30
  - - ~>
44
- - !ruby/object:Gem::Version
45
- hash: 63
46
- segments:
47
- - 0
48
- - 8
49
- - 0
31
+ - !ruby/object:Gem::Version
50
32
  version: 0.8.0
51
33
  type: :runtime
52
- version_requirements: *id002
53
- - !ruby/object:Gem::Dependency
54
- name: fluent-logger
55
34
  prerelease: false
56
- requirement: &id003 !ruby/object:Gem::Requirement
35
+ version_requirements: *70213731736040
36
+ - !ruby/object:Gem::Dependency
37
+ name: fluent-logger
38
+ requirement: &70213731735340 !ruby/object:Gem::Requirement
57
39
  none: false
58
- requirements:
40
+ requirements:
59
41
  - - ~>
60
- - !ruby/object:Gem::Version
61
- hash: 19
62
- segments:
63
- - 0
64
- - 3
65
- - 0
66
- version: 0.3.0
42
+ - !ruby/object:Gem::Version
43
+ version: 0.4.0
67
44
  type: :runtime
68
- version_requirements: *id003
45
+ prerelease: false
46
+ version_requirements: *70213731735340
69
47
  description:
70
48
  email:
71
49
  executables: []
72
-
73
50
  extensions: []
74
-
75
- extra_rdoc_files:
51
+ extra_rdoc_files:
76
52
  - ChangeLog
77
53
  - README.rdoc
78
- files:
54
+ files:
79
55
  - lib/td-logger.rb
80
56
  - lib/td/logger.rb
81
- - lib/td/logger/agent/access_log.rb
82
- - lib/td/logger/agent/middleware.rb
57
+ - lib/td/logger/agent/rack.rb
83
58
  - lib/td/logger/agent/rails.rb
59
+ - lib/td/logger/agent/rails/config.rb
84
60
  - lib/td/logger/agent/rails/controller.rb
85
61
  - lib/td/logger/agent/rails/model.rb
86
- - lib/td/logger/tdlog.rb
62
+ - lib/td/logger/event.rb
63
+ - lib/td/logger/td_logger.rb
87
64
  - lib/td/logger/version.rb
88
65
  - ChangeLog
89
66
  - README.rdoc
90
- has_rdoc: true
91
67
  homepage:
92
68
  licenses: []
93
-
94
69
  post_install_message:
95
- rdoc_options:
96
- - --charset=UTF-8
97
- require_paths:
70
+ rdoc_options: []
71
+ require_paths:
98
72
  - lib
99
- required_ruby_version: !ruby/object:Gem::Requirement
73
+ required_ruby_version: !ruby/object:Gem::Requirement
100
74
  none: false
101
- requirements:
102
- - - ">="
103
- - !ruby/object:Gem::Version
104
- hash: 3
105
- segments:
106
- - 0
107
- version: "0"
108
- required_rubygems_version: !ruby/object:Gem::Requirement
75
+ requirements:
76
+ - - ! '>='
77
+ - !ruby/object:Gem::Version
78
+ version: '0'
79
+ required_rubygems_version: !ruby/object:Gem::Requirement
109
80
  none: false
110
- requirements:
111
- - - ">="
112
- - !ruby/object:Gem::Version
113
- hash: 3
114
- segments:
115
- - 0
116
- version: "0"
81
+ requirements:
82
+ - - ! '>='
83
+ - !ruby/object:Gem::Version
84
+ version: '0'
117
85
  requirements: []
118
-
119
86
  rubyforge_project:
120
- rubygems_version: 1.3.7
87
+ rubygems_version: 1.8.10
121
88
  signing_key:
122
89
  specification_version: 3
123
90
  summary: Treasure Data logging library for Rails
124
91
  test_files: []
125
-
@@ -1,146 +0,0 @@
1
-
2
- module TreasureData
3
- module Logger
4
- module Agent
5
-
6
- ACCESS_LOG_PARAM_ENV =
7
- if defined? ::Rails
8
- if ::Rails.respond_to?(:version) && ::Rails.version =~ /^3/
9
- # Rails 3
10
- 'action_dispatch.request.path_parameters'
11
- else
12
- # Rails 2
13
- 'action_controller.request.path_parameters'
14
- end
15
- else
16
- # Rack default
17
- 'rack.routing_args'
18
- end
19
-
20
- ACCESS_LOG_PRESET_PARAM_KEYS = {
21
- :controller => :controller,
22
- :action => :action,
23
- }
24
-
25
- class MessagePackedString < String
26
- def to_msgpack(out = '')
27
- out << self
28
- end
29
- end
30
-
31
- def self.enable_access_log(config)
32
- tag = config.access_log_table
33
-
34
- if config.rails_config.respond_to?(:filter_parameters)
35
- filter_parameters = config.rails_config.filter_parameters
36
- else
37
- filter_parameters = []
38
- end
39
-
40
- Middleware.before do |env|
41
- record = {}
42
- Thread.current['td.access_log'] = record
43
- env['td.access_log'] = record
44
- env['td.access_time'] = Time.now
45
- end
46
-
47
- Middleware.after do |env,result|
48
- req = env['action_dispatch.request']
49
- if !req || !req.is_a?(Rack::Request)
50
- req = Rack::Request.new(env)
51
- end
52
-
53
- # ignore OPTIONS request
54
- if req.request_method != "OPTIONS"
55
- record = env['td.access_log'] || {}
56
-
57
- # 'elapsed' column
58
- if access_time = env['td.access_time']
59
- unless record.has_key?(:elapsed)
60
- record[:elapsed] = Time.now - access_time
61
- end
62
-
63
- # always overwrite 'time' column by access time
64
- record[:time] = access_time
65
- end
66
-
67
- # merge params
68
- req.params.each_pair {|key,val|
69
- key = key.to_sym
70
- if !record.has_key?(key) && !filter_parameters.include?(key)
71
- begin
72
- record[key] = val.to_msgpack(MessagePackedString.new)
73
- rescue
74
- # ignore
75
- end
76
- end
77
- }
78
-
79
- # 'method' column
80
- if !record.has_key?(:method)
81
- record[:method] = req.request_method
82
- end
83
-
84
- # 'ip' column
85
- unless record.has_key?(:ip)
86
- record[:ip] = (env['action_dispatch.remote_ip'] || req.ip).to_s
87
- end
88
-
89
- # 'path' column
90
- # requested path before '?'
91
- unless record.has_key?(:path)
92
- if path = env['REQUEST_URI']
93
- if m = /(?:\w{1,10}\:\/\/[^\/]+)?([^\?]*)/.match(path)
94
- record[:path] = m[1]
95
- end
96
- end
97
- end
98
-
99
- # 'host' column
100
- # Rack#host_with_port consideres HTTP_X_FORWARDED_HOST
101
- unless record.has_key?(:host)
102
- record[:host] = req.host_with_port
103
- end
104
-
105
- # 'referer' column
106
- unless record.has_key?(:referer)
107
- if referer = env['HTTP_REFERER']
108
- record[:referer] = referer.to_s
109
- end
110
- end
111
-
112
- # 'agent' column
113
- unless record.has_key?(:agent)
114
- if agent = env['HTTP_USER_AGENT']
115
- record[:agent] = agent
116
- end
117
- end
118
-
119
- # 'status' column
120
- unless record.has_key?(:status)
121
- record[:status] = result[0].to_i
122
- end
123
-
124
- # 'controller' and 'action' columns
125
- if m = env[ACCESS_LOG_PARAM_ENV]
126
- ACCESS_LOG_PRESET_PARAM_KEYS.each_pair {|key,val|
127
- unless record.has_key?(key)
128
- record[key] = m[val] if m[val]
129
- end
130
- }
131
- end
132
-
133
- TreasureData.log(tag, record)
134
- end
135
- end
136
- end
137
-
138
- end
139
- end
140
- end
141
-
142
- module TreasureData
143
- def self.access_log
144
- Thread.current['td.access_log']
145
- end
146
- end
@@ -1,41 +0,0 @@
1
-
2
- module TreasureData
3
- module Logger
4
- module Agent
5
-
6
-
7
- class Middleware
8
- @@before = []
9
- @@after = []
10
-
11
- def self.before(&block)
12
- @@before << block
13
- end
14
-
15
- def self.after(&block)
16
- @@after << block
17
- end
18
-
19
- def initialize(app, options={})
20
- @app = app
21
- end
22
-
23
- def call(env)
24
- @@before.each {|m|
25
- m.call(env)
26
- }
27
-
28
- result = @app.call(env)
29
-
30
- @@after.each {|m|
31
- m.call(env, result)
32
- }
33
-
34
- result
35
- end
36
- end
37
-
38
-
39
- end
40
- end
41
- end