td-logger 0.2.8 → 0.3.0

Sign up to get free protection for your applications and to get access to all the features.
data/ChangeLog CHANGED
@@ -1,4 +1,13 @@
1
1
 
2
+ == 2011-11-05 version 0.3.0
3
+
4
+ * Added new API: TD.event
5
+ * Added event presets: TD.event.register, login, pay
6
+ * Added feature to add set attributes to all events: TD.event.attribute
7
+ * Stop to log access logs
8
+ * Use NullLogger when logger is disabled
9
+
10
+
2
11
  == 2011-10-17 version 0.2.8
3
12
 
4
13
  * Access logger checks to_msgpack for each values and nested values
data/README.rdoc CHANGED
@@ -29,67 +29,32 @@ And then add +config/treasure_data.yml+ file as following:
29
29
  # disable logging
30
30
  test:
31
31
 
32
- === Logging access logs
32
+ == Logging events
33
33
 
34
- Following information will be logged automatically:
34
+ You can log anytime using 'TD.event.post' method:
35
35
 
36
- * controller name ('controller' column)
37
- * action name ('action' column)
38
- * remote host ip address ('remote_addr' column)
39
- * http status code ('status' column)
40
- * http referer ('refer' column)
41
-
42
- To add information to the log, call 'td_access_log' method in a Controller:
43
-
44
- class YourController < ApplicationController
45
- def myaction
46
- td_access_log[:column] = "value"
47
- end
48
- end
49
-
50
- Or call 'TreasureData.access_log' method outside of a Controller:
51
-
52
- class MyHelperModule
36
+ class MyClass
53
37
  def mymethod
54
- TreasureData.access_log[:column] = "value"
38
+ TD.event.post('table_name', {:foo=>:bar})
55
39
  end
56
40
  end
57
41
 
42
+ == Rails extension
58
43
 
59
- === Logging model changes
60
-
61
- Add 'td_enable_model_tracer' line to your Model class:
62
-
63
- class MyModel < ActiveRecord::Base
64
- td_enable_model_tracer :mytable
65
- end
66
-
67
- It logs all columns when the model is changed. Use :only option to limit columns to log:
68
-
69
- class MyModel < ActiveRecord::Base
70
- td_enable_model_tracer :mytable, :only => [:id, :mycol]
71
- end
72
-
73
- Alternatively, use :except option to except columns to log:
74
-
75
- class MyModel < ActiveRecord::Base
76
- td_enable_model_tracer :mytable, :except => [:created_at]
77
- end
78
-
79
- Add 'static' option to add constant key-value pairs:
44
+ In rails application, you can use 'TD.event.attribute' to set static attribute to all events:
80
45
 
81
- class MyModel < ActiveRecord::Base
82
- td_enable_model_tracer :mytable, :except => [:created_at], :static => {'model'=>'my'}
46
+ class ApplicationController
47
+ def authenticate
48
+ # set 'uid' attribute to succeeding event logs
49
+ TD.event.attribute[:uid] = 198
50
+ end
83
51
  end
84
52
 
85
-
86
- === Logging other logs
87
-
88
- You can log anytime using 'TreasureData.log' method:
89
-
90
- class MyClass
53
+ class MyController < ApplicationController
91
54
  def mymethod
92
- TreasureData.log('mytable', {:col1=>"val1", :col2=>"val2"}
55
+ authenticate()
56
+ # this event includes 'uid' attribute
57
+ TD.event.post('table_name', {:foo=>:bar})
93
58
  end
94
59
  end
95
60
 
@@ -0,0 +1,39 @@
1
+ module TreasureData
2
+ module Logger
3
+ module Agent::Rack
4
+
5
+ class Hook
6
+ @@before = []
7
+ @@after = []
8
+
9
+ def self.before(&block)
10
+ @@before << block
11
+ end
12
+
13
+ def self.after(&block)
14
+ @@after << block
15
+ end
16
+
17
+ def initialize(app, options={})
18
+ @app = app
19
+ end
20
+
21
+ def call(env)
22
+ @@before.each {|m|
23
+ m.call(env)
24
+ }
25
+
26
+ result = @app.call(env)
27
+
28
+ @@after.each {|m|
29
+ m.call(env, result)
30
+ }
31
+
32
+ result
33
+ end
34
+ end
35
+
36
+ end
37
+ end
38
+ end
39
+
@@ -0,0 +1,105 @@
1
+ module TreasureData
2
+ module Logger
3
+ module Agent::Rails
4
+
5
+ class Config
6
+ def initialize(conf)
7
+ if agent = conf['agent']
8
+ host, port = agent.split(':',2)
9
+ port = (port || 24224).to_i
10
+ @agent_host = host
11
+ @agent_port = port
12
+
13
+ @tag = conf['tag']
14
+ @tag ||= conf['database']
15
+ raise "'tag' nor 'database' options are not set" unless @tag
16
+
17
+ else
18
+ @apikey = conf['apikey']
19
+ raise "'apikey' option is not set" unless @apikey
20
+
21
+ @database = conf['database']
22
+ raise "'database' option is not set" unless @database
23
+
24
+ if conf.has_key?('auto_create_table')
25
+ @auto_create_table = !!conf['auto_create_table']
26
+ else
27
+ @auto_create_table = true
28
+ end
29
+ end
30
+
31
+ @access_log_table = conf['access_log_table']
32
+ end
33
+
34
+ attr_reader :agent_host, :agent_port, :tag
35
+ attr_reader :apikey, :database, :auto_create_table
36
+ attr_reader :access_log_table
37
+
38
+ def agent_mode?
39
+ @agent_host != nil
40
+ end
41
+
42
+ def access_log_enabled?
43
+ !@access_log_table.nil? && !@access_log_table.empty?
44
+ end
45
+
46
+ def self.init
47
+ logger = ::Rails.logger || ::Logger.new(STDERR)
48
+ if File.exist?("#{::Rails.root}/#{CONFIG_PATH}")
49
+ load_file(logger)
50
+ else
51
+ load_env(logger)
52
+ end
53
+ end
54
+
55
+ def self.load_file(logger)
56
+ require 'yaml'
57
+ require 'erb'
58
+
59
+ begin
60
+ src = File.read("#{::Rails.root}/#{CONFIG_PATH}")
61
+ yaml = ERB.new(src).result
62
+ env_conf = YAML.load(yaml)
63
+ rescue
64
+ logger.warn "Can't load #{CONFIG_PATH} file: #{$!}"
65
+ logger.warn "Disabling Treasure Data event logger."
66
+ return nil
67
+ end
68
+
69
+ conf = env_conf[::Rails.env]
70
+ unless conf
71
+ logger.warn "#{CONFIG_PATH} doesn't include setting for current environment (#{::Rails.env})."
72
+ logger.warn "Disabling Treasure Data event logger."
73
+ return nil
74
+ end
75
+
76
+ begin
77
+ return Config.new(conf)
78
+ rescue
79
+ logger.warn "#{CONFIG_PATH}: #{$!}."
80
+ logger.warn "Disabling Treasure Data event logger."
81
+ return nil
82
+ end
83
+ end
84
+
85
+ def self.load_env(logger)
86
+ apikey = ENV['TREASURE_DATA_API_KEY'] || ENV['TD_API_KEY']
87
+
88
+ unless apikey
89
+ logger.warn "#{CONFIG_PATH} does not exist."
90
+ logger.warn "Disabling Treasure Data event logger."
91
+ return nil
92
+ end
93
+
94
+ return Config.new({
95
+ 'apikey' => apikey,
96
+ 'database' => ENV['TREASURE_DATA_DB'] || "rails_#{::Rails.env}",
97
+ 'access_log_table' => ENV['TREASURE_DATA_ACCESS_LOG_TABLE'],
98
+ 'auto_create_table' => true
99
+ })
100
+ end
101
+ end
102
+
103
+ end
104
+ end
105
+ end
@@ -1,35 +1,33 @@
1
1
  module TreasureData
2
2
  module Logger
3
- module Agent
4
- module Rails
3
+ module Agent::Rails
4
+ module ControllerExtension
5
5
 
6
- def self.init_controller
7
- ActionController::Base.send(:include, ControllerExtension)
6
+ def self.init
7
+ ::ActionController::Base.send(:include, self)
8
8
  end
9
9
 
10
- module ControllerExtension
11
- if defined?(ActiveSupport::Concern)
12
- # Rails 2
13
- extend ActiveSupport::Concern
14
- else
15
- def self.included(mod)
16
- im = InstanceMethods
17
- cm = ClassMethods
18
- mod.class_eval do
19
- include im
20
- extend cm
21
- end
10
+ if defined?(ActiveSupport::Concern)
11
+ extend ActiveSupport::Concern
12
+ else
13
+ # Rails 2
14
+ def self.included(mod)
15
+ im = InstanceMethods
16
+ cm = ClassMethods
17
+ mod.class_eval do
18
+ include im
19
+ extend cm
22
20
  end
23
21
  end
22
+ end
24
23
 
25
- module InstanceMethods
26
- def td_access_log
27
- request.env['td.access_log'] ||= {}
28
- end
24
+ module InstanceMethods
25
+ def event
26
+ TreasureData::Logger.event
29
27
  end
28
+ end
30
29
 
31
- module ClassMethods
32
- end
30
+ module ClassMethods
33
31
  end
34
32
 
35
33
  end
@@ -1,93 +1,92 @@
1
1
  module TreasureData
2
2
  module Logger
3
- module Agent
4
- module Rails
3
+ module Agent::Rails
4
+ module ModelExtension
5
5
 
6
- def self.init_model
7
- unless defined?(::ActiveRecord)
8
- # disable model extension if Rails > 3 and
9
- # ActiveRecord is not loaded (other ORM is used)
10
- if ::Rails.respond_to?(:version) && ::Rails.version =~ /^3/
11
- return
12
- end
13
- require 'active_record'
6
+ def self.init
7
+ # disable model extension ActiveRecord is not loaded
8
+ # on Rails > 3 (other ORM is used)
9
+ if !defined?(::ActiveRecord) &&
10
+ ::Rails.respond_to?(:version) && ::Rails.version =~ /^3/
11
+ return
14
12
  end
15
- ::ActiveRecord::Base.send(:include, ModelExtension)
13
+ require 'active_record'
14
+ ::ActiveRecord::Base.send(:include, self)
16
15
  end
17
16
 
18
- module ModelExtension
19
- if defined?(ActiveSupport::Concern)
20
- # Rails 2
21
- extend ActiveSupport::Concern
22
- else
23
- def self.included(mod)
24
- im = InstanceMethods
25
- cm = ClassMethods
26
- mod.class_eval do
27
- include im
28
- extend cm
29
- end
17
+ if defined?(ActiveSupport::Concern)
18
+ extend ActiveSupport::Concern
19
+ else
20
+ # Rails 2
21
+ def self.included(mod)
22
+ im = InstanceMethods
23
+ cm = ClassMethods
24
+ mod.class_eval do
25
+ include im
26
+ extend cm
30
27
  end
31
28
  end
29
+ end
32
30
 
33
- module InstanceMethods
34
- end
31
+ module InstanceMethods
32
+ end
35
33
 
36
- module ClassMethods
37
- def td_enable_model_tracer(tag, options={})
38
- only = nil
39
- except = nil
40
- static = {}
34
+ module ClassMethods
35
+ def td_enable_model_tracer(tag, options={})
36
+ only = nil
37
+ except = nil
38
+ static = {}
41
39
 
42
- if o = options[:only]
43
- only = case o
40
+ if o = options[:only]
41
+ only = case o
42
+ when Array
43
+ o
44
+ else
45
+ [o]
46
+ end.map {|e| e.to_s }
47
+ end
48
+
49
+ if o = options[:except]
50
+ except = case o
44
51
  when Array
45
52
  o
46
53
  else
47
54
  [o]
48
55
  end.map {|e| e.to_s }
49
- end
50
-
51
- if o = options[:except]
52
- except = case o
53
- when Array
54
- o
55
- else
56
- [o]
57
- end.map {|e| e.to_s }
58
- end
56
+ end
59
57
 
60
- if o = options[:static]
61
- o.each_pair {|k,v|
62
- static[k.to_s] = v
63
- }
64
- end
58
+ if o = options[:static]
59
+ o.each_pair {|k,v|
60
+ static[k.to_s] = v
61
+ }
62
+ end
65
63
 
66
- if defined?(after_commit)
67
- # Rails 3
68
- m = :after_commit
69
- else
70
- # Rails 2
71
- m = :after_save
72
- end
64
+ if defined?(after_commit)
65
+ # Rails 3
66
+ m = :after_commit
67
+ else
68
+ # Rails 2
69
+ m = :after_save
70
+ end
73
71
 
74
- __send__(m) do |record|
75
- data = {}
76
- record.attribute_names.each {|name|
77
- name = name.to_s
78
- if (!only || only.include?(name)) && (!except || !except.include?(name))
79
- data[name] = record.read_attribute(name)
80
- end
81
- }
82
- static.each_pair {|k,v|
83
- data[k] = v
84
- }
85
- if time = data['updated_at'] && time.is_a?(Time)
86
- data['time'] = time.to_i
87
- data.delete('updated_at')
72
+ __send__(m) do |record|
73
+ data = {}
74
+ record.attribute_names.each {|name|
75
+ name = name.to_s
76
+ if (!only || only.include?(name)) && (!except || !except.include?(name))
77
+ data[name] = record.read_attribute(name)
88
78
  end
89
- TreasureData.log(tag, data)
79
+ }
80
+ static.each_pair {|k,v|
81
+ data[k] = v
82
+ }
83
+ time = data['updated_at']
84
+ if time.is_a?(Time)
85
+ data.delete('updated_at')
86
+ else
87
+ time = Time.now
90
88
  end
89
+ TreasureData::Logger.post(tag, data, time)
91
90
  end
92
91
  end
93
92
  end
@@ -1,158 +1,50 @@
1
1
  module TreasureData
2
2
  module Logger
3
- module Agent
4
- module Rails
3
+ module Agent::Rails
5
4
 
6
- CONFIG_PATH = 'config/treasure_data.yml'
5
+ CONFIG_PATH = 'config/treasure_data.yml'
7
6
 
8
- CONFIG_SAMPLE = <<EOF
9
- # logging to Treasure Data directly
10
- development:
11
- apikey: "YOUR_API_KEY"
12
- database: myapp
13
- access_log_table: access
14
- auto_create_table: true
7
+ require 'td/logger/agent/rack'
8
+ require 'td/logger/agent/rails/config'
9
+ require 'td/logger/agent/rails/controller'
10
+ #require 'td/logger/agent/rails/model'
15
11
 
16
- # logging via td-agent
17
- production:
18
- agent: "localhost:24224"
19
- tag: td.myapp
20
- access_log_table: access
21
-
22
- # disable logging
23
- test:
24
- EOF
25
-
26
- class Config
27
- def initialize(conf, rails_config)
28
- @rails_config = rails_config
29
-
30
- if agent = conf['agent']
31
- host, port = agent.split(':',2)
32
- port = (port || 24224).to_i
33
- @agent_host = host
34
- @agent_port = port
35
-
36
- @tag = conf['tag']
37
- @tag ||= conf['database']
38
- raise "'tag' nor 'database' options are not set" unless @tag
39
-
40
- else
41
- @apikey = conf['apikey']
42
- raise "'apikey' option is not set" unless @apikey
43
-
44
- @database = conf['database']
45
- raise "'database' option is not set" unless @database
46
-
47
- @auto_create_table = !!conf['auto_create_table']
48
- end
49
-
50
- @access_log_table = conf['access_log_table']
51
- end
52
-
53
- attr_reader :rails_config
54
- attr_reader :agent_host, :agent_port, :tag
55
- attr_reader :apikey, :database, :auto_create_table
56
- attr_reader :access_log_table
57
-
58
- def agent_mode?
59
- @agent_host != nil
60
- end
61
-
62
- def access_log_enabled?
63
- !@access_log_table.nil? && !@access_log_table.empty?
64
- end
12
+ def self.init(rails)
13
+ c = Config.init
14
+ unless c
15
+ ::TreasureData::Logger.open_null
16
+ return false
65
17
  end
66
18
 
67
- def self.read_config(rails)
68
- require 'yaml'
69
- require 'erb'
70
- logger = ::Rails.logger || ::Logger.new(STDERR)
71
-
72
- unless File.exist?(CONFIG_PATH)
73
- apikey = ENV['TREASURE_DATA_API_KEY'] || ENV['TD_API_KEY']
74
- unless apikey
75
- logger.warn "TREASURE_DATA_API_KEY environment variable is not set"
76
- logger.warn "#{CONFIG_PATH} does not exist."
77
- logger.warn "Disabling Treasure Data logger."
78
- return
79
- end
80
- return Config.new({
81
- 'apikey' => apikey,
82
- 'database' => ENV['TREASURE_DATA_DB'] || "rails_#{::Rails.env}",
83
- 'access_log_table' => ENV['TREASURE_DATA_TABLE'] || 'web_access',
84
- 'auto_create_table' => true
85
- }, rails)
86
- end
87
-
88
- begin
89
- src = File.read("#{::Rails.root}/#{CONFIG_PATH}")
90
- yaml = ERB.new(src).result
91
- env_conf = YAML.load(yaml)
92
- rescue
93
- logger.warn "Can't load #{CONFIG_PATH} file: #{$!}"
94
- logger.warn "Disabling Treasure Data logger."
95
- logger.warn "Example:"
96
- logger.warn CONFIG_SAMPLE
97
- return
98
- end
99
-
100
- conf = env_conf[::Rails.env]
101
- unless conf
102
- logger.warn "#{CONFIG_PATH} doesn't include setting for current environment (#{::Rails.env})."
103
- logger.warn "Disabling Treasure Data logger."
104
- return
105
- end
106
-
107
- begin
108
- return Config.new(conf, rails)
109
- rescue
110
- logger.warn "#{CONFIG_PATH}: #{$!}."
111
- logger.warn "Disabling Treasure Data logger."
112
- return
113
- end
19
+ if c.agent_mode?
20
+ ::TreasureData::Logger.open_agent(c.tag, :host=>c.agent_host, :port=>c.agent_port)
21
+ else
22
+ ::TreasureData::Logger.open(c.database, :apikey=>c.apikey, :auto_create_table=>c.auto_create_table)
114
23
  end
115
24
 
116
- def self.init(rails)
117
- require 'td/logger/agent/middleware'
118
- require 'td/logger/agent/access_log'
119
- require 'td/logger/agent/rails/controller'
120
- require 'td/logger/agent/rails/model'
121
-
122
- c = read_config(rails)
123
- return unless c
25
+ rails.middleware.use Agent::Rack::Hook
124
26
 
125
- if c.agent_mode?
126
- ::TreasureData.open_agent(c.tag, c.agent_host, c.agent_port)
127
- else
128
- ::TreasureData.open(c.apikey, c.database, c.auto_create_table)
129
- end
130
-
131
- rails.middleware.use Agent::Middleware
132
-
133
- if c.access_log_enabled?
134
- Agent.enable_access_log(c)
135
- end
136
- Agent::Rails.init_controller
137
- Agent::Rails.init_model
27
+ Agent::Rack::Hook.before do |env|
28
+ TreasureData::Logger.event.attribute.clear
138
29
  end
139
30
 
31
+ Agent::Rails::ControllerExtension.init
32
+ #Agent::Rails::AccessLogger.init(c.access_log_table) if c.access_log_enabled?
33
+ #Agent::Rails::ModelExtension.init
34
+
35
+ true
140
36
  end
141
- end
142
- end
143
- end
144
37
 
145
- if defined? ::Rails
146
38
  if ::Rails.respond_to?(:version) && ::Rails.version =~ /^3/
147
- module TreasureData
148
- class Railtie < ::Rails::Railtie
149
- initializer "treasure_data_agent.start_plugin" do |app|
150
- TreasureData::Logger::Agent::Rails.init(app.config)
151
- end
39
+ class Railtie < ::Rails::Railtie
40
+ initializer "treasure_data_logger.start_plugin" do |app|
41
+ TreasureData::Logger::Agent::Rails.init(app.config)
152
42
  end
153
43
  end
154
44
  else
155
45
  TreasureData::Logger::Agent::Rails.init(::Rails.configuration)
156
46
  end
157
- end
158
47
 
48
+ end
49
+ end
50
+ end
@@ -0,0 +1,61 @@
1
+ module TreasureData
2
+ module Logger
3
+
4
+ module Event
5
+ def self.use(mod)
6
+ send(:include, mod)
7
+ end
8
+ end
9
+
10
+ module EventPreset
11
+ def action(name, record, uid=TD.event.attribute[:uid])
12
+ unless uid
13
+ raise ArgumentError, "wrong number of arguments (2 for 3): :uid attribute is required"
14
+ end
15
+ post(name, record.merge({:uid=>uid}))
16
+ end
17
+
18
+ def register(uid=TD.event.attribute[:uid])
19
+ unless uid
20
+ raise ArgumentError, "wrong number of arguments (0 for 1): :uid attribute is required"
21
+ end
22
+ action(:register, {}, uid)
23
+ end
24
+
25
+ def login(uid=TD.event.attribute[:uid])
26
+ unless uid
27
+ raise ArgumentError, "wrong number of arguments (0 for 1): :uid attribute is required"
28
+ end
29
+ action(:login, {}, uid)
30
+ end
31
+
32
+ def pay(category, sub_category, name, price, count, uid=TD.event.attribute[:uid])
33
+ unless uid
34
+ raise ArgumentError, "wrong number of arguments (3 for 4): :uid attribute is required"
35
+ end
36
+ action(:pay, {:category=>category, :sub_category=>sub_category, :name=>name, :price=>price, :count=>count}, uid)
37
+ end
38
+ end
39
+
40
+ Event.use EventPreset
41
+
42
+ class EventCollector
43
+ def initialize
44
+ @attribute = {}
45
+ end
46
+
47
+ attr_accessor :attribute
48
+
49
+ def post(action, record, time=nil)
50
+ TreasureData::Logger.post(action, @attribute.merge(record), time)
51
+ end
52
+
53
+ include Event
54
+ end
55
+
56
+ def self.event
57
+ Thread.current[:td_event_collector] ||= EventCollector.new
58
+ end
59
+
60
+ end
61
+ end
@@ -1,4 +1,3 @@
1
-
2
1
  module TreasureData
3
2
  module Logger
4
3
 
@@ -23,20 +22,31 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
23
22
  end
24
23
  extend Finalizable
25
24
 
26
- def initialize(apikey, tag, auto_create_table)
25
+ def initialize(tag_prefix, options={})
26
+ defaults = {
27
+ :auto_create_table => false,
28
+ }
29
+ options = defaults.merge!(options)
30
+
31
+ @tag_prefix = tag_prefix
32
+ @auto_create_table = !!options[:auto_create_table]
33
+
34
+ apikey = options[:apikey]
35
+ unless apikey
36
+ raise ArgumentError, ":apikey options is required"
37
+ end
38
+
27
39
  require 'thread'
28
40
  require 'stringio'
29
41
  require 'zlib'
30
42
  require 'msgpack'
43
+ require 'json'
31
44
  require 'time'
32
45
  require 'net/http'
33
46
  require 'cgi'
34
47
  require 'logger'
35
48
  require 'td/client'
36
49
 
37
- @tag = tag
38
- @auto_create_table = auto_create_table
39
-
40
50
  @logger = ::Logger.new(STDERR)
41
51
  @logger.level = ::Logger::INFO
42
52
 
@@ -82,30 +92,14 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
82
92
  end
83
93
  end
84
94
 
85
- def post(tag, record)
86
- record[:time] ||= Time.now.to_i
95
+ def post(tag, record, time=nil)
96
+ time ||= Time.now
97
+ record[:time] ||= time.to_i
87
98
 
88
- tag = "#{@tag}.#{tag}"
99
+ tag = "#{@tag_prefix}.#{tag}" if @tag_prefix
89
100
  db, table = tag.split('.')[-2, 2]
90
101
 
91
- key = [db, table]
92
- @mutex.synchronize do
93
- buffer = (@map[key] ||= '')
94
- record.to_msgpack(buffer)
95
-
96
- if buffer.size > @chunk_limit
97
- @queue << [db, table, buffer]
98
- @map.delete(key)
99
- @cond.signal
100
- end
101
-
102
- # stat upload thread if it's not run
103
- unless @upload_thread
104
- @upload_thread = Thread.new(&method(:upload_main))
105
- end
106
- end
107
-
108
- nil
102
+ add(db, table, record)
109
103
  end
110
104
 
111
105
  def upload_main
@@ -146,6 +140,44 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
146
140
  end
147
141
 
148
142
  private
143
+ def add(db, table, msg)
144
+ begin
145
+ data = msg.to_msgpack
146
+ rescue
147
+ @logger.error("TreasureDataLogger: Can't convert to msgpack: #{msg.inspect}: #{$!}")
148
+ return false
149
+ end
150
+
151
+ key = [db, table]
152
+
153
+ @mutex.synchronize do
154
+ buffer = (@map[key] ||= '')
155
+
156
+ buffer << data
157
+
158
+ if buffer.size > @chunk_limit
159
+ @queue << [db, table, buffer]
160
+ @map.delete(key)
161
+ @cond.signal
162
+ end
163
+
164
+ # stat upload thread if it's not run
165
+ unless @upload_thread
166
+ @upload_thread = Thread.new(&method(:upload_main))
167
+ end
168
+ end
169
+
170
+ true
171
+ end
172
+
173
+ def to_msgpack(msg)
174
+ begin
175
+ msg.to_msgpack
176
+ rescue NoMethodError
177
+ JSON.load(JSON.dump(msg)).to_msgpack
178
+ end
179
+ end
180
+
149
181
  def try_flush
150
182
  @mutex.synchronize do
151
183
  if @queue.empty?
@@ -167,10 +199,10 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
167
199
  flushed = true
168
200
  rescue
169
201
  if @error_count < @retry_limit
170
- @logger.error "Failed to import logs to Treasure Data, retrying: #{$!}"
202
+ @logger.error "Failed to upload event logs to Treasure Data, retrying: #{$!}"
171
203
  @error_count += 1
172
204
  else
173
- @logger.error "Failed to import logs to Treasure Data, trashed: #{$!}"
205
+ @logger.error "Failed to upload event logs to Treasure Data, trashed: #{$!}"
174
206
  $!.backtrace.each {|bt|
175
207
  @logger.info bt
176
208
  }
@@ -185,7 +217,7 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
185
217
  end
186
218
 
187
219
  def upload(db, table, buffer)
188
- @logger.debug "Importing logs to #{db}.#{table} table on TreasureData"
220
+ @logger.debug "Uploading event logs to #{db}.#{table} table on Treasure Data"
189
221
  begin
190
222
  out = StringIO.new
191
223
  Zlib::GzipWriter.wrap(out) {|gz| gz.write buffer }
@@ -196,7 +228,7 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
196
228
  unless @auto_create_table
197
229
  raise $!
198
230
  end
199
- @logger.info "Creating table #{db}.#{table} on TreasureData"
231
+ @logger.info "Creating table #{db}.#{table} on Treasure Data"
200
232
  begin
201
233
  @client.create_log_table(db, table)
202
234
  rescue TreasureData::NotFoundError
@@ -212,7 +244,7 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
212
244
  end
213
245
 
214
246
  if ConditionVariable.new.method(:wait).arity == 1
215
- #$log.warn "WARNING: Running on Ruby 1.8. Ruby 1.9 is recommended."
247
+ # "WARNING: Running on Ruby 1.8. Ruby 1.9 is recommended."
216
248
  require 'timeout'
217
249
  def cond_wait(sec)
218
250
  Timeout.timeout(sec) {
@@ -1,7 +1,7 @@
1
1
  module TreasureData
2
2
  module Logger
3
3
 
4
- VERSION = '0.2.8'
4
+ VERSION = '0.3.0'
5
5
 
6
6
  end
7
7
  end
data/lib/td/logger.rb CHANGED
@@ -1,31 +1,77 @@
1
1
  require 'fluent/logger'
2
2
 
3
3
  module TreasureData
4
+ module Logger
5
+ autoload :TreasureDataLogger, 'td/logger/td_logger'
4
6
 
5
- def self.open(apikey, database, auto_create_table=false)
6
- require 'td/logger/tdlog'
7
- TreasureData::Logger::TreasureDataLogger.open(apikey, database, auto_create_table)
8
- end
7
+ @@logger = nil
9
8
 
10
- def self.open_agent(tag, agent_host, agent_port)
11
- Fluent::Logger::FluentLogger.open(tag, agent_host, agent_port)
12
- end
9
+ def self.open(database, options={})
10
+ @@logger = TreasureData::Logger::TreasureDataLogger.new(database, options)
11
+ end
13
12
 
14
- def self.log(tag, record)
15
- Fluent::Logger.post(tag, record)
16
- end
13
+ def self.open_agent(tag, options={})
14
+ @@logger = Fluent::Logger::FluentLogger.new(tag, options)
15
+ end
16
+
17
+ def self.open_null
18
+ @@logger = Fluent::Logger::NullLogger.new
19
+ end
17
20
 
21
+ def self.post(tag, record, time=nil)
22
+ @@logger.post(tag, record, time)
23
+ end
24
+ end
18
25
  end
19
26
 
20
27
 
21
- class Time
22
- def to_msgpack(out = '')
23
- to_i.to_msgpack(out)
28
+ # shortcut methods
29
+ module TreasureData
30
+ require 'td/logger/event'
31
+
32
+ def self.open(database, options={})
33
+ TreasureData::Logger.open(database, options)
34
+ end
35
+
36
+ def self.open_agent(tag, options={})
37
+ TreasureData::Logger.open_agent(tag, options)
38
+ end
39
+
40
+ def self.open_null
41
+ TreasureData::Logger.open_null
42
+ end
43
+
44
+ def self.post(tag, record, time=nil)
45
+ TreasureData::Logger.post(tag, record, time)
46
+ end
47
+
48
+ def self.event
49
+ TreasureData::Logger.event
50
+ end
51
+
52
+ Event = TreasureData::Logger::Event
53
+
54
+ # backward compatibility
55
+ def self.log(*args) # :nodoc:
56
+ TreasureData::Logger.post(*args)
24
57
  end
25
58
  end
26
59
 
60
+ # shortcut constants
61
+ TD = TreasureData
62
+
63
+ # implement Time#to_msgpack
64
+ unless Time.now.respond_to?(:to_msgpack)
65
+ class Time
66
+ def to_msgpack(out='')
67
+ strftime("%Y-%m-%d %H:%M:%S %z").to_msgpack(out)
68
+ end
69
+ end
70
+ end
27
71
 
28
- if defined? Rails
29
- require 'td/logger/agent/rails'
72
+ module TreasureData::Logger::Agent
73
+ if defined? ::Rails
74
+ require 'td/logger/agent/rails'
75
+ end
30
76
  end
31
77
 
metadata CHANGED
@@ -1,125 +1,91 @@
1
- --- !ruby/object:Gem::Specification
1
+ --- !ruby/object:Gem::Specification
2
2
  name: td-logger
3
- version: !ruby/object:Gem::Version
4
- hash: 7
5
- prerelease: false
6
- segments:
7
- - 0
8
- - 2
9
- - 8
10
- version: 0.2.8
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.3.0
5
+ prerelease:
11
6
  platform: ruby
12
- authors:
7
+ authors:
13
8
  - Sadayuki Furuhashi
14
9
  autorequire:
15
10
  bindir: bin
16
11
  cert_chain: []
17
-
18
- date: 2011-10-18 00:00:00 +09:00
19
- default_executable:
20
- dependencies:
21
- - !ruby/object:Gem::Dependency
12
+ date: 2011-11-04 00:00:00.000000000Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
22
15
  name: msgpack
23
- prerelease: false
24
- requirement: &id001 !ruby/object:Gem::Requirement
16
+ requirement: &70213731736700 !ruby/object:Gem::Requirement
25
17
  none: false
26
- requirements:
18
+ requirements:
27
19
  - - ~>
28
- - !ruby/object:Gem::Version
29
- hash: 7
30
- segments:
31
- - 0
32
- - 4
33
- - 4
20
+ - !ruby/object:Gem::Version
34
21
  version: 0.4.4
35
22
  type: :runtime
36
- version_requirements: *id001
37
- - !ruby/object:Gem::Dependency
38
- name: td-client
39
23
  prerelease: false
40
- requirement: &id002 !ruby/object:Gem::Requirement
24
+ version_requirements: *70213731736700
25
+ - !ruby/object:Gem::Dependency
26
+ name: td-client
27
+ requirement: &70213731736040 !ruby/object:Gem::Requirement
41
28
  none: false
42
- requirements:
29
+ requirements:
43
30
  - - ~>
44
- - !ruby/object:Gem::Version
45
- hash: 63
46
- segments:
47
- - 0
48
- - 8
49
- - 0
31
+ - !ruby/object:Gem::Version
50
32
  version: 0.8.0
51
33
  type: :runtime
52
- version_requirements: *id002
53
- - !ruby/object:Gem::Dependency
54
- name: fluent-logger
55
34
  prerelease: false
56
- requirement: &id003 !ruby/object:Gem::Requirement
35
+ version_requirements: *70213731736040
36
+ - !ruby/object:Gem::Dependency
37
+ name: fluent-logger
38
+ requirement: &70213731735340 !ruby/object:Gem::Requirement
57
39
  none: false
58
- requirements:
40
+ requirements:
59
41
  - - ~>
60
- - !ruby/object:Gem::Version
61
- hash: 19
62
- segments:
63
- - 0
64
- - 3
65
- - 0
66
- version: 0.3.0
42
+ - !ruby/object:Gem::Version
43
+ version: 0.4.0
67
44
  type: :runtime
68
- version_requirements: *id003
45
+ prerelease: false
46
+ version_requirements: *70213731735340
69
47
  description:
70
48
  email:
71
49
  executables: []
72
-
73
50
  extensions: []
74
-
75
- extra_rdoc_files:
51
+ extra_rdoc_files:
76
52
  - ChangeLog
77
53
  - README.rdoc
78
- files:
54
+ files:
79
55
  - lib/td-logger.rb
80
56
  - lib/td/logger.rb
81
- - lib/td/logger/agent/access_log.rb
82
- - lib/td/logger/agent/middleware.rb
57
+ - lib/td/logger/agent/rack.rb
83
58
  - lib/td/logger/agent/rails.rb
59
+ - lib/td/logger/agent/rails/config.rb
84
60
  - lib/td/logger/agent/rails/controller.rb
85
61
  - lib/td/logger/agent/rails/model.rb
86
- - lib/td/logger/tdlog.rb
62
+ - lib/td/logger/event.rb
63
+ - lib/td/logger/td_logger.rb
87
64
  - lib/td/logger/version.rb
88
65
  - ChangeLog
89
66
  - README.rdoc
90
- has_rdoc: true
91
67
  homepage:
92
68
  licenses: []
93
-
94
69
  post_install_message:
95
- rdoc_options:
96
- - --charset=UTF-8
97
- require_paths:
70
+ rdoc_options: []
71
+ require_paths:
98
72
  - lib
99
- required_ruby_version: !ruby/object:Gem::Requirement
73
+ required_ruby_version: !ruby/object:Gem::Requirement
100
74
  none: false
101
- requirements:
102
- - - ">="
103
- - !ruby/object:Gem::Version
104
- hash: 3
105
- segments:
106
- - 0
107
- version: "0"
108
- required_rubygems_version: !ruby/object:Gem::Requirement
75
+ requirements:
76
+ - - ! '>='
77
+ - !ruby/object:Gem::Version
78
+ version: '0'
79
+ required_rubygems_version: !ruby/object:Gem::Requirement
109
80
  none: false
110
- requirements:
111
- - - ">="
112
- - !ruby/object:Gem::Version
113
- hash: 3
114
- segments:
115
- - 0
116
- version: "0"
81
+ requirements:
82
+ - - ! '>='
83
+ - !ruby/object:Gem::Version
84
+ version: '0'
117
85
  requirements: []
118
-
119
86
  rubyforge_project:
120
- rubygems_version: 1.3.7
87
+ rubygems_version: 1.8.10
121
88
  signing_key:
122
89
  specification_version: 3
123
90
  summary: Treasure Data logging library for Rails
124
91
  test_files: []
125
-
@@ -1,146 +0,0 @@
1
-
2
- module TreasureData
3
- module Logger
4
- module Agent
5
-
6
- ACCESS_LOG_PARAM_ENV =
7
- if defined? ::Rails
8
- if ::Rails.respond_to?(:version) && ::Rails.version =~ /^3/
9
- # Rails 3
10
- 'action_dispatch.request.path_parameters'
11
- else
12
- # Rails 2
13
- 'action_controller.request.path_parameters'
14
- end
15
- else
16
- # Rack default
17
- 'rack.routing_args'
18
- end
19
-
20
- ACCESS_LOG_PRESET_PARAM_KEYS = {
21
- :controller => :controller,
22
- :action => :action,
23
- }
24
-
25
- class MessagePackedString < String
26
- def to_msgpack(out = '')
27
- out << self
28
- end
29
- end
30
-
31
- def self.enable_access_log(config)
32
- tag = config.access_log_table
33
-
34
- if config.rails_config.respond_to?(:filter_parameters)
35
- filter_parameters = config.rails_config.filter_parameters
36
- else
37
- filter_parameters = []
38
- end
39
-
40
- Middleware.before do |env|
41
- record = {}
42
- Thread.current['td.access_log'] = record
43
- env['td.access_log'] = record
44
- env['td.access_time'] = Time.now
45
- end
46
-
47
- Middleware.after do |env,result|
48
- req = env['action_dispatch.request']
49
- if !req || !req.is_a?(Rack::Request)
50
- req = Rack::Request.new(env)
51
- end
52
-
53
- # ignore OPTIONS request
54
- if req.request_method != "OPTIONS"
55
- record = env['td.access_log'] || {}
56
-
57
- # 'elapsed' column
58
- if access_time = env['td.access_time']
59
- unless record.has_key?(:elapsed)
60
- record[:elapsed] = Time.now - access_time
61
- end
62
-
63
- # always overwrite 'time' column by access time
64
- record[:time] = access_time
65
- end
66
-
67
- # merge params
68
- req.params.each_pair {|key,val|
69
- key = key.to_sym
70
- if !record.has_key?(key) && !filter_parameters.include?(key)
71
- begin
72
- record[key] = val.to_msgpack(MessagePackedString.new)
73
- rescue
74
- # ignore
75
- end
76
- end
77
- }
78
-
79
- # 'method' column
80
- if !record.has_key?(:method)
81
- record[:method] = req.request_method
82
- end
83
-
84
- # 'ip' column
85
- unless record.has_key?(:ip)
86
- record[:ip] = (env['action_dispatch.remote_ip'] || req.ip).to_s
87
- end
88
-
89
- # 'path' column
90
- # requested path before '?'
91
- unless record.has_key?(:path)
92
- if path = env['REQUEST_URI']
93
- if m = /(?:\w{1,10}\:\/\/[^\/]+)?([^\?]*)/.match(path)
94
- record[:path] = m[1]
95
- end
96
- end
97
- end
98
-
99
- # 'host' column
100
- # Rack#host_with_port consideres HTTP_X_FORWARDED_HOST
101
- unless record.has_key?(:host)
102
- record[:host] = req.host_with_port
103
- end
104
-
105
- # 'referer' column
106
- unless record.has_key?(:referer)
107
- if referer = env['HTTP_REFERER']
108
- record[:referer] = referer.to_s
109
- end
110
- end
111
-
112
- # 'agent' column
113
- unless record.has_key?(:agent)
114
- if agent = env['HTTP_USER_AGENT']
115
- record[:agent] = agent
116
- end
117
- end
118
-
119
- # 'status' column
120
- unless record.has_key?(:status)
121
- record[:status] = result[0].to_i
122
- end
123
-
124
- # 'controller' and 'action' columns
125
- if m = env[ACCESS_LOG_PARAM_ENV]
126
- ACCESS_LOG_PRESET_PARAM_KEYS.each_pair {|key,val|
127
- unless record.has_key?(key)
128
- record[key] = m[val] if m[val]
129
- end
130
- }
131
- end
132
-
133
- TreasureData.log(tag, record)
134
- end
135
- end
136
- end
137
-
138
- end
139
- end
140
- end
141
-
142
- module TreasureData
143
- def self.access_log
144
- Thread.current['td.access_log']
145
- end
146
- end
@@ -1,41 +0,0 @@
1
-
2
- module TreasureData
3
- module Logger
4
- module Agent
5
-
6
-
7
- class Middleware
8
- @@before = []
9
- @@after = []
10
-
11
- def self.before(&block)
12
- @@before << block
13
- end
14
-
15
- def self.after(&block)
16
- @@after << block
17
- end
18
-
19
- def initialize(app, options={})
20
- @app = app
21
- end
22
-
23
- def call(env)
24
- @@before.each {|m|
25
- m.call(env)
26
- }
27
-
28
- result = @app.call(env)
29
-
30
- @@after.each {|m|
31
- m.call(env, result)
32
- }
33
-
34
- result
35
- end
36
- end
37
-
38
-
39
- end
40
- end
41
- end