fluent-logger 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,3 @@
1
+ Gemfile.lock
2
+ pkg/*
3
+ coverage/*
@@ -0,0 +1,3 @@
1
+ [submodule "vendor/fluentd"]
2
+ path = vendor/fluentd
3
+ url = git://github.com/fluent/fluentd.git
data/.rspec ADDED
@@ -0,0 +1 @@
1
+ --color
@@ -0,0 +1,10 @@
1
+ rvm:
2
+ - 1.8.7
3
+ - 1.9.2
4
+ - ree
5
+ - jruby
6
+
7
+ before_script:
8
+ - git submodule update -i
9
+
10
+ script: bundle exec rake spec
data/AUTHORS ADDED
@@ -0,0 +1 @@
1
+ FURUHASHI Sadayuki <frsyuki _at_ gmail.com>
data/COPYING ADDED
@@ -0,0 +1,14 @@
1
+ Copyright (C) 2011 FURUHASHI Sadayuki
2
+
3
+ Licensed under the Apache License, Version 2.0 (the "License");
4
+ you may not use this file except in compliance with the License.
5
+ You may obtain a copy of the License at
6
+
7
+ http://www.apache.org/licenses/LICENSE-2.0
8
+
9
+ Unless required by applicable law or agreed to in writing, software
10
+ distributed under the License is distributed on an "AS IS" BASIS,
11
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ See the License for the specific language governing permissions and
13
+ limitations under the License.
14
+
data/ChangeLog CHANGED
@@ -1,4 +1,18 @@
1
1
 
2
+ Release 0.4.0 - 2011/11/05
3
+
4
+ * Wait before reconnecting to fluentd to prevent burst
5
+ * Flush logs when process stops using finalizer
6
+ * Added rspec and coverage
7
+ * Supports objects that don't support to_msgpack by
8
+ JSON.load(JSON.dump(obj)).to_msgpack
9
+ * FluentLogger uses IO#sync=true + IO#write instead of IO#syswrite to
10
+ avoid unexpected blocking
11
+ * Logger#post(tag, map) -> Logger#post(tag, map, time=Time.now)
12
+ * Removed Event classes
13
+ * Added NullLogger
14
+
15
+
2
16
  Release 0.3.1 - 2011/08/28
3
17
 
4
18
  * FluentLogger#initialize doesn't raise error when connection is failed.
data/Gemfile ADDED
@@ -0,0 +1,9 @@
1
+
2
+ source :rubygems
3
+
4
+ gemspec
5
+
6
+ gem "simplecov", :require => false
7
+ gem 'yajl-ruby' # FIXME ruby 1.8.7 don't work add_dependency('yajl-ruby')
8
+ gem "fluentd", :path => 'vendor/fluentd' if RUBY_VERSION >= "1.9.2"
9
+
@@ -8,109 +8,47 @@ A structured event loger
8
8
 
9
9
  require 'fluent-logger'
10
10
 
11
- EventLogger = Fluent::Logger::ConsoleLogger.new(STDOUT)
11
+ log = Fluent::Logger::FluentLogger.new(nil, :host=>'localhost', :port=>24224)
12
+ log.post("myapp.access", {"agent"=>"foo"})
12
13
 
13
- module Event
14
- AccessEvent = EventLogger.create_event('access', :agent, :action=>'access')
15
- end
16
-
17
- #=> access: action="access" agent="foo"
18
- Event::AccessEvent.agent('foo').post!
19
-
14
+ # output: myapp.access {"agent":"foo"}
20
15
 
21
16
  === Singleton
22
17
 
23
18
  require 'fluent-logger'
24
19
 
25
- Fluent::Logger::ConsoleLogger.open(STDOUT)
20
+ Fluent::Logger::FluentLogger.open(nil, :host=>'localhost', :port=>24224)
21
+ Fluent::Logger.post("myapp.access", {"agent"=>"foo"})
26
22
 
27
- module Event
28
- AccessEvent = Fluent::Logger.create_event('access', :agent, :action=>'access')
29
- end
30
-
31
- #=> access: action="access" agent="foo"
32
- Event::AccessEvent.agent('foo').post!
33
-
23
+ # output: myapp.access {"agent":"foo"}
34
24
 
35
- === Combine events
25
+ === Tag prefix
36
26
 
37
27
  require 'fluent-logger'
38
28
 
39
- EventLogger = Fluent::Logger::ConsoleLogger.new(STDOUT)
40
-
41
- module Event
42
- User = EventLogger.create_event('user', :name, :age)
43
- LoginEvent = EventLogger.create_event('login', :action=>'login')
44
- BuyEvent = EventLogger.create_event('buy', :item, :action=>'login')
45
- end
46
-
47
- e_user = Event::User.name('me').age(24)
48
-
49
- #=> login: action="login" name="me" age=24
50
- Event::LoginEvent.with(e_user).post!
29
+ log = Fluent::Logger::FluentLogger.new('myapp', :host=>'localhost', :port=>24224)
30
+ log.post("access", {"agent"=>"foo"})
51
31
 
52
- #=> buy: action="login" name="me" age=24 item="item01"
53
- Event::BuyEvent.with(e_user).item("item01").post!
32
+ # output: myapp.access {"agent":"foo"}
54
33
 
55
- === Update created events
56
-
57
- require 'fluent-logger'
58
-
59
- EventLogger = Fluent::Logger::ConsoleLogger.new(STDOUT)
60
-
61
- module Event
62
- User = EventLogger.create_event('user', :name, :age)
63
- AgeChangeEvent = EventLogger.create_event('age_change', :changed_age, :action=>'age_change')
64
- BuyEvent = EventLogger.create_event('buy', :item, :action=>'buy')
65
- end
66
-
67
- e_user = User.name('me').age(24)
68
-
69
- #=> age_change: action="age_change" name="me" age=24 changed_age=25
70
- Event::AgeChangeEvent.with(e_user).changed_age(25).post!
71
- e_user.age!(25)
72
-
73
- #=> buy: action="buy" name="me" age=25 item="item01"
74
- Event::BuyEvent.with(e_user).item("item01").post!
75
-
76
- === Update created events by combining
77
-
78
- require 'fluent-logger'
79
-
80
- EventLogger = Fluent::Logger::ConsoleLogger.new(STDOUT)
81
-
82
- module Event
83
- User = EventLogger.create_event('user', :name, :age)
84
- Browser = EventLogger.create_event('browser', :host, :agent)
85
- LoginEvent = EventLogger.create_event('login', :action=>'login')
86
- end
87
-
88
- e_user = Event::User.name('me').age(24)
89
- e_browser = Event::Browser.host('remoteip').agent('firefox')
90
- e_user.with!(e_browser)
91
-
92
- #=> login: action="login" name="me" age=24 host="remoteip" agent="firefox"
93
- Event::LoginEvent.with(e_user).post!
34
+ == Loggers
94
35
 
36
+ === Fluent
95
37
 
96
- == Loggers
38
+ Fluent::Logger::FluentLogger.open('tag_prefix', :host=>'localhost', :port=24224)
97
39
 
98
40
  === Console
99
41
 
100
42
  Fluent::Logger::ConsoleLogger.open(io)
101
43
 
102
- === Syslog
103
-
104
- Fluent::Logger::SyslogLogger.open(ident=$0, level=:info)
105
-
106
- === Fluent
44
+ === Null
107
45
 
108
- Fluent::Logger::FluentLogger.open('tag_prefix', 'host', port=24224)
46
+ Fluent::Logger::NullLogger.open
109
47
 
110
48
 
111
49
  Web site:: http://fluent.github.com/
112
50
  Documents:: http://fluent.github.com/doc/
113
- Source repository:: http://github.com/fluent
51
+ Source repository:: https://github.com/fluent/fluent-logger-ruby
114
52
  Author:: Sadayuki Furuhashi
115
53
  Copyright:: (c) 2011 FURUHASHI Sadayuki
116
54
  License:: Apache License, Version 2.0
@@ -0,0 +1,18 @@
1
+
2
+ require 'bundler'
3
+ Bundler::GemHelper.install_tasks
4
+
5
+ require 'rspec/core'
6
+ require 'rspec/core/rake_task'
7
+
8
+ RSpec::Core::RakeTask.new(:spec) do |spec|
9
+ spec.pattern = FileList['spec/**/*_spec.rb']
10
+ end
11
+
12
+ task :coverage do |t|
13
+ ENV['SIMPLE_COV'] = '1'
14
+ Rake::Task["spec"].invoke
15
+ end
16
+
17
+ task :default => :build
18
+
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 0.4.0
@@ -0,0 +1,44 @@
1
+ # encoding: utf-8
2
+ $:.push File.expand_path('../lib', __FILE__)
3
+
4
+ Gem::Specification.new do |gem|
5
+ version_file = "lib/fluent/logger/version.rb"
6
+ version = File.read("VERSION").strip
7
+ File.open(version_file, "w") {|f|
8
+ f.write <<EOF
9
+ module Fluent
10
+ module Logger
11
+
12
+ VERSION = '#{version}'
13
+
14
+ end
15
+ end
16
+ EOF
17
+ }
18
+
19
+ unless File.exist?("vendor/fluentd/Gemfile")
20
+ puts "git submodule update -i"
21
+ system("git submodule update -i")
22
+ end
23
+
24
+ gem.name = %q{fluent-logger}
25
+ gem.version = version
26
+ # gem.platform = Gem::Platform::RUBY
27
+ gem.authors = ["Sadayuki Furuhashi"]
28
+ gem.email = %q{frsyuki@gmail.com}
29
+ gem.homepage = %q{https://github.com/fluent/fluent-logger-ruby}
30
+ gem.description = %q{fluent logger for ruby}
31
+ gem.summary = gem.description
32
+
33
+ gem.files = `git ls-files`.split("\n")
34
+ gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
35
+ gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
36
+ gem.require_paths = ['lib']
37
+
38
+ gem.add_dependency 'yajl-ruby', '~> 1.0.0'
39
+ gem.add_dependency 'msgpack', '~> 0.4.4'
40
+ gem.add_development_dependency 'rake', '>= 0.9.2'
41
+ gem.add_development_dependency 'rspec', '>= 2.7.0'
42
+ gem.add_development_dependency 'simplecov', '>= 0.5.4'
43
+ gem.add_development_dependency 'timecop', '>= 0.3.0'
44
+ end
@@ -17,30 +17,13 @@
17
17
  #
18
18
  module Fluent
19
19
 
20
-
21
20
  module Logger
22
- require 'fluent/logger/event'
23
- require 'fluent/logger/base'
24
-
25
- class DefaultLogger < LoggerBase
26
- INSTANCE = self.new
27
-
28
- def self.instance
29
- INSTANCE
30
- end
31
-
32
- def self.new
33
- INSTANCE
34
- end
35
-
36
- def post(tag, map)
37
- Fluent::Logger.default.post(tag, map)
38
- end
39
-
40
- def close
41
- Fluent::Logger.default.close
42
- end
43
- end
21
+ autoload :ConsoleLogger , 'fluent/logger/console_logger'
22
+ autoload :FluentLogger , 'fluent/logger/fluent_logger'
23
+ autoload :LoggerBase , 'fluent/logger/logger_base'
24
+ autoload :TestLogger , 'fluent/logger/test_logger'
25
+ autoload :TextLogger , 'fluent/logger/text_logger'
26
+ autoload :NullLogger , 'fluent/logger/null_logger'
44
27
 
45
28
  @@default_logger = nil
46
29
 
@@ -65,12 +48,8 @@ module Logger
65
48
  end
66
49
  end
67
50
 
68
- def self.create_event(*args)
69
- DefaultLogger.instance.create_event(*args)
70
- end
71
-
72
- def self.post(tag, map)
73
- DefaultLogger.instance.post(tag, map)
51
+ def self.post(tag, map, time=nil)
52
+ @@default_logger.post(tag, map, time)
74
53
  end
75
54
 
76
55
  def self.default
@@ -80,13 +59,6 @@ module Logger
80
59
  def self.default=(logger)
81
60
  @@default_logger = logger
82
61
  end
83
-
84
- autoload :FluentLogger, 'fluent/logger/fluent'
85
- autoload :ConsoleLogger, 'fluent/logger/console'
86
- autoload :SyslogLogger, 'fluent/logger/syslog'
87
- autoload :TestLogger, 'fluent/logger/test'
88
62
  end
89
63
 
90
-
91
64
  end
92
-
@@ -15,10 +15,11 @@
15
15
  # See the License for the specific language governing permissions and
16
16
  # limitations under the License.
17
17
  #
18
+ require 'fluent/logger/text_logger'
19
+
18
20
  module Fluent
19
21
  module Logger
20
22
 
21
-
22
23
  class ConsoleLogger < TextLogger
23
24
  def initialize(out)
24
25
  super()
@@ -0,0 +1,214 @@
1
+ #
2
+ # Fluent
3
+ #
4
+ # Copyright (C) 2011 FURUHASHI Sadayuki
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+ require 'msgpack'
19
+ require 'socket'
20
+ require 'monitor'
21
+ require 'logger'
22
+ require 'yajl'
23
+
24
+ module Fluent
25
+ module Logger
26
+
27
+
28
+ class FluentLogger < LoggerBase
29
+ module Finalizable
30
+ require 'delegate'
31
+ def new(*args, &block)
32
+ obj = allocate
33
+ obj.instance_eval { initialize(*args, &block) }
34
+ dc = DelegateClass(obj.class).new(obj)
35
+ ObjectSpace.define_finalizer(dc, finalizer(obj))
36
+ dc
37
+ end
38
+
39
+ def finalizer(obj)
40
+ fin = obj.method(:finalize)
41
+ proc {|id|
42
+ fin.call
43
+ }
44
+ end
45
+ end
46
+ extend Finalizable
47
+
48
+ BUFFER_LIMIT = 8*1024*1024
49
+ RECONNECT_WAIT = 0.5
50
+ RECONNECT_WAIT_INCR_RATE = 1.5
51
+ RECONNECT_WAIT_MAX = 60
52
+ RECONNECT_WAIT_MAX_COUNT =
53
+ (1..100).inject(RECONNECT_WAIT_MAX / RECONNECT_WAIT) {|r,i|
54
+ break i + 1 if r < RECONNECT_WAIT_INCR_RATE
55
+ r / RECONNECT_WAIT_INCR_RATE
56
+ }
57
+
58
+ def initialize(tag_prefix, *args)
59
+ super()
60
+
61
+ options = {
62
+ :host => 'localhost',
63
+ :port => 24224
64
+ }
65
+
66
+ case args.first
67
+ when String, Symbol
68
+ # backward compatible
69
+ options[:host] = args[0]
70
+ options[:port] = args[1] if args[1]
71
+ when Hash
72
+ options.update args.first
73
+ end
74
+
75
+ @tag_prefix = tag_prefix
76
+ @host = options[:host]
77
+ @port = options[:port]
78
+
79
+ @mon = Monitor.new
80
+ @pending = nil
81
+ @connect_error_history = []
82
+
83
+ @limit = options[:buffer_limit] || BUFFER_LIMIT
84
+ @logger = options[:logger] || ::Logger.new(STDERR)
85
+
86
+ begin
87
+ connect!
88
+ rescue
89
+ @logger.error "Failed to connect fluentd: #{$!}"
90
+ @logger.error "Connection will be retried."
91
+ end
92
+ end
93
+
94
+ attr_accessor :limit, :logger
95
+
96
+ def post(tag, map, time=nil)
97
+ time ||= Time.now
98
+ tag = "#{@tag_prefix}.#{tag}" if @tag_prefix
99
+ write [tag, time.to_i, map]
100
+ end
101
+
102
+ def close
103
+ @mon.synchronize {
104
+ if @pending
105
+ begin
106
+ send_data(@pending)
107
+ rescue
108
+ @logger.error("FluentLogger: Can't send logs to #{@host}:#{@port}: #{$!}")
109
+ end
110
+ end
111
+ @con.close if connect?
112
+ @con = nil
113
+ @pending = nil
114
+ }
115
+ self
116
+ end
117
+
118
+ def connect?
119
+ !!@con
120
+ end
121
+
122
+ def finalize
123
+ close
124
+ end
125
+
126
+ private
127
+ def to_msgpack(msg)
128
+ begin
129
+ msg.to_msgpack
130
+ rescue NoMethodError
131
+ Yajl::Parser.parse( Yajl::Encoder.encode(msg) ).to_msgpack
132
+ end
133
+ end
134
+
135
+ def write(msg)
136
+ begin
137
+ data = to_msgpack(msg)
138
+ rescue
139
+ @logger.error("FluentLogger: Can't convert to msgpack: #{msg.inspect}: #{$!}")
140
+ return false
141
+ end
142
+
143
+ @mon.synchronize {
144
+ if @pending
145
+ @pending << data
146
+ else
147
+ @pending = data
148
+ end
149
+
150
+ # suppress reconnection burst
151
+ if !@connect_error_history.empty? && @pending.bytesize <= @limit
152
+ if (sz = @connect_error_history.size) < RECONNECT_WAIT_MAX_COUNT
153
+ suppress_sec = RECONNECT_WAIT * (RECONNECT_WAIT_INCR_RATE ** (sz - 1))
154
+ else
155
+ suppress_sec = RECONNECT_WAIT_MAX
156
+ end
157
+ if Time.now.to_i - @connect_error_history.last < suppress_sec
158
+ return false
159
+ end
160
+ end
161
+
162
+ begin
163
+ send_data(@pending)
164
+ @pending = nil
165
+ true
166
+ rescue
167
+ if @pending.bytesize > @limit
168
+ @logger.error("FluentLogger: Can't send logs to #{@host}:#{@port}: #{$!}")
169
+ @pending = nil
170
+ end
171
+ @con.close if connect?
172
+ @con = nil
173
+ false
174
+ end
175
+ }
176
+ end
177
+
178
+ def send_data(data)
179
+ unless connect?
180
+ connect!
181
+ end
182
+ @con.write data
183
+ #while true
184
+ # puts "sending #{data.length} bytes"
185
+ # if data.length > 32*1024
186
+ # n = @con.syswrite(data[0..32*1024])
187
+ # else
188
+ # n = @con.syswrite(data)
189
+ # end
190
+ # puts "sent #{n}"
191
+ # if n >= data.bytesize
192
+ # break
193
+ # end
194
+ # data = data[n..-1]
195
+ #end
196
+ true
197
+ end
198
+
199
+ def connect!
200
+ @con = TCPSocket.new(@host, @port)
201
+ @con.sync = true
202
+ @connect_error_history.clear
203
+ rescue
204
+ @connect_error_history << Time.now.to_i
205
+ if @connect_error_history.size > RECONNECT_WAIT_MAX_COUNT
206
+ @connect_error_history.shift
207
+ end
208
+ raise
209
+ end
210
+ end
211
+
212
+
213
+ end
214
+ end