jmoses_fluent-logger 0.4.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/.gitignore ADDED
@@ -0,0 +1,4 @@
1
+ Gemfile.lock
2
+ pkg/*
3
+ coverage/*
4
+ coverage.vim
data/.gitmodules ADDED
@@ -0,0 +1,3 @@
1
+ [submodule "vendor/fluentd"]
2
+ path = vendor/fluentd
3
+ url = git://github.com/fluent/fluentd.git
data/.rspec ADDED
@@ -0,0 +1 @@
1
+ --color
data/.travis.yml ADDED
@@ -0,0 +1,15 @@
1
+ rvm:
2
+ - 1.8.7
3
+ - 1.9.2
4
+ - 1.9.3
5
+ - 2.0.0
6
+ - ree
7
+
8
+ before_install: git submodule update -i
9
+
10
+ script: bundle exec rake spec
11
+
12
+ matrix:
13
+ allow_failures:
14
+ - rvm: 1.8.7
15
+ - rvm: ree
data/AUTHORS ADDED
@@ -0,0 +1 @@
1
+ FURUHASHI Sadayuki <frsyuki _at_ gmail.com>
data/COPYING ADDED
@@ -0,0 +1,14 @@
1
+ Copyright (C) 2011 FURUHASHI Sadayuki
2
+
3
+ Licensed under the Apache License, Version 2.0 (the "License");
4
+ you may not use this file except in compliance with the License.
5
+ You may obtain a copy of the License at
6
+
7
+ http://www.apache.org/licenses/LICENSE-2.0
8
+
9
+ Unless required by applicable law or agreed to in writing, software
10
+ distributed under the License is distributed on an "AS IS" BASIS,
11
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ See the License for the specific language governing permissions and
13
+ limitations under the License.
14
+
data/ChangeLog ADDED
@@ -0,0 +1,81 @@
1
+ Release jmoses_0.4.8 - 2014/01/28
2
+
3
+ Note that you probably do *not* want this version, but the official version
4
+ instead.
5
+
6
+ * Support for batch posting events in a single connection attempt
7
+ (semi-tested)
8
+
9
+ Release 0.4.7 - 2013/11/21
10
+
11
+ * Suppress log error message with :log_reconnect_error_threshold option
12
+ * Add FluentLogger#last_error method to get last exception in user code
13
+
14
+ Release 0.4.6 - 2013/06/17
15
+
16
+ * Raise an ArgumentError when passes invalid argument to post method
17
+ * Relax msgpack gem version
18
+
19
+ Release 0.4.5 - 2013/02/26
20
+
21
+ * Use https scheme for rubygems source
22
+ * Fix broken spec
23
+
24
+ Release 0.4.4 - 2012/12/26
25
+
26
+ * Change msgpack dependency version == 0.4.7 for avoding 0.4.8 yanked issue
27
+
28
+ Release 0.4.3 - 2012/04/24
29
+
30
+ * Update yajl dependency version >= 1.0 (thanks shun0102)
31
+
32
+ Release 0.4.2 - 2012/03/02
33
+
34
+ * Added TestLogger#tag_queue(tag_name)
35
+ * Added bin/fluent-post cli command
36
+ * Impl default LoggerBase#close
37
+ * Don't change logger.level if :debug=> option is sepcified
38
+
39
+ Release 0.4.1 - 2011/11/07
40
+
41
+ * added Logger#post_with_time(tag, map, time)
42
+ * Logger#post(tag, map, time=Time.now) -> Logger#post(tag, map)
43
+ * FluentLogger supports :debug=>true option to write all events to STDERR
44
+
45
+
46
+ Release 0.4.0 - 2011/11/05
47
+
48
+ * Wait before reconnecting to fluentd to prevent burst
49
+ * Flush logs when process stops using finalizer
50
+ * Added rspec and coverage
51
+ * Supports objects that don't support to_msgpack by
52
+ JSON.load(JSON.dump(obj)).to_msgpack
53
+ * FluentLogger uses IO#sync=true + IO#write instead of IO#syswrite to
54
+ avoid unexpected blocking
55
+ * Logger#post(tag, map) -> Logger#post(tag, map, time=Time.now)
56
+ * Removed Event classes
57
+ * Added NullLogger
58
+
59
+
60
+ Release 0.3.1 - 2011/08/28
61
+
62
+ * FluentLogger#initialize doesn't raise error when connection is failed.
63
+ Instead, it tries to reconnect.
64
+
65
+
66
+ Release 0.3.0 - 2011/08/21
67
+
68
+ * Added 'tag' for event logs
69
+
70
+
71
+ Release 0.2.0 - 2011/08/05
72
+
73
+ * Redesigned Event class
74
+ * Added TestLogger (Fluent.open(:test))
75
+ * Added test programs
76
+
77
+
78
+ Release 0.1.0 - 2011/08/04
79
+
80
+ * First release
81
+
data/Gemfile ADDED
@@ -0,0 +1,11 @@
1
+
2
+ source 'https://rubygems.org/'
3
+
4
+ gemspec
5
+
6
+ gem "simplecov", :require => false
7
+ gem "simplecov-vim"
8
+
9
+ gem 'yajl-ruby' # FIXME ruby 1.8.7 don't work add_dependency('yajl-ruby')
10
+ gem "fluentd", :path => 'vendor/fluentd' if RUBY_VERSION >= "1.9.2"
11
+
data/README.md ADDED
@@ -0,0 +1,63 @@
1
+ # Fluent logger
2
+ A structured event loger
3
+
4
+ ## Examples
5
+
6
+ ### Simple
7
+
8
+ ```ruby
9
+ require 'fluent-logger'
10
+
11
+ log = Fluent::Logger::FluentLogger.new(nil, :host=>'localhost', :port=>24224)
12
+ unless log.post("myapp.access", {"agent"=>"foo"})
13
+ p log.last_error # You can get last error object via last_error method
14
+ end
15
+
16
+ # output: myapp.access {"agent":"foo"}
17
+ ```
18
+
19
+ ### Singleton
20
+ ```ruby
21
+ require 'fluent-logger'
22
+
23
+ Fluent::Logger::FluentLogger.open(nil, :host=>'localhost', :port=>24224)
24
+ Fluent::Logger.post("myapp.access", {"agent"=>"foo"})
25
+
26
+ # output: myapp.access {"agent":"foo"}
27
+ ```
28
+
29
+ ### Tag prefix
30
+ ```ruby
31
+ require 'fluent-logger'
32
+
33
+ log = Fluent::Logger::FluentLogger.new('myapp', :host=>'localhost', :port=>24224)
34
+ log.post("access", {"agent"=>"foo"})
35
+
36
+ # output: myapp.access {"agent":"foo"}
37
+ ```
38
+
39
+ ## Loggers
40
+
41
+ ### Fluent
42
+ ```ruby
43
+ Fluent::Logger::FluentLogger.open('tag_prefix', :host=>'localhost', :port=24224)
44
+ ```
45
+
46
+ ### Console
47
+ ```ruby
48
+ Fluent::Logger::ConsoleLogger.open(io)
49
+ ```
50
+
51
+ ### Null
52
+ ```ruby
53
+ Fluent::Logger::NullLogger.open
54
+ ```
55
+
56
+ |name|description|
57
+ |---|---|
58
+ |Web site|http://fluent.github.com/|
59
+ |Documents|http://fluent.github.com/doc/|
60
+ |Source repository|https://github.com/fluent/fluent-logger-ruby|
61
+ |Author|Sadayuki Furuhashi|
62
+ |Copyright|(c) 2011 FURUHASHI Sadayuki|
63
+ |License|Apache License, Version 2.0|
data/Rakefile ADDED
@@ -0,0 +1,18 @@
1
+
2
+ require 'bundler'
3
+ Bundler::GemHelper.install_tasks
4
+
5
+ require 'rspec/core'
6
+ require 'rspec/core/rake_task'
7
+
8
+ RSpec::Core::RakeTask.new(:spec) do |spec|
9
+ spec.pattern = FileList['spec/**/*_spec.rb']
10
+ end
11
+
12
+ task :coverage do |t|
13
+ ENV['SIMPLE_COV'] = '1'
14
+ Rake::Task["spec"].invoke
15
+ end
16
+
17
+ task :default => :build
18
+
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 0.4.8
data/bin/fluent-post ADDED
@@ -0,0 +1,11 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'fluent/logger/fluent_logger/cui'
4
+
5
+ res = Fluent::Logger::FluentLogger::CUI.post(ARGV)
6
+ if res[:success]
7
+ warn "post successed. #=> #{res[:data].inspect}"
8
+ else
9
+ warn "post failed. #=> #{res[:data].inspect}"
10
+ exit 1
11
+ end
@@ -0,0 +1,44 @@
1
+ # encoding: utf-8
2
+ $:.push File.expand_path('../lib', __FILE__)
3
+
4
+ Gem::Specification.new do |gem|
5
+ version_file = "lib/fluent/logger/version.rb"
6
+ version = File.read("VERSION").strip
7
+ File.open(version_file, "w") {|f|
8
+ f.write <<EOF
9
+ module Fluent
10
+ module Logger
11
+
12
+ VERSION = '#{version}'
13
+
14
+ end
15
+ end
16
+ EOF
17
+ }
18
+
19
+ unless File.exist?("vendor/fluentd/Gemfile")
20
+ puts "git submodule update -i"
21
+ system("git submodule update -i")
22
+ end
23
+
24
+ gem.name = %q{jmoses_fluent-logger}
25
+ gem.version = version
26
+ # gem.platform = Gem::Platform::RUBY
27
+ gem.authors = ["Jon Moses", "Sadayuki Furuhashi"]
28
+ gem.email = %q{jon@burningbush.us frsyuki@gmail.com}
29
+ gem.homepage = %q{https://github.com/jmoses/fluent-logger-ruby}
30
+ gem.description = %q{fluent logger for ruby}
31
+ gem.summary = gem.description + " (fork by jmoses)"
32
+
33
+ gem.files = `git ls-files`.split("\n")
34
+ gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
35
+ gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
36
+ gem.require_paths = ['lib']
37
+
38
+ gem.add_dependency 'yajl-ruby', '~> 1.0'
39
+ gem.add_dependency "msgpack", [">= 0.4.4", "!= 0.5.0", "!= 0.5.1", "!= 0.5.2", "!= 0.5.3", "< 0.6.0"]
40
+ gem.add_development_dependency 'rake', '>= 0.9.2'
41
+ gem.add_development_dependency 'rspec', '>= 2.7.0'
42
+ gem.add_development_dependency 'simplecov', '>= 0.5.4'
43
+ gem.add_development_dependency 'timecop', '>= 0.3.0'
44
+ end
@@ -0,0 +1,57 @@
1
+ #
2
+ # Fluent
3
+ #
4
+ # Copyright (C) 2011 FURUHASHI Sadayuki
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+ require 'fluent/logger/text_logger'
19
+
20
+ module Fluent
21
+ module Logger
22
+
23
+ class ConsoleLogger < TextLogger
24
+ def initialize(out)
25
+ super()
26
+ require 'time'
27
+
28
+ if out.is_a?(String)
29
+ @io = File.open(out, "a")
30
+ @on_reopen = Proc.new { @io.reopen(out, "a") }
31
+ elsif out.respond_to?(:write)
32
+ @io = out
33
+ @on_reopen = Proc.new { }
34
+ else
35
+ raise "Invalid output: #{out.inspect}"
36
+ end
37
+ end
38
+
39
+ attr_accessor :time_format
40
+
41
+ def reopen!
42
+ @on_reopen.call
43
+ end
44
+
45
+ def post_text(text)
46
+ @io.puts text
47
+ end
48
+
49
+ def close
50
+ @io.close
51
+ self
52
+ end
53
+ end
54
+
55
+
56
+ end
57
+ end
@@ -0,0 +1,46 @@
1
+
2
+ require 'fluent/logger'
3
+ require 'optparse'
4
+
5
+ module Fluent
6
+ module Logger
7
+ class FluentLogger
8
+
9
+ module CUI
10
+ def post(args)
11
+ options = {
12
+ :port => '24224',
13
+ :host => 'localhost'
14
+ }
15
+
16
+ o = OptionParser.new
17
+ o.version = Fluent::Logger::VERSION
18
+ o.on('-t [tag (default nil)]') {|v| options[:tag] = v }
19
+ o.on('-p [port (default 24224)]') {|v| options[:port] = v }
20
+ o.on('-h [host (default localhost)]') {|v| options[:host] = v }
21
+ o.on('-v [key=value]') {|v|
22
+ key, value = v.split('=')
23
+ (options[:data] ||= {})[key] = value
24
+ }
25
+ o.banner = 'Usage: fluent-post -t tag.foo.bar -v key1=value1 -v key2=value2'
26
+ args = args.to_a
27
+ args << '--help' if args.empty?
28
+ o.parse(args)
29
+
30
+ f = Fluent::Logger::FluentLogger.new(nil, {
31
+ :host => options[:host],
32
+ :port => options[:port]
33
+ })
34
+
35
+ {
36
+ :success => f.post(options[:tag], options[:data]),
37
+ :data => options[:data]
38
+ }
39
+ end
40
+
41
+ extend self
42
+ end
43
+
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,298 @@
1
+ #
2
+ # Fluent
3
+ #
4
+ # Copyright (C) 2011 FURUHASHI Sadayuki
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+ require 'msgpack'
19
+ require 'socket'
20
+ require 'monitor'
21
+ require 'logger'
22
+ require 'yajl'
23
+
24
+ module Fluent
25
+ module Logger
26
+
27
+
28
+ class FluentLogger < LoggerBase
29
+ module Finalizable
30
+ require 'delegate'
31
+ def new(*args, &block)
32
+ obj = allocate
33
+ obj.instance_eval { initialize(*args, &block) }
34
+ dc = DelegateClass(obj.class).new(obj)
35
+ ObjectSpace.define_finalizer(dc, finalizer(obj))
36
+ dc
37
+ end
38
+
39
+ def finalizer(obj)
40
+ fin = obj.method(:finalize)
41
+ proc {|id|
42
+ fin.call
43
+ }
44
+ end
45
+ end
46
+ extend Finalizable
47
+
48
+ BUFFER_LIMIT = 8*1024*1024
49
+ RECONNECT_WAIT = 0.5
50
+ RECONNECT_WAIT_INCR_RATE = 1.5
51
+ RECONNECT_WAIT_MAX = 60
52
+ RECONNECT_WAIT_MAX_COUNT =
53
+ (1..100).inject(RECONNECT_WAIT_MAX / RECONNECT_WAIT) {|r,i|
54
+ break i + 1 if r < RECONNECT_WAIT_INCR_RATE
55
+ r / RECONNECT_WAIT_INCR_RATE
56
+ }
57
+
58
+ def initialize(tag_prefix, *args)
59
+ super()
60
+
61
+ options = {
62
+ :host => 'localhost',
63
+ :port => 24224
64
+ }
65
+
66
+ case args.first
67
+ when String, Symbol
68
+ # backward compatible
69
+ options[:host] = args[0]
70
+ options[:port] = args[1] if args[1]
71
+ when Hash
72
+ options.update args.first
73
+ end
74
+
75
+ @tag_prefix = tag_prefix
76
+ @host = options[:host]
77
+ @port = options[:port]
78
+
79
+ @mon = Monitor.new
80
+ @pending = nil
81
+ @connect_error_history = []
82
+
83
+ @limit = options[:buffer_limit] || BUFFER_LIMIT
84
+ @log_reconnect_error_threshold = options[:log_reconnect_error_threshold] || RECONNECT_WAIT_MAX_COUNT
85
+
86
+ if logger = options[:logger]
87
+ @logger = logger
88
+ else
89
+ @logger = ::Logger.new(STDERR)
90
+ if options[:debug]
91
+ @logger.level = ::Logger::DEBUG
92
+ else
93
+ @logger.level = ::Logger::INFO
94
+ end
95
+ end
96
+
97
+ @last_error = {}
98
+
99
+ begin
100
+ connect!
101
+ rescue => e
102
+ set_last_error(e)
103
+ @logger.error "Failed to connect fluentd: #{$!}"
104
+ @logger.error "Connection will be retried."
105
+ end
106
+ end
107
+
108
+ attr_accessor :limit, :logger, :log_reconnect_error_threshold
109
+ attr_reader :last_error
110
+
111
+ def last_error
112
+ @last_error[Thread.current.object_id]
113
+ end
114
+
115
+ def post_with_time(tag, map, time)
116
+ @logger.debug { "event: #{tag} #{map.to_json}" rescue nil }
117
+ tag = "#{@tag_prefix}.#{tag}" if @tag_prefix
118
+ write [tag, time.to_i, map]
119
+ end
120
+
121
+ def batch_post_with_time(messages)
122
+ # convert each message
123
+ # batch send
124
+ # if batch > byte max, send in multiple requests
125
+ #
126
+ # how to handle errors? immediate fail batch, or skip bad messageS?
127
+
128
+ # what if pending is not null here? if we check needs to be syncronized
129
+
130
+ # Reorder since #post_with_time takes a different order than #write
131
+ # NOTE Should we mangle the time? It should be .to_i here
132
+ payloads = messages.map do |msg|
133
+ proper = msg.values_at(0, 2, 1)
134
+ proper[1] = proper[1].to_i unless proper[1].is_a?(Fixnum) # Convert time
135
+ prepare_msg proper
136
+ end
137
+
138
+ # Check for 'false' payloads, those are errors, and last message will be set.
139
+
140
+ payload = ""
141
+
142
+ payloads.each do |data|
143
+ if payload.bytesize + data.bytesize > @limit
144
+ raw_write payload
145
+ payload = ""
146
+ end
147
+
148
+ payload << data
149
+ end
150
+
151
+ raw_write payload if payload.bytesize > 0
152
+ end
153
+
154
+ def close
155
+ @mon.synchronize {
156
+ if @pending
157
+ begin
158
+ send_data(@pending)
159
+ rescue => e
160
+ set_last_error(e)
161
+ @logger.error("FluentLogger: Can't send logs to #{@host}:#{@port}: #{$!}")
162
+ end
163
+ end
164
+ @con.close if connect?
165
+ @con = nil
166
+ @pending = nil
167
+ }
168
+ self
169
+ end
170
+
171
+ def connect?
172
+ !!@con
173
+ end
174
+
175
+ def finalize
176
+ close
177
+ end
178
+
179
+ private
180
+ def to_msgpack(msg)
181
+ begin
182
+ msg.to_msgpack
183
+ rescue NoMethodError
184
+ Yajl::Parser.parse( Yajl::Encoder.encode(msg) ).to_msgpack
185
+ end
186
+ end
187
+
188
+ def suppress_sec
189
+ if (sz = @connect_error_history.size) < RECONNECT_WAIT_MAX_COUNT
190
+ RECONNECT_WAIT * (RECONNECT_WAIT_INCR_RATE ** (sz - 1))
191
+ else
192
+ RECONNECT_WAIT_MAX
193
+ end
194
+ end
195
+
196
+ def write(msg)
197
+ data = prepare_msg(msg)
198
+
199
+ return false if data === false
200
+
201
+ raw_write data
202
+ end
203
+
204
+ def prepare_msg(msg)
205
+ begin
206
+ to_msgpack(msg)
207
+ rescue => e
208
+ set_last_error(e)
209
+ @logger.error("FluentLogger: Can't convert to msgpack: #{msg.inspect}: #{$!}")
210
+ return false
211
+ end
212
+ end
213
+
214
+ def raw_write(data)
215
+ @mon.synchronize {
216
+ if @pending
217
+ @pending << data
218
+ else
219
+ @pending = data
220
+ end
221
+
222
+ # suppress reconnection burst
223
+ if !@connect_error_history.empty? && @pending.bytesize <= @limit
224
+ if Time.now.to_i - @connect_error_history.last < suppress_sec
225
+ return false
226
+ end
227
+ end
228
+
229
+ begin
230
+ send_data(@pending)
231
+ @pending = nil
232
+ true
233
+ rescue => e
234
+ set_last_error(e)
235
+ if @pending.bytesize > @limit
236
+ @logger.error("FluentLogger: Can't send logs to #{@host}:#{@port}: #{$!}")
237
+ @pending = nil
238
+ end
239
+ @con.close if connect?
240
+ @con = nil
241
+ false
242
+ end
243
+ }
244
+ end
245
+
246
+ def send_data(data)
247
+ unless connect?
248
+ connect!
249
+ end
250
+ @con.write data
251
+ #while true
252
+ # puts "sending #{data.length} bytes"
253
+ # if data.length > 32*1024
254
+ # n = @con.syswrite(data[0..32*1024])
255
+ # else
256
+ # n = @con.syswrite(data)
257
+ # end
258
+ # puts "sent #{n}"
259
+ # if n >= data.bytesize
260
+ # break
261
+ # end
262
+ # data = data[n..-1]
263
+ #end
264
+ true
265
+ end
266
+
267
+ def connect!
268
+ @con = TCPSocket.new(@host, @port)
269
+ @con.sync = true
270
+ @connect_error_history.clear
271
+ @logged_reconnect_error = false
272
+ rescue => e
273
+ @connect_error_history << Time.now.to_i
274
+ if @connect_error_history.size > RECONNECT_WAIT_MAX_COUNT
275
+ @connect_error_history.shift
276
+ end
277
+
278
+ if @connect_error_history.size >= @log_reconnect_error_threshold && !@logged_reconnect_error
279
+ log_reconnect_error
280
+ @logged_reconnect_error = true
281
+ end
282
+
283
+ raise e
284
+ end
285
+
286
+ def log_reconnect_error
287
+ @logger.error("FluentLogger: Can't connect to #{@host}:#{@port}(#{@connect_error_history.size} retried): #{$!}")
288
+ end
289
+
290
+ def set_last_error(e)
291
+ # TODO: Check non GVL env
292
+ @last_error[Thread.current.object_id] = e
293
+ end
294
+ end
295
+
296
+
297
+ end
298
+ end