fluentd 0.14.17-x86-mingw32 → 1.3.1-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (159) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +16 -5
  3. data/ADOPTERS.md +5 -0
  4. data/{ChangeLog → CHANGELOG.md} +495 -6
  5. data/CONTRIBUTING.md +5 -2
  6. data/GOVERNANCE.md +55 -0
  7. data/LICENSE +202 -0
  8. data/MAINTAINERS.md +7 -5
  9. data/README.md +17 -10
  10. data/bin/fluent-ca-generate +6 -0
  11. data/example/counter.conf +18 -0
  12. data/example/secondary_file.conf +3 -2
  13. data/fluentd.gemspec +3 -3
  14. data/lib/fluent/agent.rb +1 -1
  15. data/lib/fluent/command/binlog_reader.rb +11 -2
  16. data/lib/fluent/command/ca_generate.rb +181 -0
  17. data/lib/fluent/command/cat.rb +28 -15
  18. data/lib/fluent/command/debug.rb +4 -4
  19. data/lib/fluent/command/fluentd.rb +2 -2
  20. data/lib/fluent/command/plugin_config_formatter.rb +24 -2
  21. data/lib/fluent/command/plugin_generator.rb +26 -8
  22. data/lib/fluent/config/configure_proxy.rb +7 -1
  23. data/lib/fluent/config/dsl.rb +8 -5
  24. data/lib/fluent/config/element.rb +5 -0
  25. data/lib/fluent/config/literal_parser.rb +7 -1
  26. data/lib/fluent/config/types.rb +28 -2
  27. data/lib/fluent/config/v1_parser.rb +1 -2
  28. data/lib/fluent/configurable.rb +1 -0
  29. data/lib/fluent/counter.rb +23 -0
  30. data/lib/fluent/counter/base_socket.rb +46 -0
  31. data/lib/fluent/counter/client.rb +297 -0
  32. data/lib/fluent/counter/error.rb +86 -0
  33. data/lib/fluent/counter/mutex_hash.rb +163 -0
  34. data/lib/fluent/counter/server.rb +273 -0
  35. data/lib/fluent/counter/store.rb +205 -0
  36. data/lib/fluent/counter/validator.rb +145 -0
  37. data/lib/fluent/env.rb +1 -0
  38. data/lib/fluent/event_router.rb +1 -1
  39. data/lib/fluent/log.rb +119 -29
  40. data/lib/fluent/plugin/base.rb +12 -0
  41. data/lib/fluent/plugin/buf_file.rb +20 -16
  42. data/lib/fluent/plugin/buffer.rb +130 -32
  43. data/lib/fluent/plugin/buffer/file_chunk.rb +23 -4
  44. data/lib/fluent/plugin/compressable.rb +1 -1
  45. data/lib/fluent/plugin/filter_grep.rb +135 -21
  46. data/lib/fluent/plugin/filter_parser.rb +13 -2
  47. data/lib/fluent/plugin/filter_record_transformer.rb +16 -14
  48. data/lib/fluent/plugin/formatter_stdout.rb +3 -2
  49. data/lib/fluent/plugin/formatter_tsv.rb +5 -1
  50. data/lib/fluent/plugin/in_debug_agent.rb +8 -1
  51. data/lib/fluent/plugin/in_forward.rb +1 -1
  52. data/lib/fluent/plugin/in_http.rb +84 -3
  53. data/lib/fluent/plugin/in_monitor_agent.rb +7 -1
  54. data/lib/fluent/plugin/in_syslog.rb +31 -10
  55. data/lib/fluent/plugin/in_tail.rb +142 -53
  56. data/lib/fluent/plugin/in_tcp.rb +5 -6
  57. data/lib/fluent/plugin/in_udp.rb +6 -2
  58. data/lib/fluent/plugin/in_unix.rb +1 -1
  59. data/lib/fluent/plugin/multi_output.rb +1 -0
  60. data/lib/fluent/plugin/out_copy.rb +25 -2
  61. data/lib/fluent/plugin/out_file.rb +26 -7
  62. data/lib/fluent/plugin/out_forward.rb +81 -42
  63. data/lib/fluent/plugin/out_secondary_file.rb +2 -2
  64. data/lib/fluent/plugin/out_stdout.rb +0 -1
  65. data/lib/fluent/plugin/out_stream.rb +1 -1
  66. data/lib/fluent/plugin/output.rb +221 -57
  67. data/lib/fluent/plugin/parser_apache.rb +1 -1
  68. data/lib/fluent/plugin/parser_apache2.rb +5 -1
  69. data/lib/fluent/plugin/parser_apache_error.rb +1 -1
  70. data/lib/fluent/plugin/parser_json.rb +10 -3
  71. data/lib/fluent/plugin/parser_ltsv.rb +7 -0
  72. data/lib/fluent/plugin/parser_multiline.rb +2 -1
  73. data/lib/fluent/plugin/parser_nginx.rb +1 -1
  74. data/lib/fluent/plugin/parser_none.rb +1 -0
  75. data/lib/fluent/plugin/parser_regexp.rb +15 -14
  76. data/lib/fluent/plugin/parser_syslog.rb +9 -5
  77. data/lib/fluent/plugin_helper.rb +2 -0
  78. data/lib/fluent/plugin_helper/cert_option.rb +28 -9
  79. data/lib/fluent/plugin_helper/compat_parameters.rb +3 -1
  80. data/lib/fluent/plugin_helper/counter.rb +51 -0
  81. data/lib/fluent/plugin_helper/event_loop.rb +9 -0
  82. data/lib/fluent/plugin_helper/record_accessor.rb +210 -0
  83. data/lib/fluent/plugin_helper/retry_state.rb +15 -7
  84. data/lib/fluent/plugin_helper/server.rb +87 -25
  85. data/lib/fluent/plugin_helper/socket_option.rb +5 -2
  86. data/lib/fluent/plugin_helper/timer.rb +8 -7
  87. data/lib/fluent/root_agent.rb +18 -9
  88. data/lib/fluent/supervisor.rb +63 -23
  89. data/lib/fluent/system_config.rb +30 -2
  90. data/lib/fluent/test/helpers.rb +1 -1
  91. data/lib/fluent/time.rb +15 -7
  92. data/lib/fluent/timezone.rb +26 -2
  93. data/lib/fluent/version.rb +1 -1
  94. data/templates/new_gem/README.md.erb +2 -2
  95. data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +1 -1
  96. data/templates/new_gem/lib/fluent/plugin/input.rb.erb +1 -1
  97. data/templates/new_gem/lib/fluent/plugin/output.rb.erb +1 -1
  98. data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +4 -4
  99. data/test/command/test_ca_generate.rb +70 -0
  100. data/test/command/test_fluentd.rb +2 -2
  101. data/test/command/test_plugin_config_formatter.rb +8 -7
  102. data/test/command/test_plugin_generator.rb +65 -39
  103. data/test/config/test_config_parser.rb +7 -2
  104. data/test/config/test_configurable.rb +7 -2
  105. data/test/config/test_configure_proxy.rb +41 -3
  106. data/test/config/test_dsl.rb +10 -10
  107. data/test/config/test_element.rb +10 -0
  108. data/test/config/test_literal_parser.rb +8 -0
  109. data/test/config/test_plugin_configuration.rb +56 -0
  110. data/test/config/test_system_config.rb +19 -1
  111. data/test/config/test_types.rb +37 -0
  112. data/test/counter/test_client.rb +559 -0
  113. data/test/counter/test_error.rb +44 -0
  114. data/test/counter/test_mutex_hash.rb +179 -0
  115. data/test/counter/test_server.rb +589 -0
  116. data/test/counter/test_store.rb +258 -0
  117. data/test/counter/test_validator.rb +137 -0
  118. data/test/plugin/test_buf_file.rb +124 -0
  119. data/test/plugin/test_buffer.rb +3 -2
  120. data/test/plugin/test_filter_grep.rb +580 -2
  121. data/test/plugin/test_filter_parser.rb +33 -2
  122. data/test/plugin/test_filter_record_transformer.rb +22 -1
  123. data/test/plugin/test_formatter_ltsv.rb +3 -0
  124. data/test/plugin/test_formatter_tsv.rb +68 -0
  125. data/test/plugin/test_in_debug_agent.rb +21 -0
  126. data/test/plugin/test_in_exec.rb +3 -5
  127. data/test/plugin/test_in_http.rb +178 -0
  128. data/test/plugin/test_in_monitor_agent.rb +1 -1
  129. data/test/plugin/test_in_syslog.rb +64 -0
  130. data/test/plugin/test_in_tail.rb +116 -6
  131. data/test/plugin/test_in_tcp.rb +21 -0
  132. data/test/plugin/test_in_udp.rb +78 -0
  133. data/test/plugin/test_metadata.rb +89 -0
  134. data/test/plugin/test_out_copy.rb +31 -0
  135. data/test/plugin/test_out_file.rb +108 -2
  136. data/test/plugin/test_out_forward.rb +195 -2
  137. data/test/plugin/test_out_secondary_file.rb +14 -0
  138. data/test/plugin/test_output.rb +159 -45
  139. data/test/plugin/test_output_as_buffered.rb +19 -0
  140. data/test/plugin/test_output_as_buffered_backup.rb +307 -0
  141. data/test/plugin/test_output_as_buffered_retries.rb +70 -0
  142. data/test/plugin/test_output_as_buffered_secondary.rb +1 -1
  143. data/test/plugin/test_parser_apache2.rb +1 -0
  144. data/test/plugin/test_parser_labeled_tsv.rb +17 -0
  145. data/test/plugin/test_parser_nginx.rb +40 -0
  146. data/test/plugin/test_parser_regexp.rb +6 -7
  147. data/test/plugin/test_parser_syslog.rb +155 -5
  148. data/test/plugin_helper/test_child_process.rb +4 -4
  149. data/test/plugin_helper/test_compat_parameters.rb +22 -0
  150. data/test/plugin_helper/test_record_accessor.rb +197 -0
  151. data/test/plugin_helper/test_retry_state.rb +20 -0
  152. data/test/plugin_helper/test_server.rb +30 -2
  153. data/test/test_config.rb +3 -3
  154. data/test/test_configdsl.rb +2 -2
  155. data/test/test_log.rb +51 -1
  156. data/test/test_root_agent.rb +33 -0
  157. data/test/test_supervisor.rb +105 -0
  158. metadata +68 -8
  159. data/COPYING +0 -14
@@ -0,0 +1,205 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/config'
18
+ require 'fluent/counter/error'
19
+ require 'fluent/plugin/storage_local'
20
+ require 'fluent/time'
21
+
22
+ module Fluent
23
+ module Counter
24
+ class Store
25
+ def self.gen_key(scope, key)
26
+ "#{scope}\t#{key}"
27
+ end
28
+
29
+ def initialize(opt = {})
30
+ @log = opt[:log] || $log
31
+
32
+ # Notice: This storage is not be implemented auto save.
33
+ @storage = Plugin.new_storage('local', parent: DummyParent.new(@log))
34
+ conf = if opt[:path]
35
+ {'persistent' => true, 'path' => opt[:path] }
36
+ else
37
+ {'persistent' => false }
38
+ end
39
+ @storage.configure(Fluent::Config::Element.new('storage', {}, conf, []))
40
+ end
41
+
42
+ # This class behaves as a configurable plugin for using in storage (OwnedByMixin).
43
+ class DummyParent
44
+ include Configurable
45
+
46
+ attr_reader :log
47
+
48
+ def initialize(log)
49
+ @log = log
50
+ end
51
+
52
+ def plugin_id
53
+ 'dummy_parent_store'
54
+ end
55
+
56
+ def plugin_id_configured?
57
+ false
58
+ end
59
+
60
+ # storage_local calls PluginId#plugin_root_dir
61
+ def plugin_root_dir
62
+ nil
63
+ end
64
+ end
65
+
66
+ def start
67
+ @storage.load
68
+ end
69
+
70
+ def stop
71
+ @storage.save
72
+ end
73
+
74
+ def init(key, data, ignore: false)
75
+ ret = if v = get(key)
76
+ raise InvalidParams.new("#{key} already exists in counter") unless ignore
77
+ v
78
+ else
79
+ @storage.put(key, build_value(data))
80
+ end
81
+
82
+ build_response(ret)
83
+ end
84
+
85
+ def get(key, raise_error: false, raw: false)
86
+ ret = if raise_error
87
+ @storage.get(key) or raise UnknownKey.new("`#{key}` doesn't exist in counter")
88
+ else
89
+ @storage.get(key)
90
+ end
91
+ if raw
92
+ ret
93
+ else
94
+ ret && build_response(ret)
95
+ end
96
+ end
97
+
98
+ def key?(key)
99
+ !!@storage.get(key)
100
+ end
101
+
102
+ def delete(key)
103
+ ret = @storage.delete(key) or raise UnknownKey.new("`#{key}` doesn't exist in counter")
104
+ build_response(ret)
105
+ end
106
+
107
+ def inc(key, data, force: false)
108
+ value = data.delete('value')
109
+ init(key, data) if !key?(key) && force
110
+ v = get(key, raise_error: true, raw: true)
111
+ valid_type!(v, value)
112
+
113
+ v['total'] += value
114
+ v['current'] += value
115
+ t = EventTime.now
116
+ v['last_modified_at'] = [t.sec, t.nsec]
117
+ @storage.put(key, v)
118
+
119
+ build_response(v)
120
+ end
121
+
122
+ def reset(key)
123
+ v = get(key, raise_error: true, raw: true)
124
+ success = false
125
+ old_data = v.dup
126
+ now = EventTime.now
127
+ last_reset_at = EventTime.new(*v['last_reset_at'])
128
+
129
+ # Does it need reset?
130
+ if (last_reset_at + v['reset_interval']) <= now
131
+ success = true
132
+ v['current'] = initial_value(v['type'])
133
+ t = [now.sec, now.nsec]
134
+ v['last_reset_at'] = t
135
+ v['last_modified_at'] = t
136
+ @storage.put(key, v)
137
+ end
138
+
139
+ {
140
+ 'elapsed_time' => now - last_reset_at,
141
+ 'success' => success,
142
+ 'counter_data' => build_response(old_data)
143
+ }
144
+ end
145
+
146
+ private
147
+
148
+ def build_response(d)
149
+ {
150
+ 'name' => d['name'],
151
+ 'total' => d['total'],
152
+ 'current' => d['current'],
153
+ 'type' => d['type'],
154
+ 'reset_interval' => d['reset_interval'],
155
+ 'last_reset_at' => EventTime.new(*d['last_reset_at']),
156
+ }
157
+ end
158
+
159
+ # value is Hash. value requires these fileds.
160
+ # :name, :total, :current, :type, :reset_interval, :last_reset_at, :last_modified_at
161
+ def build_value(data)
162
+ type = data['type'] || 'numeric'
163
+ now = EventTime.now
164
+ t = [now.sec, now.nsec]
165
+
166
+ v = initial_value(type)
167
+
168
+ data.merge(
169
+ 'type' => type,
170
+ 'last_reset_at' => t,
171
+ 'last_modified_at' => t,
172
+ 'current' => v,
173
+ 'total' => v,
174
+ )
175
+ end
176
+
177
+ def initial_value(type)
178
+ case type
179
+ when 'numeric', 'integer' then 0
180
+ when 'float' then 0.0
181
+ else raise InvalidParams.new('`type` should be integer, float, or numeric')
182
+ end
183
+ end
184
+
185
+ def valid_type!(v, value)
186
+ type = v['type']
187
+ return unless (type != 'numeric') && (type_str(value) != type)
188
+ raise InvalidParams.new("`type` is #{type}. You should pass #{type} value as a `value`")
189
+ end
190
+
191
+ def type_str(v)
192
+ case v
193
+ when Integer
194
+ 'integer'
195
+ when Float
196
+ 'float'
197
+ when Numeric
198
+ 'numeric'
199
+ else
200
+ raise InvalidParams.new("`type` should be integer, float, or numeric")
201
+ end
202
+ end
203
+ end
204
+ end
205
+ end
@@ -0,0 +1,145 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/counter/error'
18
+
19
+ module Fluent
20
+ module Counter
21
+ class Validator
22
+ VALID_NAME = /\A[a-z][a-zA-Z0-9\-_]*\Z/
23
+ VALID_SCOPE_NAME = /\A[a-z][\ta-zA-Z0-9\-_]*\Z/
24
+ VALID_METHODS = %w(establish init delete inc get reset)
25
+
26
+ def self.request(data)
27
+ errors = []
28
+ raise "Received data is not Hash: #{data}" unless data.is_a?(Hash)
29
+
30
+ unless data['id']
31
+ errors << Fluent::Counter::InvalidRequest.new('Request should include `id`')
32
+ end
33
+
34
+ if !data['method']
35
+ errors << Fluent::Counter::InvalidRequest.new('Request should include `method`')
36
+ elsif !(VALID_NAME =~ data['method'])
37
+ errors << Fluent::Counter::InvalidRequest.new('`method` is the invalid format')
38
+ elsif !VALID_METHODS.include?(data['method'])
39
+ errors << Fluent::Counter::MethodNotFound.new("Unknown method name passed: #{data['method']}")
40
+ end
41
+
42
+ errors.map(&:to_hash)
43
+ end
44
+
45
+ def initialize(*types)
46
+ @types = types.map(&:to_s)
47
+ @empty = @types.delete('empty')
48
+ end
49
+
50
+ def call(data)
51
+ success = []
52
+ errors = []
53
+
54
+ if @empty && data.empty?
55
+ errors << Fluent::Counter::InvalidParams.new('One or more `params` are required')
56
+ else
57
+ data.each do |d|
58
+ begin
59
+ @types.each { |type| dispatch(type, d) }
60
+ success << d
61
+ rescue => e
62
+ errors << e
63
+ end
64
+ end
65
+ end
66
+
67
+ [success, errors]
68
+ end
69
+
70
+ private
71
+
72
+ def dispatch(type, data)
73
+ send("validate_#{type}!", data)
74
+ rescue NoMethodError => e
75
+ raise Fluent::Counter::InternalServerError.new(e)
76
+ end
77
+ end
78
+
79
+ class ArrayValidator < Validator
80
+ def validate_key!(name)
81
+ unless name.is_a?(String)
82
+ raise Fluent::Counter::InvalidParams.new('The type of `key` should be String')
83
+ end
84
+
85
+ unless VALID_NAME =~ name
86
+ raise Fluent::Counter::InvalidParams.new('`key` is the invalid format')
87
+ end
88
+ end
89
+
90
+ def validate_scope!(name)
91
+ unless name.is_a?(String)
92
+ raise Fluent::Counter::InvalidParams.new('The type of `scope` should be String')
93
+ end
94
+
95
+ unless VALID_SCOPE_NAME =~ name
96
+ raise Fluent::Counter::InvalidParams.new('`scope` is the invalid format')
97
+ end
98
+ end
99
+ end
100
+
101
+ class HashValidator < Validator
102
+ def validate_name!(hash)
103
+ name = hash['name']
104
+ unless name
105
+ raise Fluent::Counter::InvalidParams.new('`name` is required')
106
+ end
107
+
108
+ unless name.is_a?(String)
109
+ raise Fluent::Counter::InvalidParams.new('The type of `name` should be String')
110
+ end
111
+
112
+ unless VALID_NAME =~ name
113
+ raise Fluent::Counter::InvalidParams.new("`name` is the invalid format")
114
+ end
115
+ end
116
+
117
+ def validate_value!(hash)
118
+ value = hash['value']
119
+ unless value
120
+ raise Fluent::Counter::InvalidParams.new('`value` is required')
121
+ end
122
+
123
+ unless value.is_a?(Numeric)
124
+ raise Fluent::Counter::InvalidParams.new("The type of `value` type should be Numeric")
125
+ end
126
+ end
127
+
128
+ def validate_reset_interval!(hash)
129
+ interval = hash['reset_interval']
130
+
131
+ unless interval
132
+ raise Fluent::Counter::InvalidParams.new('`reset_interval` is required')
133
+ end
134
+
135
+ unless interval.is_a?(Numeric)
136
+ raise Fluent::Counter::InvalidParams.new('The type of `reset_interval` should be Numeric')
137
+ end
138
+
139
+ if interval < 0
140
+ raise Fluent::Counter::InvalidParams.new('`reset_interval` should be a positive number')
141
+ end
142
+ end
143
+ end
144
+ end
145
+ end
data/lib/fluent/env.rb CHANGED
@@ -20,6 +20,7 @@ module Fluent
20
20
  DEFAULT_CONFIG_PATH = ENV['FLUENT_CONF'] || '/etc/fluent/fluent.conf'
21
21
  DEFAULT_PLUGIN_DIR = ENV['FLUENT_PLUGIN'] || '/etc/fluent/plugin'
22
22
  DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
23
+ DEFAULT_BACKUP_DIR = ENV['FLUENT_BACKUP_DIR'] || '/tmp/fluent'
23
24
  DEFAULT_OJ_OPTIONS = {bigdecimal_load: :float, mode: :compat, use_to_json: true}
24
25
 
25
26
  def self.windows?
@@ -217,7 +217,7 @@ module Fluent
217
217
  @optimizable = if fs_filters.empty?
218
218
  true
219
219
  else
220
- # skip log message when filter is only 1, because its performace is same as non optimized chain.
220
+ # skip log message when filter is only 1, because its performance is same as non optimized chain.
221
221
  if @filters.size > 1 && fs_filters.size >= 1
222
222
  $log.info "disable filter chain optimization because #{fs_filters.map(&:class)} uses `#filter_stream` method."
223
223
  end
data/lib/fluent/log.rb CHANGED
@@ -96,8 +96,12 @@ module Fluent
96
96
  @level = logger.level + 1
97
97
  @debug_mode = false
98
98
  @log_event_enabled = false
99
- @time_format = '%Y-%m-%d %H:%M:%S %z '
100
99
  @depth_offset = 1
100
+ @format = nil
101
+ @time_format = nil
102
+ @formatter = nil
103
+
104
+ self.format = :text
101
105
  enable_color out.tty?
102
106
  # TODO: This variable name is unclear so we should change to better name.
103
107
  @threads_exclude_events = []
@@ -136,12 +140,14 @@ module Fluent
136
140
  dl_opts[:log_level] = @level - 1
137
141
  logger = ServerEngine::DaemonLogger.new(@out, dl_opts)
138
142
  clone = self.class.new(logger, suppress_repeated_stacktrace: @suppress_repeated_stacktrace, process_type: @process_type, worker_id: @worker_id)
143
+ clone.format = @format
139
144
  clone.time_format = @time_format
140
145
  clone.log_event_enabled = @log_event_enabled
141
146
  # optional headers/attrs are not copied, because new PluginLogger should have another one of it
142
147
  clone
143
148
  end
144
149
 
150
+ attr_reader :format
145
151
  attr_accessor :log_event_enabled
146
152
  attr_accessor :out
147
153
  attr_accessor :level
@@ -154,6 +160,43 @@ module Fluent
154
160
  nil
155
161
  end
156
162
 
163
+ def format=(fmt)
164
+ return if @format == fmt
165
+
166
+ @time_format = '%Y-%m-%d %H:%M:%S %z'
167
+ @time_formatter = Strftime.new(@time_format) rescue nil
168
+
169
+ case fmt
170
+ when :text
171
+ @format = :text
172
+ @formatter = Proc.new { |type, time, level, msg|
173
+ r = caller_line(type, time, @depth_offset, level)
174
+ r << msg
175
+ r
176
+ }
177
+ when :json
178
+ @format = :json
179
+ @formatter = Proc.new { |type, time, level, msg|
180
+ r = {
181
+ 'time' => format_time(time),
182
+ 'level' => LEVEL_TEXT[level],
183
+ 'message' => msg
184
+ }
185
+ if wid = get_worker_id(type)
186
+ r['worker_id'] = wid
187
+ end
188
+ Yajl.dump(r)
189
+ }
190
+ end
191
+
192
+ nil
193
+ end
194
+
195
+ def time_format=(time_fmt)
196
+ @time_format = time_fmt
197
+ @time_formatter = Strftime.new(@time_format) rescue nil
198
+ end
199
+
157
200
  def reopen!
158
201
  # do nothing in @logger.reopen! because it's already reopened in Supervisor.load_config
159
202
  @logger.reopen! if @logger
@@ -224,9 +267,9 @@ module Fluent
224
267
  end
225
268
  end
226
269
 
227
- def on_trace(&block)
270
+ def on_trace
228
271
  return if @level > LEVEL_TRACE
229
- block.call if block
272
+ yield
230
273
  end
231
274
 
232
275
  def trace(*args, &block)
@@ -235,7 +278,7 @@ module Fluent
235
278
  return if skipped_type?(type)
236
279
  args << block.call if block
237
280
  time, msg = event(:trace, args)
238
- puts [@color_trace, caller_line(type, time, @depth_offset, LEVEL_TRACE), msg, @color_reset].join
281
+ puts [@color_trace, @formatter.call(type, time, LEVEL_TRACE, msg), @color_reset].join
239
282
  rescue
240
283
  # logger should not raise an exception. This rescue prevents unexpected behaviour.
241
284
  end
@@ -245,9 +288,9 @@ module Fluent
245
288
  dump_stacktrace(type, backtrace, LEVEL_TRACE)
246
289
  end
247
290
 
248
- def on_debug(&block)
291
+ def on_debug
249
292
  return if @level > LEVEL_DEBUG
250
- block.call if block
293
+ yield
251
294
  end
252
295
 
253
296
  def debug(*args, &block)
@@ -256,7 +299,7 @@ module Fluent
256
299
  return if skipped_type?(type)
257
300
  args << block.call if block
258
301
  time, msg = event(:debug, args)
259
- puts [@color_debug, caller_line(type, time, @depth_offset, LEVEL_DEBUG), msg, @color_reset].join
302
+ puts [@color_debug, @formatter.call(type, time, LEVEL_DEBUG, msg), @color_reset].join
260
303
  rescue
261
304
  end
262
305
  alias DEBUG debug
@@ -265,9 +308,9 @@ module Fluent
265
308
  dump_stacktrace(type, backtrace, LEVEL_DEBUG)
266
309
  end
267
310
 
268
- def on_info(&block)
311
+ def on_info
269
312
  return if @level > LEVEL_INFO
270
- block.call if block
313
+ yield
271
314
  end
272
315
 
273
316
  def info(*args, &block)
@@ -276,7 +319,7 @@ module Fluent
276
319
  return if skipped_type?(type)
277
320
  args << block.call if block
278
321
  time, msg = event(:info, args)
279
- puts [@color_info, caller_line(type, time, @depth_offset, LEVEL_INFO), msg, @color_reset].join
322
+ puts [@color_info, @formatter.call(type, time, LEVEL_INFO, msg), @color_reset].join
280
323
  rescue
281
324
  end
282
325
  alias INFO info
@@ -285,9 +328,9 @@ module Fluent
285
328
  dump_stacktrace(type, backtrace, LEVEL_INFO)
286
329
  end
287
330
 
288
- def on_warn(&block)
331
+ def on_warn
289
332
  return if @level > LEVEL_WARN
290
- block.call if block
333
+ yield
291
334
  end
292
335
 
293
336
  def warn(*args, &block)
@@ -296,7 +339,7 @@ module Fluent
296
339
  return if skipped_type?(type)
297
340
  args << block.call if block
298
341
  time, msg = event(:warn, args)
299
- puts [@color_warn, caller_line(type, time, @depth_offset, LEVEL_WARN), msg, @color_reset].join
342
+ puts [@color_warn, @formatter.call(type, time, LEVEL_WARN, msg), @color_reset].join
300
343
  rescue
301
344
  end
302
345
  alias WARN warn
@@ -305,9 +348,9 @@ module Fluent
305
348
  dump_stacktrace(type, backtrace, LEVEL_WARN)
306
349
  end
307
350
 
308
- def on_error(&block)
351
+ def on_error
309
352
  return if @level > LEVEL_ERROR
310
- block.call if block
353
+ yield
311
354
  end
312
355
 
313
356
  def error(*args, &block)
@@ -316,7 +359,7 @@ module Fluent
316
359
  return if skipped_type?(type)
317
360
  args << block.call if block
318
361
  time, msg = event(:error, args)
319
- puts [@color_error, caller_line(type, time, @depth_offset, LEVEL_ERROR), msg, @color_reset].join
362
+ puts [@color_error, @formatter.call(type, time, LEVEL_ERROR, msg), @color_reset].join
320
363
  rescue
321
364
  end
322
365
  alias ERROR error
@@ -325,9 +368,9 @@ module Fluent
325
368
  dump_stacktrace(type, backtrace, LEVEL_ERROR)
326
369
  end
327
370
 
328
- def on_fatal(&block)
371
+ def on_fatal
329
372
  return if @level > LEVEL_FATAL
330
- block.call if block
373
+ yield
331
374
  end
332
375
 
333
376
  def fatal(*args, &block)
@@ -336,7 +379,7 @@ module Fluent
336
379
  return if skipped_type?(type)
337
380
  args << block.call if block
338
381
  time, msg = event(:fatal, args)
339
- puts [@color_fatal, caller_line(type, time, @depth_offset, LEVEL_FATAL), msg, @color_reset].join
382
+ puts [@color_fatal, @formatter.call(type, time, LEVEL_FATAL, msg), @color_reset].join
340
383
  rescue
341
384
  end
342
385
  alias FATAL fatal
@@ -373,19 +416,47 @@ module Fluent
373
416
  return if @level > level
374
417
 
375
418
  time = Time.now
376
- line = caller_line(type, time, 5, level)
377
- if @suppress_repeated_stacktrace && (Thread.current[:last_repeated_stacktrace] == backtrace)
378
- puts [" ", line, 'suppressed same stacktrace'].join
419
+
420
+ if @format == :text
421
+ line = caller_line(type, time, 5, level)
422
+ if @suppress_repeated_stacktrace && (Thread.current[:last_repeated_stacktrace] == backtrace)
423
+ puts [" ", line, 'suppressed same stacktrace'].join
424
+ else
425
+ backtrace.each { |msg|
426
+ puts [" ", line, msg].join
427
+ }
428
+ Thread.current[:last_repeated_stacktrace] = backtrace if @suppress_repeated_stacktrace
429
+ end
379
430
  else
380
- backtrace.each { |msg|
381
- puts [" ", line, msg].join
431
+ r = {
432
+ 'time' => format_time(time),
433
+ 'level' => LEVEL_TEXT[level],
382
434
  }
383
- Thread.current[:last_repeated_stacktrace] = backtrace if @suppress_repeated_stacktrace
435
+ if wid = get_worker_id(type)
436
+ r['worker_id'] = wid
437
+ end
438
+
439
+ if @suppress_repeated_stacktrace && (Thread.current[:last_repeated_stacktrace] == backtrace)
440
+ r['message'] = 'suppressed same stacktrace'
441
+ else
442
+ r['message'] = backtrace.join("\n")
443
+ Thread.current[:last_repeated_stacktrace] = backtrace if @suppress_repeated_stacktrace
444
+ end
445
+
446
+ puts Yajl.dump(r)
384
447
  end
385
448
 
386
449
  nil
387
450
  end
388
451
 
452
+ def get_worker_id(type)
453
+ if type == :default && (@process_type == :worker0 || @process_type == :workers)
454
+ @worker_id
455
+ else
456
+ nil
457
+ end
458
+ end
459
+
389
460
  def event(level, args)
390
461
  time = Time.now
391
462
  message = @optional_header ? @optional_header.dup : ''
@@ -426,7 +497,7 @@ module Fluent
426
497
  else
427
498
  "".freeze
428
499
  end
429
- log_msg = "#{time.strftime(@time_format)}[#{LEVEL_TEXT[level]}]: #{worker_id_part}"
500
+ log_msg = "#{format_time(time)} [#{LEVEL_TEXT[level]}]: #{worker_id_part}"
430
501
  if @debug_mode
431
502
  line = caller(depth+1)[0]
432
503
  if match = /^(.+?):(\d+)(?::in `(.*)')?/.match(line)
@@ -438,6 +509,10 @@ module Fluent
438
509
  end
439
510
  return log_msg
440
511
  end
512
+
513
+ def format_time(time)
514
+ @time_formatter ? @time_formatter.exec(time) : time.strftime(@time_format)
515
+ end
441
516
  end
442
517
 
443
518
 
@@ -450,11 +525,13 @@ module Fluent
450
525
  def initialize(logger)
451
526
  @logger = logger
452
527
  @level = @logger.level
528
+ @format = nil
453
529
  @depth_offset = 2
454
530
  if logger.instance_variable_defined?(:@suppress_repeated_stacktrace)
455
531
  @suppress_repeated_stacktrace = logger.instance_variable_get(:@suppress_repeated_stacktrace)
456
532
  end
457
533
 
534
+ self.format = @logger.format
458
535
  enable_color @logger.enable_color?
459
536
  end
460
537
 
@@ -462,18 +539,24 @@ module Fluent
462
539
  @level = Log.str_to_level(log_level_str)
463
540
  end
464
541
 
542
+ alias orig_format= format=
465
543
  alias orig_enable_color enable_color
466
544
 
545
+ def format=(fmt)
546
+ self.orig_format = fmt
547
+ @logger.format = fmt
548
+ end
549
+
467
550
  def enable_color(b = true)
468
551
  orig_enable_color b
469
552
  @logger.enable_color b
470
553
  end
471
554
 
472
555
  extend Forwardable
473
- def_delegators '@logger', :enable_color?, :enable_debug, :enable_event,
556
+ def_delegators '@logger', :get_worker_id, :enable_color?, :enable_debug, :enable_event,
474
557
  :disable_events, :log_event_enabled, :log_event_enamed=, :time_format, :time_format=,
475
- :event, :caller_line, :puts, :write, :<<, :flush, :reset, :out, :out=,
476
- :optional_header, :optional_header=, :optional_attrs, :optional_attrs=
558
+ :time_formatter, :time_formatter=, :event, :caller_line, :puts, :write, :<<, :flush,
559
+ :reset, :out, :out=, :optional_header, :optional_header=, :optional_attrs, :optional_attrs=
477
560
  end
478
561
 
479
562
 
@@ -542,5 +625,12 @@ module Fluent
542
625
  super
543
626
  end
544
627
  end
628
+
629
+ def reopen(path, mode)
630
+ if mode != 'a'
631
+ raise "Unsupported mode: #{mode}"
632
+ end
633
+ super(path)
634
+ end
545
635
  end
546
636
  end