fluentd 0.14.5-x64-mingw32 → 0.14.7-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (103) hide show
  1. checksums.yaml +4 -4
  2. data/ChangeLog +55 -0
  3. data/bin/fluent-binlog-reader +7 -0
  4. data/example/in_dummy_with_compression.conf +23 -0
  5. data/lib/fluent/agent.rb +8 -12
  6. data/lib/fluent/command/binlog_reader.rb +234 -0
  7. data/lib/fluent/command/fluentd.rb +17 -1
  8. data/lib/fluent/compat/file_util.rb +1 -1
  9. data/lib/fluent/compat/output.rb +5 -1
  10. data/lib/fluent/config/configure_proxy.rb +18 -3
  11. data/lib/fluent/config/element.rb +1 -1
  12. data/lib/fluent/config/section.rb +1 -1
  13. data/lib/fluent/config/v1_parser.rb +1 -1
  14. data/lib/fluent/env.rb +1 -0
  15. data/lib/fluent/event.rb +49 -2
  16. data/lib/fluent/event_router.rb +6 -2
  17. data/lib/fluent/label.rb +8 -0
  18. data/lib/fluent/log.rb +30 -1
  19. data/lib/fluent/plugin.rb +1 -1
  20. data/lib/fluent/plugin/base.rb +3 -0
  21. data/lib/fluent/plugin/buf_file.rb +2 -2
  22. data/lib/fluent/plugin/buf_memory.rb +1 -1
  23. data/lib/fluent/plugin/buffer.rb +12 -2
  24. data/lib/fluent/plugin/buffer/chunk.rb +68 -7
  25. data/lib/fluent/plugin/buffer/file_chunk.rb +4 -4
  26. data/lib/fluent/plugin/buffer/memory_chunk.rb +4 -4
  27. data/lib/fluent/plugin/compressable.rb +91 -0
  28. data/lib/fluent/plugin/filter_grep.rb +4 -4
  29. data/lib/fluent/plugin/formatter.rb +2 -2
  30. data/lib/fluent/plugin/formatter_json.rb +2 -1
  31. data/lib/fluent/plugin/formatter_out_file.rb +3 -30
  32. data/lib/fluent/plugin/in_forward.rb +6 -5
  33. data/lib/fluent/plugin/in_monitor_agent.rb +7 -21
  34. data/lib/fluent/plugin/in_syslog.rb +1 -1
  35. data/lib/fluent/plugin/in_tail.rb +11 -2
  36. data/lib/fluent/plugin/multi_output.rb +63 -3
  37. data/lib/fluent/plugin/out_exec.rb +1 -1
  38. data/lib/fluent/plugin/out_file.rb +5 -1
  39. data/lib/fluent/plugin/out_forward.rb +17 -5
  40. data/lib/fluent/plugin/out_stdout.rb +2 -1
  41. data/lib/fluent/plugin/output.rb +205 -19
  42. data/lib/fluent/plugin/parser.rb +5 -49
  43. data/lib/fluent/plugin/parser_apache2.rb +1 -1
  44. data/lib/fluent/plugin/parser_json.rb +4 -4
  45. data/lib/fluent/plugin/parser_multiline.rb +5 -5
  46. data/lib/fluent/plugin/parser_regexp.rb +1 -2
  47. data/lib/fluent/plugin/parser_syslog.rb +2 -2
  48. data/lib/fluent/plugin/storage_local.rb +2 -1
  49. data/lib/fluent/plugin_helper.rb +1 -0
  50. data/lib/fluent/plugin_helper/compat_parameters.rb +39 -21
  51. data/lib/fluent/plugin_helper/extract.rb +92 -0
  52. data/lib/fluent/plugin_helper/inject.rb +10 -12
  53. data/lib/fluent/plugin_helper/thread.rb +23 -3
  54. data/lib/fluent/registry.rb +1 -1
  55. data/lib/fluent/root_agent.rb +2 -1
  56. data/lib/fluent/supervisor.rb +28 -8
  57. data/lib/fluent/test/base.rb +0 -7
  58. data/lib/fluent/test/driver/base.rb +1 -0
  59. data/lib/fluent/test/driver/output.rb +3 -0
  60. data/lib/fluent/test/helpers.rb +18 -0
  61. data/lib/fluent/test/input_test.rb +4 -2
  62. data/lib/fluent/test/log.rb +3 -1
  63. data/lib/fluent/time.rb +232 -1
  64. data/lib/fluent/timezone.rb +1 -1
  65. data/lib/fluent/version.rb +1 -1
  66. data/test/command/test_binlog_reader.rb +351 -0
  67. data/test/config/test_config_parser.rb +6 -0
  68. data/test/config/test_configurable.rb +47 -1
  69. data/test/helper.rb +0 -1
  70. data/test/plugin/test_buffer.rb +22 -2
  71. data/test/plugin/test_buffer_chunk.rb +34 -4
  72. data/test/plugin/test_buffer_file_chunk.rb +73 -0
  73. data/test/plugin/test_buffer_memory_chunk.rb +73 -0
  74. data/test/plugin/test_compressable.rb +81 -0
  75. data/test/plugin/test_formatter_json.rb +14 -1
  76. data/test/plugin/test_in_forward.rb +67 -3
  77. data/test/plugin/test_in_monitor_agent.rb +17 -1
  78. data/test/plugin/test_in_tail.rb +8 -8
  79. data/test/plugin/test_out_file.rb +0 -8
  80. data/test/plugin/test_out_forward.rb +85 -0
  81. data/test/plugin/test_out_secondary_file.rb +20 -12
  82. data/test/plugin/test_out_stdout.rb +11 -10
  83. data/test/plugin/test_output.rb +234 -0
  84. data/test/plugin/test_output_as_buffered.rb +223 -0
  85. data/test/plugin/test_output_as_buffered_compress.rb +165 -0
  86. data/test/plugin/test_parser_json.rb +8 -0
  87. data/test/plugin/test_parser_regexp.rb +1 -1
  88. data/test/plugin_helper/test_child_process.rb +2 -2
  89. data/test/plugin_helper/test_extract.rb +195 -0
  90. data/test/plugin_helper/test_inject.rb +0 -7
  91. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +7 -0
  92. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +7 -0
  93. data/test/test_event.rb +186 -0
  94. data/test/test_event_router.rb +1 -1
  95. data/test/test_formatter.rb +0 -7
  96. data/test/test_log.rb +121 -0
  97. data/test/test_plugin_classes.rb +62 -0
  98. data/test/test_root_agent.rb +125 -0
  99. data/test/test_supervisor.rb +25 -2
  100. data/test/test_time_formatter.rb +103 -7
  101. data/test/test_time_parser.rb +211 -0
  102. metadata +22 -4
  103. data/test/plugin/test_parser_time.rb +0 -46
@@ -21,12 +21,11 @@ require 'fluent/mixin' # for TypeConverter
21
21
  require 'fluent/time'
22
22
  require 'fluent/plugin/string_util'
23
23
 
24
- require 'strptime'
25
-
26
24
  module Fluent
27
25
  module Plugin
28
26
  class Parser < Base
29
27
  include OwnedByMixin
28
+ include TimeMixin::Parser
30
29
 
31
30
  class ParserError < StandardError; end
32
31
 
@@ -52,49 +51,7 @@ module Fluent
52
51
  parse(*a, &b)
53
52
  end
54
53
 
55
- class TimeParser
56
- def initialize(time_format)
57
- @cache1_key = nil
58
- @cache1_time = nil
59
- @cache2_key = nil
60
- @cache2_time = nil
61
- @parser =
62
- if time_format
63
- begin
64
- strptime = Strptime.new(time_format)
65
- Proc.new { |value| Fluent::EventTime.from_time(strptime.exec(value)) }
66
- rescue
67
- Proc.new { |value| Fluent::EventTime.from_time(Time.strptime(value, time_format)) }
68
- end
69
- else
70
- Proc.new { |value| Fluent::EventTime.parse(value) }
71
- end
72
- end
73
-
74
- # TODO: new cache mechanism using format string
75
- def parse(value)
76
- unless value.is_a?(String)
77
- raise ParserError, "value must be string: #{value}"
78
- end
79
-
80
- if @cache1_key == value
81
- return @cache1_time
82
- elsif @cache2_key == value
83
- return @cache2_time
84
- else
85
- begin
86
- time = @parser.call(value)
87
- rescue => e
88
- raise ParserError, "invalid time format: value = #{value}, error_class = #{e.class.name}, error = #{e.message}"
89
- end
90
- @cache1_key = @cache2_key
91
- @cache1_time = @cache2_time
92
- @cache2_key = value
93
- @cache2_time = time
94
- return time
95
- end
96
- end
97
- end
54
+ TimeParser = Fluent::TimeParser
98
55
  end
99
56
 
100
57
  class ValuesParser < Parser
@@ -102,7 +59,6 @@ module Fluent
102
59
 
103
60
  config_param :keys, :array, default: []
104
61
  config_param :time_key, :string, default: nil
105
- config_param :time_format, :string, default: nil
106
62
  config_param :null_value_pattern, :string, default: nil
107
63
  config_param :null_empty_string, :bool, default: false
108
64
 
@@ -110,14 +66,14 @@ module Fluent
110
66
  super
111
67
 
112
68
  if @time_key && !@keys.include?(@time_key) && @estimate_current_event
113
- raise ConfigError, "time_key (#{@time_key.inspect}) is not included in keys (#{@keys.inspect})"
69
+ raise Fluent::ConfigError, "time_key (#{@time_key.inspect}) is not included in keys (#{@keys.inspect})"
114
70
  end
115
71
 
116
72
  if @time_format && !@time_key
117
- raise ConfigError, "time_format parameter is ignored because time_key parameter is not set. at #{conf.inspect}"
73
+ raise Fluent::ConfigError, "time_format parameter is ignored because time_key parameter is not set. at #{conf.inspect}"
118
74
  end
119
75
 
120
- @time_parser = TimeParser.new(@time_format)
76
+ @time_parser = time_parser_create
121
77
 
122
78
  if @null_value_pattern
123
79
  @null_value_pattern = Regexp.new(@null_value_pattern)
@@ -26,7 +26,7 @@ module Fluent
26
26
 
27
27
  def initialize
28
28
  super
29
- @time_parser = TimeParser.new(TIME_FORMAT)
29
+ @time_parser = time_parser_create(format: TIME_FORMAT)
30
30
  @mutex = Mutex.new
31
31
  end
32
32
 
@@ -15,6 +15,7 @@
15
15
  #
16
16
 
17
17
  require 'fluent/plugin/parser'
18
+ require 'fluent/env'
18
19
  require 'fluent/time'
19
20
 
20
21
  require 'yajl'
@@ -25,21 +26,20 @@ module Fluent
25
26
  Plugin.register_parser('json', self)
26
27
 
27
28
  config_param :time_key, :string, default: 'time'
28
- config_param :time_format, :string, default: nil
29
29
  config_param :json_parser, :string, default: 'oj'
30
30
 
31
31
  def configure(conf)
32
32
  super
33
33
 
34
- unless @time_format.nil?
35
- @time_parser = TimeParser.new(@time_format)
34
+ if @time_format
35
+ @time_parser = time_parser_create
36
36
  @mutex = Mutex.new
37
37
  end
38
38
 
39
39
  begin
40
40
  raise LoadError unless @json_parser == 'oj'
41
41
  require 'oj'
42
- Oj.default_options = {bigdecimal_load: :float, mode: :strict}
42
+ Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
43
43
  @load_proc = Oj.method(:load)
44
44
  @error_class = Oj::ParseError
45
45
  rescue LoadError
@@ -38,7 +38,7 @@ module Fluent
38
38
  @parser = Fluent::Plugin::RegexpParser.new
39
39
  @parser.configure(conf + regexp_conf)
40
40
  rescue => e
41
- raise ConfigError, "Invalid regexp '#{formats}': #{e}"
41
+ raise Fluent::ConfigError, "Invalid regexp '#{formats}': #{e}"
42
42
  end
43
43
 
44
44
  if @format_firstline
@@ -68,7 +68,7 @@ module Fluent
68
68
  (1..FORMAT_MAX_NUM).map { |i|
69
69
  format = conf["format#{i}"]
70
70
  if (i > 1) && prev_format.nil? && !format.nil?
71
- raise ConfigError, "Jump of format index found. format#{i - 1} is missing."
71
+ raise Fluent::ConfigError, "Jump of format index found. format#{i - 1} is missing."
72
72
  end
73
73
  prev_format = format
74
74
  next if format.nil?
@@ -84,7 +84,7 @@ module Fluent
84
84
  m ? !((1..FORMAT_MAX_NUM).include?(m[1].to_i)) : false
85
85
  }
86
86
  unless invalid_formats.empty?
87
- raise ConfigError, "Invalid formatN found. N should be 1 - #{FORMAT_MAX_NUM}: " + invalid_formats.join(",")
87
+ raise Fluent::ConfigError, "Invalid formatN found. N should be 1 - #{FORMAT_MAX_NUM}: " + invalid_formats.join(",")
88
88
  end
89
89
  end
90
90
 
@@ -93,10 +93,10 @@ module Fluent
93
93
  begin
94
94
  Regexp.new(format[1..-2], Regexp::MULTILINE)
95
95
  rescue => e
96
- raise ConfigError, "Invalid regexp in #{key}: #{e}"
96
+ raise Fluent::ConfigError, "Invalid regexp in #{key}: #{e}"
97
97
  end
98
98
  else
99
- raise ConfigError, "format should be Regexp, need //, in #{key}: '#{format}'"
99
+ raise Fluent::ConfigError, "format should be Regexp, need //, in #{key}: '#{format}'"
100
100
  end
101
101
  end
102
102
  end
@@ -9,7 +9,6 @@ module Fluent
9
9
  config_param :ignorecase, :bool, default: false
10
10
  config_param :multiline, :bool, default: false
11
11
  config_param :time_key, :string, default: 'time'
12
- config_param :time_format, :string, default: nil
13
12
 
14
13
  def initialize
15
14
  super
@@ -18,7 +17,7 @@ module Fluent
18
17
 
19
18
  def configure(conf)
20
19
  super
21
- @time_parser = TimeParser.new(@time_format)
20
+ @time_parser = time_parser_create
22
21
  unless @expression.empty?
23
22
  if @expression[0] == "/" && @expression[-1] == "/"
24
23
  regexp_option = 0
@@ -28,7 +28,7 @@ module Fluent
28
28
  # From in_syslog default pattern
29
29
  REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
30
30
 
31
- config_param :time_format, :string, default: "%b %d %H:%M:%S"
31
+ config_set_default :time_format, "%b %d %H:%M:%S"
32
32
  config_param :with_priority, :bool, default: false
33
33
 
34
34
  def initialize
@@ -40,7 +40,7 @@ module Fluent
40
40
  super
41
41
 
42
42
  @regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
43
- @time_parser = TimeParser.new(@time_format)
43
+ @time_parser = time_parser_create
44
44
  end
45
45
 
46
46
  def patterns
@@ -56,7 +56,8 @@ module Fluent
56
56
  elsif @path
57
57
  # ok
58
58
  else # @_plugin_id_configured is true
59
- raise NotImplementedError, "implement this feature later with system_config"
59
+ log.warn "path for <storage> is not specified. Using on-memory store temporarily, but will use file store after support global storage path"
60
+ @on_memory = true
60
61
  ## TODO: get process-wide directory for plugin storage, and generate path for this plugin storage instance
61
62
  # path =
62
63
  end
@@ -23,6 +23,7 @@ require 'fluent/plugin_helper/storage'
23
23
  require 'fluent/plugin_helper/parser'
24
24
  require 'fluent/plugin_helper/formatter'
25
25
  require 'fluent/plugin_helper/inject'
26
+ require 'fluent/plugin_helper/extract'
26
27
  require 'fluent/plugin_helper/retry_state'
27
28
  require 'fluent/plugin_helper/compat_parameters'
28
29
 
@@ -44,6 +44,7 @@ module Fluent
44
44
  BUFFER_TIME_SLICED_PARAMS = {
45
45
  "time_slice_format" => nil,
46
46
  "time_slice_wait" => "timekey_wait",
47
+ "timezone" => "timekey_zone",
47
48
  }
48
49
 
49
50
  PARSER_PARAMS = {
@@ -56,6 +57,7 @@ module Fluent
56
57
  "format_firstline" => "format_firstline", # MultilineParser
57
58
  "message_key" => "message_key", # NoneParser
58
59
  "with_priority" => "with_priority", # SyslogParser
60
+ # There has been no parsers which can handle timezone in v0.12
59
61
  }
60
62
 
61
63
  INJECT_PARAMS = {
@@ -77,7 +79,10 @@ module Fluent
77
79
  "json_parser" => "json_parser", # JSONFormatter
78
80
  "label_delimiter" => "label_delimiter", # LabeledTSVFormatter
79
81
  "output_time" => "output_time", # OutFileFormatter
80
- "output_tag" => "output_tag", # OutFileFormatter
82
+ "output_tag" => "output_tag", # OutFileFormatter
83
+ "localtime" => "localtime", # OutFileFormatter
84
+ "utc" => "utc", # OutFileFormatter
85
+ "timezone" => "timezone", # OutFileFormatter
81
86
  "message_key" => "message_key", # SingleValueFormatter
82
87
  "add_newline" => "add_newline", # SingleValueFormatter
83
88
  "output_type" => "output_type", # StdoutFormatter
@@ -109,7 +114,7 @@ module Fluent
109
114
 
110
115
  # TODO: warn obsolete parameters if these are deprecated
111
116
  buffer_params = BUFFER_PARAMS.merge(BUFFER_TIME_SLICED_PARAMS)
112
- attr = compat_parameters_copy_to_subsection_attributes(conf, buffer_params) do |compat_key, value|
117
+ hash = compat_parameters_copy_to_subsection_attributes(conf, buffer_params) do |compat_key, value|
113
118
  if compat_key == 'buffer_queue_full_action' && value == 'exception'
114
119
  'throw_exception'
115
120
  else
@@ -121,7 +126,7 @@ module Fluent
121
126
 
122
127
  if conf.has_key?('time_slice_format')
123
128
  chunk_key = 'time'
124
- attr['timekey'] = case conf['time_slice_format']
129
+ hash['timekey'] = case conf['time_slice_format']
125
130
  when /\%S/ then 1
126
131
  when /\%M/ then 60
127
132
  when /\%H/ then 3600
@@ -129,13 +134,22 @@ module Fluent
129
134
  else
130
135
  raise Fluent::ConfigError, "time_slice_format only with %Y or %m is too long"
131
136
  end
137
+ if conf.has_key?('localtime') || conf.has_key?('utc')
138
+ if conf.has_key?('localtime') && conf.has_key?('utc')
139
+ raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
140
+ elsif conf.has_key?('localtime')
141
+ hash['timekey_use_utc'] = !(Fluent::Config.bool_value(conf['localtime']))
142
+ elsif conf.has_key?('utc')
143
+ hash['timekey_use_utc'] = Fluent::Config.bool_value(conf['utc'])
144
+ end
145
+ end
132
146
  else
133
147
  if chunk_key == 'time'
134
- attr['timekey'] = 86400 # TimeSliceOutput.time_slice_format default value is '%Y%m%d'
148
+ hash['timekey'] = 86400 # TimeSliceOutput.time_slice_format default value is '%Y%m%d'
135
149
  end
136
150
  end
137
151
 
138
- e = Fluent::Config::Element.new('buffer', chunk_key, attr, [])
152
+ e = Fluent::Config::Element.new('buffer', chunk_key, hash, [])
139
153
  conf.elements << e
140
154
 
141
155
  conf
@@ -146,22 +160,22 @@ module Fluent
146
160
  return if INJECT_PARAMS.keys.all?{|k| !conf.has_key?(k) }
147
161
 
148
162
  # TODO: warn obsolete parameters if these are deprecated
149
- attr = compat_parameters_copy_to_subsection_attributes(conf, INJECT_PARAMS)
163
+ hash = compat_parameters_copy_to_subsection_attributes(conf, INJECT_PARAMS)
150
164
 
151
165
  if conf.has_key?('include_time_key') && Fluent::Config.bool_value(conf['include_time_key'])
152
- attr['time_key'] ||= 'time'
153
- attr['time_type'] ||= 'string'
166
+ hash['time_key'] ||= 'time'
167
+ hash['time_type'] ||= 'string'
154
168
  end
155
169
  if conf.has_key?('time_as_epoch') && Fluent::Config.bool_value(conf['time_as_epoch'])
156
- attr['time_type'] = 'unixtime'
170
+ hash['time_type'] = 'unixtime'
157
171
  end
158
172
  if conf.has_key?('localtime') || conf.has_key?('utc')
159
173
  if conf.has_key?('localtime') && conf.has_key?('utc')
160
174
  raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
161
175
  elsif conf.has_key?('localtime')
162
- attr['localtime'] = Fluent::Config.bool_value(conf['localtime'])
176
+ hash['localtime'] = Fluent::Config.bool_value(conf['localtime'])
163
177
  elsif conf.has_key?('utc')
164
- attr['localtime'] = !(Fluent::Config.bool_value(conf['utc']))
178
+ hash['localtime'] = !(Fluent::Config.bool_value(conf['utc']))
165
179
  # Specifying "localtime false" means using UTC in TimeFormatter
166
180
  # And specifying "utc" is different from specifying "timezone +0000"(it's not always UTC).
167
181
  # There are difference between "Z" and "+0000" in timezone formatting.
@@ -170,10 +184,10 @@ module Fluent
170
184
  end
171
185
 
172
186
  if conf.has_key?('include_tag_key') && Fluent::Config.bool_value(conf['include_tag_key'])
173
- attr['tag_key'] ||= 'tag'
187
+ hash['tag_key'] ||= 'tag'
174
188
  end
175
189
 
176
- e = Fluent::Config::Element.new('inject', '', attr, [])
190
+ e = Fluent::Config::Element.new('inject', '', hash, [])
177
191
  conf.elements << e
178
192
 
179
193
  conf
@@ -184,9 +198,9 @@ module Fluent
184
198
  return if PARSER_PARAMS.keys.all?{|k| !conf.has_key?(k) }
185
199
 
186
200
  # TODO: warn obsolete parameters if these are deprecated
187
- attr = compat_parameters_copy_to_subsection_attributes(conf, PARSER_PARAMS)
201
+ hash = compat_parameters_copy_to_subsection_attributes(conf, PARSER_PARAMS)
188
202
 
189
- e = Fluent::Config::Element.new('parse', '', attr, [])
203
+ e = Fluent::Config::Element.new('parse', '', hash, [])
190
204
  conf.elements << e
191
205
 
192
206
  conf
@@ -197,27 +211,31 @@ module Fluent
197
211
  return if FORMATTER_PARAMS.keys.all?{|k| !conf.has_key?(k) }
198
212
 
199
213
  # TODO: warn obsolete parameters if these are deprecated
200
- attr = compat_parameters_copy_to_subsection_attributes(conf, FORMATTER_PARAMS)
214
+ hash = compat_parameters_copy_to_subsection_attributes(conf, FORMATTER_PARAMS)
215
+
216
+ if conf.has_key?('time_as_epoch') && Fluent::Config.bool_value(conf['time_as_epoch'])
217
+ hash['time_type'] = 'unixtime'
218
+ end
201
219
 
202
- e = Fluent::Config::Element.new('format', '', attr, [])
220
+ e = Fluent::Config::Element.new('format', '', hash, [])
203
221
  conf.elements << e
204
222
 
205
223
  conf
206
224
  end
207
225
 
208
226
  def compat_parameters_copy_to_subsection_attributes(conf, params, &block)
209
- attr = {}
227
+ hash = {}
210
228
  params.each do |compat, current|
211
229
  next unless current
212
230
  if conf.has_key?(compat)
213
231
  if block_given?
214
- attr[current] = block.call(compat, conf[compat])
232
+ hash[current] = block.call(compat, conf[compat])
215
233
  else
216
- attr[current] = conf[compat]
234
+ hash[current] = conf[compat]
217
235
  end
218
236
  end
219
237
  end
220
- attr
238
+ hash
221
239
  end
222
240
  end
223
241
  end
@@ -0,0 +1,92 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/event'
18
+ require 'fluent/time'
19
+ require 'fluent/configurable'
20
+
21
+ module Fluent
22
+ module PluginHelper
23
+ module Extract
24
+ def extract_tag_from_record(record)
25
+ return nil unless @_extract_enabled
26
+
27
+ if @_extract_tag_key && record.has_key?(@_extract_tag_key)
28
+ return record[@_extract_tag_key].to_s
29
+ end
30
+
31
+ nil
32
+ end
33
+
34
+ def extract_time_from_record(record)
35
+ return nil unless @_extract_enabled
36
+
37
+ if @_extract_time_key && record.has_key?(@_extract_time_key)
38
+ return @_extract_time_parser.call(record[@_extract_time_key])
39
+ end
40
+
41
+ nil
42
+ end
43
+
44
+ module ExtractParams
45
+ include Fluent::Configurable
46
+ config_section :extract, required: false, multi: false, param_name: :extract_config do
47
+ config_param :tag_key, :string, default: nil
48
+ config_param :time_key, :string, default: nil
49
+
50
+ # To avoid defining :time_type twice
51
+ config_param :time_type, :enum, list: [:float, :unixtime, :string], default: :float
52
+
53
+ Fluent::TimeMixin::TIME_PARAMETERS.each do |name, type, opts|
54
+ config_param name, type, opts
55
+ end
56
+ end
57
+ end
58
+
59
+ def self.included(mod)
60
+ mod.include ExtractParams
61
+ end
62
+
63
+ def initialize
64
+ super
65
+ @_extract_enabled = false
66
+ @_extract_tag_key = nil
67
+ @_extract_time_key = nil
68
+ @_extract_time_parser = nil
69
+ end
70
+
71
+ def configure(conf)
72
+ super
73
+
74
+ if @extract_config
75
+ @_extract_tag_key = @extract_config.tag_key
76
+ @_extract_time_key = @extract_config.time_key
77
+ if @_extract_time_key
78
+ @_extract_time_parser = case @extract_config.time_type
79
+ when :float then ->(v){ Fluent::EventTime.new(v.to_i, ((v.to_f - v.to_i) * 1_000_000_000).to_i) }
80
+ when :unixtime then ->(v){ Fluent::EventTime.new(v.to_i, 0) }
81
+ else
82
+ localtime = @extract_config.localtime && !@extract_config.utc
83
+ Fluent::TimeParser.new(@extract_config.time_format, localtime, @extract_config.timezone)
84
+ end
85
+ end
86
+
87
+ @_extract_enabled = @_extract_tag_key || @_extract_time_key
88
+ end
89
+ end
90
+ end
91
+ end
92
+ end