fluent-plugin-record-reformer 0.7.2 → 0.8.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: a489c46189c76ae3c606242b7b5d2a955d044f7c
4
- data.tar.gz: 72dc9bea1cab4b0153f7d8b2847a5f75334a2e64
3
+ metadata.gz: bf0c0a50d2b8509f3fa6917e0f700abe30e5249a
4
+ data.tar.gz: fba94b0ab7c6259354c70be688ef5ad81cc29efe
5
5
  SHA512:
6
- metadata.gz: 43d512aeb2f8e63779ba894716e17b5c1c0c666a7860497da43c1566f2e00265faa182166272ce9d65e22b83a3a878e7e206cbb72664aa6203588fa8dd34e0ac
7
- data.tar.gz: a29e0f8fd38e867411fe2787d1074fabc280b73328c19b7df49d38f81808d97392cd98b1ce988bd517085f7c8aa673a088d66bfc3f8eef72d3b5cb5187d21399
6
+ metadata.gz: 132876d93292dd82888bbf7a34010dbc3c9e99ea746952165c919a71be3ad8118b3572009f5a8c5deb18654b714142a9b100b39c07ff2e3e40ae9d1e1c014ca9
7
+ data.tar.gz: 98e35238beb1848b8076ab6c66095a9cb15eb91301bd21f8eed18ad4f70e2eae2b932ad6c525c516f57794d0fee651a162d25a2b519c9fb19bceda90947aaad7
@@ -1,8 +1,9 @@
1
1
  rvm:
2
- - 1.9.3
3
2
  - 2.0.0
4
3
  - 2.1.*
5
4
  - 2.2.*
5
+ - 2.3.0
6
6
  gemfile:
7
7
  - Gemfile
8
+ - Gemfile.fluentd.0.10
8
9
  before_install: gem update bundler
@@ -1,3 +1,10 @@
1
+ ## 0.8.0 (2016/01/28)
2
+
3
+ Enhancements
4
+
5
+ * Support `${record["key"]}` placeholder
6
+ * Speed up `enable_ruby true`
7
+
1
8
  ## 0.7.2 (2015/12/29)
2
9
 
3
10
  Enhancements
@@ -0,0 +1,4 @@
1
+ source "http://rubygems.org"
2
+
3
+ gem 'fluentd', '~> 0.10.43'
4
+ gemspec
data/README.md CHANGED
@@ -126,14 +126,10 @@ This results in same, but please note that following option parameters are reser
126
126
 
127
127
  ## Placeholders
128
128
 
129
- The keys of input json are available as placeholders. In the above example,
130
-
131
- * ${remove_me}
132
- * ${not_remove_me}
133
- * ${message}
134
-
135
- shall be available. In addition, following placeholders are reserved:
129
+ Following placeholders are available:
136
130
 
131
+ * ${record["key"]} Record value of `key` such as `${record["message"]}` in the above example (available from v0.8.0).
132
+ * Originally, record placeholders were available as `${key}` such as `${message}`. This is still kept for the backward compatibility, but would be removed in the future.
137
133
  * ${hostname} Hostname of the running machine
138
134
  * ${tag} Input tag
139
135
  * ${time} Time of the event
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
3
3
 
4
4
  Gem::Specification.new do |gem|
5
5
  gem.name = "fluent-plugin-record-reformer"
6
- gem.version = "0.7.2"
6
+ gem.version = "0.8.0"
7
7
  gem.authors = ["Naotoshi Seo"]
8
8
  gem.email = "sonots@gmail.com"
9
9
  gem.homepage = "https://github.com/sonots/fluent-plugin-record-reformer"
@@ -10,21 +10,21 @@ module Fluent
10
10
  end
11
11
 
12
12
  config_param :output_tag, :string, :default => nil, # obsolete
13
- :desc => 'The output tag name. This option is deprecated. Use `tag` option instead.'
13
+ :desc => 'The output tag name. This option is deprecated. Use `tag` option instead.'
14
14
  config_param :tag, :string, :default => nil,
15
- :desc => 'The output tag name.'
15
+ :desc => 'The output tag name.'
16
16
  config_param :remove_keys, :string, :default => nil,
17
- :desc => 'Specify record keys to be removed by a string separated by , (comma).'
17
+ :desc => 'Specify record keys to be removed by a string separated by , (comma).'
18
18
  config_param :keep_keys, :string, :default => nil,
19
- :desc => 'Specify record keys to be kept by a string separated by , (comma).'
19
+ :desc => 'Specify record keys to be kept by a string separated by , (comma).'
20
20
  config_param :renew_record, :bool, :default => false,
21
- :desc => 'Creates an output record newly without extending (merging) the input record fields.'
21
+ :desc => 'Creates an output record newly without extending (merging) the input record fields.'
22
22
  config_param :renew_time_key, :string, :default => nil,
23
- :desc => 'Overwrites the time of events with a value of the record field.'
23
+ :desc => 'Overwrites the time of events with a value of the record field.'
24
24
  config_param :enable_ruby, :bool, :default => true, # true for lower version compatibility
25
- :desc => 'Enable to use ruby codes in placeholders.'
25
+ :desc => 'Enable to use ruby codes in placeholders.'
26
26
  config_param :auto_typecast, :bool, :default => false, # false for lower version compatibility
27
- :desc => 'Automatically cast the field types.'
27
+ :desc => 'Automatically cast the field types.'
28
28
 
29
29
  BUILTIN_CONFIGURATIONS = %W(@id @type @label type tag output_tag remove_keys renew_record keep_keys enable_ruby renew_time_key auto_typecast)
30
30
 
@@ -41,17 +41,17 @@ module Fluent
41
41
  def configure(conf)
42
42
  super
43
43
 
44
- @map = {}
44
+ map = {}
45
45
  conf.each_pair { |k, v|
46
46
  next if BUILTIN_CONFIGURATIONS.include?(k)
47
47
  conf.has_key?(k) # to suppress unread configuration warning
48
- @map[k] = parse_value(v)
48
+ map[k] = parse_value(v)
49
49
  }
50
50
  # <record></record> directive
51
51
  conf.elements.select { |element| element.name == 'record' }.each { |element|
52
52
  element.each_pair { |k, v|
53
53
  element.has_key?(k) # to suppress unread configuration warning
54
- @map[k] = parse_value(v)
54
+ map[k] = parse_value(v)
55
55
  }
56
56
  }
57
57
 
@@ -86,6 +86,8 @@ module Fluent
86
86
  else
87
87
  PlaceholderExpander.new(placeholder_expander_params)
88
88
  end
89
+ @map = @placeholder_expander.preprocess_map(map)
90
+ @tag = @placeholder_expander.preprocess_map(@tag)
89
91
 
90
92
  @hostname = Socket.gethostname
91
93
  end
@@ -94,18 +96,22 @@ module Fluent
94
96
  tag_parts = tag.split('.')
95
97
  tag_prefix = tag_prefix(tag_parts)
96
98
  tag_suffix = tag_suffix(tag_parts)
97
- placeholders = {
98
- 'tag' => tag,
99
- 'tags' => tag_parts,
100
- 'tag_parts' => tag_parts,
99
+ placeholder_values = {
100
+ 'tag' => tag,
101
+ 'tags' => tag_parts, # for old version compatibility
102
+ 'tag_parts' => tag_parts,
101
103
  'tag_prefix' => tag_prefix,
102
104
  'tag_suffix' => tag_suffix,
103
- 'hostname' => @hostname,
105
+ 'hostname' => @hostname,
104
106
  }
105
107
  last_record = nil
106
108
  es.each {|time, record|
107
109
  last_record = record # for debug log
108
- new_tag, new_record = reform(@tag, time, record, placeholders)
110
+ placeholder_values.merge!({
111
+ 'time' => @placeholder_expander.time_value(time),
112
+ 'record' => record,
113
+ })
114
+ new_tag, new_record = reform(@tag, record, placeholder_values)
109
115
  if new_tag
110
116
  if @renew_time_key && new_record.has_key?(@renew_time_key)
111
117
  time = new_record[@renew_time_key].to_i
@@ -116,7 +122,7 @@ module Fluent
116
122
  chain.next
117
123
  rescue => e
118
124
  log.warn "record_reformer: #{e.class} #{e.message} #{e.backtrace.first}"
119
- log.debug "record_reformer: tag:#{@tag} map:#{@map} record:#{last_record} placeholders:#{placeholders}"
125
+ log.debug "record_reformer: tag:#{@tag} map:#{@map} record:#{last_record} placeholder_values:#{placeholder_values}"
120
126
  end
121
127
 
122
128
  private
@@ -132,9 +138,10 @@ module Fluent
132
138
  value_str # emit as string
133
139
  end
134
140
 
135
- def reform(tag, time, record, opts)
136
- @placeholder_expander.prepare_placeholders(time, record, opts)
137
- new_tag = @placeholder_expander.expand(tag)
141
+ def reform(tag, record, placeholder_values)
142
+ @placeholder_expander.prepare_placeholders(placeholder_values)
143
+
144
+ new_tag = expand_placeholders(tag)
138
145
 
139
146
  new_record = @renew_record ? {} : record.dup
140
147
  @keep_keys.each {|k| new_record[k] = record[k]} if @keep_keys and @renew_record
@@ -190,17 +197,29 @@ module Fluent
190
197
  @auto_typecast = params[:auto_typecast]
191
198
  end
192
199
 
193
- def prepare_placeholders(time, record, opts)
194
- placeholders = { '${time}' => Time.at(time).to_s }
195
- record.each {|key, value| placeholders.store("${#{key}}", value) }
200
+ def time_value(time)
201
+ Time.at(time).to_s
202
+ end
203
+
204
+ def preprocess_map(value, force_stringify = false)
205
+ value
206
+ end
196
207
 
197
- opts.each do |key, value|
208
+ def prepare_placeholders(placeholder_values)
209
+ placeholders = {}
210
+
211
+ placeholder_values.each do |key, value|
198
212
  if value.kind_of?(Array) # tag_parts, etc
199
213
  size = value.size
200
- value.each_with_index { |v, idx|
214
+ value.each_with_index do |v, idx|
201
215
  placeholders.store("${#{key}[#{idx}]}", v)
202
216
  placeholders.store("${#{key}[#{idx-size}]}", v) # support [-1]
203
- }
217
+ end
218
+ elsif value.kind_of?(Hash) # record, etc
219
+ value.each do |k, v|
220
+ placeholders.store("${#{k}}", v) # foo
221
+ placeholders.store(%Q[${#{key}["#{k}"]}], v) # record["foo"]
222
+ end
204
223
  else # string, interger, float, and others?
205
224
  placeholders.store("${#{key}}", value)
206
225
  end
@@ -209,22 +228,27 @@ module Fluent
209
228
  @placeholders = placeholders
210
229
  end
211
230
 
212
- def expand(str, force_stringify=false)
231
+ # Expand string with placeholders
232
+ #
233
+ # @param [String] str
234
+ # @param [Boolean] force_stringify the value must be string, used for hash key
235
+ def expand(str, force_stringify = false)
213
236
  if @auto_typecast and !force_stringify
214
237
  single_placeholder_matched = str.match(/\A(\${[^}]+}|__[A-Z_]+__)\z/)
215
238
  if single_placeholder_matched
216
- log_unknown_placeholder($1)
239
+ log_if_unknown_placeholder($1)
217
240
  return @placeholders[single_placeholder_matched[1]]
218
241
  end
219
242
  end
220
243
  str.gsub(/(\${[^}]+}|__[A-Z_]+__)/) {
221
- log_unknown_placeholder($1)
244
+ log_if_unknown_placeholder($1)
222
245
  @placeholders[$1]
223
246
  }
224
247
  end
225
248
 
226
249
  private
227
- def log_unknown_placeholder(placeholder)
250
+
251
+ def log_if_unknown_placeholder(placeholder)
228
252
  unless @placeholders.include?(placeholder)
229
253
  log.warn "record_reformer: unknown placeholder `#{placeholder}` found"
230
254
  end
@@ -232,45 +256,96 @@ module Fluent
232
256
  end
233
257
 
234
258
  class RubyPlaceholderExpander
235
- attr_reader :placeholders, :log
259
+ attr_reader :log
236
260
 
237
261
  def initialize(params)
238
262
  @log = params[:log]
239
263
  @auto_typecast = params[:auto_typecast]
264
+ @cleanroom_expander = CleanroomExpander.new
240
265
  end
241
266
 
242
- # Get placeholders as a struct
243
- #
244
- # @param [Time] time the time
245
- # @param [Hash] record the record
246
- # @param [Hash] opts others
247
- def prepare_placeholders(time, record, opts)
248
- struct = UndefOpenStruct.new(record)
249
- struct.time = Time.at(time)
250
- opts.each {|key, value| struct.__send__("#{key}=", value) }
251
- @placeholders = struct
267
+ def time_value(time)
268
+ Time.at(time)
252
269
  end
253
270
 
254
- # Replace placeholders in a string
271
+ # Preprocess record map to convert into ruby string expansion
255
272
  #
256
- # @param [String] str the string to be replaced
257
- def expand(str, force_stringify=false)
258
- if @auto_typecast and !force_stringify
259
- single_placeholder_matched = str.match(/\A\${([^}]+)}\z/)
260
- if single_placeholder_matched
261
- code = single_placeholder_matched[1]
262
- return eval code, @placeholders.instance_eval { binding }
273
+ # @param [Hash|String|Array] value record map config
274
+ # @param [Boolean] force_stringify the value must be string, used for hash key
275
+ def preprocess_map(value, force_stringify = false)
276
+ new_value = nil
277
+ if value.is_a?(String)
278
+ if @auto_typecast and !force_stringify
279
+ if single_placeholder_matched = value.match(/\A\${([^}]+)}\z/) # ${..} => ..
280
+ new_value = single_placeholder_matched[1]
281
+ end
282
+ end
283
+ unless new_value
284
+ new_value = %Q{%Q[#{value.gsub(/\$\{([^}]+)\}/, '#{\1}')}]} # xx${..}xx => %Q[xx#{..}xx]
285
+ end
286
+ elsif value.is_a?(Hash)
287
+ new_value = {}
288
+ value.each_pair do |k, v|
289
+ new_value[preprocess_map(k, true)] = preprocess_map(v)
263
290
  end
291
+ elsif value.is_a?(Array)
292
+ new_value = []
293
+ value.each_with_index do |v, i|
294
+ new_value[i] = preprocess_map(v)
295
+ end
296
+ else
297
+ new_value = value
264
298
  end
265
- interpolated = str.gsub(/\$\{([^}]+)\}/, '#{\1}') # ${..} => #{..}
266
- eval "\"#{interpolated}\"", @placeholders.instance_eval { binding }
299
+ new_value
300
+ end
301
+
302
+ def prepare_placeholders(placeholder_values)
303
+ @tag = placeholder_values['tag']
304
+ @time = placeholder_values['time']
305
+ @record = placeholder_values['record']
306
+ @tag_parts = placeholder_values['tag_parts']
307
+ @tag_prefix = placeholder_values['tag_prefix']
308
+ @tag_suffix = placeholder_values['tag_suffix']
309
+ @hostname = placeholder_values['hostname']
310
+ end
311
+
312
+ # Expand string with placeholders
313
+ #
314
+ # @param [String] str
315
+ def expand(str, force_stringify = false)
316
+ @cleanroom_expander.expand(
317
+ str,
318
+ @tag,
319
+ @time,
320
+ @record,
321
+ @tag_parts,
322
+ @tag_prefix,
323
+ @tag_suffix,
324
+ @hostname,
325
+ )
267
326
  rescue => e
268
327
  log.warn "record_reformer: failed to expand `#{str}`", :error_class => e.class, :error => e.message
269
328
  log.warn_backtrace
270
329
  nil
271
330
  end
272
331
 
273
- class UndefOpenStruct < OpenStruct
332
+ class CleanroomExpander
333
+ def expand(__str_to_eval__, tag, time, record, tag_parts, tag_prefix, tag_suffix, hostname)
334
+ tags = tag_parts # for old version compatibility
335
+ @record = record # for old version compatibility
336
+ instance_eval(__str_to_eval__)
337
+ end
338
+
339
+ # for old version compatibility
340
+ def method_missing(name)
341
+ key = name.to_s
342
+ if @record.has_key?(key)
343
+ @record[key]
344
+ else
345
+ raise NameError, "undefined local variable or method `#{key}'"
346
+ end
347
+ end
348
+
274
349
  (Object.instance_methods).each do |m|
275
350
  undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member/
276
351
  end
@@ -7,16 +7,17 @@ unless defined?(Test::Unit::AssertionFailedError)
7
7
  end
8
8
  end
9
9
 
10
- # Stop non required sleep at
11
- # https://github.com/fluent/fluentd/blob/018791f6b1b0400b71e37df2fb3ad80e456d2c11/lib/fluent/test/base.rb#L56
10
+ # Reduce sleep period at
11
+ # https://github.com/fluent/fluentd/blob/a271b3ec76ab7cf89ebe4012aa5b3912333dbdb7/lib/fluent/test/base.rb#L81
12
12
  module Fluent
13
13
  module Test
14
14
  class TestDriver
15
- def run(&block)
15
+ def run(num_waits = 10, &block)
16
16
  @instance.start
17
17
  begin
18
18
  # wait until thread starts
19
- # 10.times { sleep 0.05 }
19
+ # num_waits.times { sleep 0.05 }
20
+ sleep 0.05
20
21
  return yield
21
22
  ensure
22
23
  @instance.shutdown
@@ -470,6 +470,28 @@ EOC
470
470
  end
471
471
  assert_equal(expected_results, actual_results)
472
472
  end
473
+
474
+ test %Q[record["key"] with enable_ruby #{enable_ruby}] do
475
+ config = %[
476
+ tag tag
477
+ enable_ruby #{enable_ruby}
478
+ auto_typecast true
479
+ <record>
480
+ _timestamp ${record["@timestamp"]}
481
+ _foo_bar ${record["foo.bar"]}
482
+ </record>
483
+ ]
484
+ d = create_driver(config, use_v1)
485
+ record = {
486
+ "foo.bar" => "foo.bar",
487
+ "@timestamp" => 10,
488
+ }
489
+ es = emit(config, use_v1, [record])
490
+ es.each_with_index do |(tag, time, r), i|
491
+ assert { r['_timestamp'] == record['@timestamp'] }
492
+ assert { r['_foo_bar'] == record['foo.bar'] }
493
+ end
494
+ end
473
495
  end
474
496
 
475
497
  test 'unknown placeholder (enable_ruby no)' do
@@ -495,7 +517,7 @@ EOC
495
517
  </record>
496
518
  ]
497
519
  d = create_driver(config, use_v1)
498
- mock(d.instance.log).warn("record_reformer: failed to expand `${unknown['bar']}`", anything)
520
+ mock(d.instance.log).warn("record_reformer: failed to expand `%Q[\#{unknown['bar']}]`", anything)
499
521
  d.run { d.emit({}, @time) }
500
522
  # emit, but nil value
501
523
  assert_equal 1, d.emits.size
@@ -510,7 +532,7 @@ EOC
510
532
  enable_ruby yes
511
533
  ]
512
534
  d = create_driver(config, use_v1)
513
- mock(d.instance.log).warn("record_reformer: failed to expand `${unknown['bar']}`", anything)
535
+ mock(d.instance.log).warn("record_reformer: failed to expand `%Q[\#{unknown['bar']}]`", anything)
514
536
  d.run { d.emit({}, @time) }
515
537
  # nil tag message should not be emitted
516
538
  assert_equal 0, d.emits.size
@@ -548,5 +570,38 @@ EOC
548
570
  end
549
571
  end
550
572
  end
573
+
574
+ test "compatibility test (enable_ruby yes) (use_v1 #{use_v1})" do
575
+ config = %[
576
+ tag tag
577
+ enable_ruby yes
578
+ auto_typecast yes
579
+ <record>
580
+ _message prefix-${message}-suffix
581
+ _time ${Time.at(time)}
582
+ _number ${number == '-' ? 0 : number}
583
+ _match ${/0x[0-9a-f]+/.match(hex)[0]}
584
+ _timestamp ${__send__("@timestamp")}
585
+ _foo_bar ${__send__('foo.bar')}
586
+ </record>
587
+ ]
588
+ d = create_driver(config, use_v1)
589
+ record = {
590
+ "number" => "-",
591
+ "hex" => "0x10",
592
+ "foo.bar" => "foo.bar",
593
+ "@timestamp" => 10,
594
+ "message" => "10",
595
+ }
596
+ es = emit(config, use_v1, [record])
597
+ es.each_with_index do |(tag, time, r), i|
598
+ assert { r['_message'] == "prefix-#{record['message']}-suffix" }
599
+ assert { r['_time'] == Time.at(@time) }
600
+ assert { r['_number'] == 0 }
601
+ assert { r['_match'] == record['hex'] }
602
+ assert { r['_timestamp'] == record['@timestamp'] }
603
+ assert { r['_foo_bar'] == record['foo.bar'] }
604
+ end
605
+ end
551
606
  end
552
607
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-record-reformer
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.7.2
4
+ version: 0.8.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Naotoshi Seo
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-12-29 00:00:00.000000000 Z
11
+ date: 2016-01-27 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -119,6 +119,7 @@ files:
119
119
  - ".travis.yml"
120
120
  - CHANGELOG.md
121
121
  - Gemfile
122
+ - Gemfile.fluentd.0.10
122
123
  - LICENSE
123
124
  - README.md
124
125
  - Rakefile