fluent-plugin-record-reformer 0.2.0 → 0.2.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (7) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +8 -2
  3. data/README.md +1 -1
  4. data/fluent-plugin-record-reformer.gemspec +2 -2
  5. data/lib/fluent/plugin/out_record_reformer.rb +72 -57
  6. data/spec/out_record_reformer_bench.rb +40 -0
  7. metadata +5 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 2166c7d160e63370e3400203a591cbb9c2de22e7
4
- data.tar.gz: e01e518088e90bdb32e4040ed5a2fbbca5ee6bf8
3
+ metadata.gz: 55ba6719a96af4f1c33c52d970579a9006ef9a7c
4
+ data.tar.gz: 15cd6f42ed2f5b0b995f20560cf9974d12e23bda
5
5
  SHA512:
6
- metadata.gz: b2e3901a0af27f800d9a81e34477409f8696b13bed9c2e47425911cd45ae0295455a5686e42969daa31a4a327c8cf779b821f0fe4a8b4625eedfe72fce58b0f9
7
- data.tar.gz: 8812a93c884c231525b95bf29b14e5f9d4381ea0f9cece5a3109b3d7439b3cfff677c99831741d7aa7f509798bc4cea68348cbe4747a15676c45c471ff26cb62
6
+ metadata.gz: 8758e09cd82e2f2fd50076811bf7a25961a2782f5ac6081185567ff8e874b1bb312aeaae546acfa17411a26f6c0af2c85ed47af99416cb7448ac4a65176e5815
7
+ data.tar.gz: 4ae98eedf0aafea25057b67a9d2f2f189f54c1063d00ecd70f90075c317171089e614bc3f00115f027037ca5a23eee51c0c7b3c75f6f8b150174792f1c0cf972
data/CHANGELOG.md CHANGED
@@ -1,10 +1,16 @@
1
+ ## 0.2.1 (2014/01/15)
2
+
3
+ Enhancement:
4
+
5
+ * Speed up
6
+
1
7
  ## 0.2.0 (2014/01/15)
2
8
 
3
9
  Enhancement:
4
10
 
5
- * Support a record directive
11
+ * Support a `record` directive
6
12
  * Add `remove_keys` option
7
- * Add `extends` option
13
+ * Add `renew_record` option
8
14
  * Add `enable_ruby` option
9
15
 
10
16
  ## 0.1.1 (2013/11/21)
data/README.md CHANGED
@@ -2,7 +2,7 @@
2
2
 
3
3
  [![Build Status](https://secure.travis-ci.org/sonots/fluent-plugin-record-reformer.png?branch=master)](http://travis-ci.org/sonots/fluent-plugin-record-reformer)
4
4
 
5
- Fluentd pluging to add or replace fields of a event record
5
+ Fluentd plugin to add or replace fields of a event record
6
6
 
7
7
  ## Installation
8
8
 
data/fluent-plugin-record-reformer.gemspec CHANGED
@@ -3,11 +3,11 @@ $:.push File.expand_path('../lib', __FILE__)
3
3
 
4
4
  Gem::Specification.new do |gem|
5
5
  gem.name = "fluent-plugin-record-reformer"
6
- gem.version = "0.2.0"
6
+ gem.version = "0.2.1"
7
7
  gem.authors = ["Naotoshi Seo"]
8
8
  gem.email = "sonots@gmail.com"
9
9
  gem.homepage = "https://github.com/sonots/fluent-plugin-record-reformer"
10
- gem.description = "Output filter plugin for reforming each event record"
10
+ gem.description = "Fluentd plugin to add or replace fields of a event record"
11
11
  gem.summary = gem.description
12
12
  gem.licenses = ["MIT"]
13
13
  gem.has_rdoc = false
data/lib/fluent/plugin/out_record_reformer.rb CHANGED
@@ -40,11 +40,11 @@ module Fluent
40
40
  @remove_keys = @remove_keys.split(',')
41
41
  end
42
42
 
43
- @expand_placeholder_proc =
43
+ @placeholder_expander =
44
44
  if @enable_ruby
45
- Proc.new {|str, record, tag, tag_parts, time| expand_ruby_placeholder(str, record, tag, tag_parts, time) }
45
+ RubyPlaceholderExpander.new
46
46
  else
47
- Proc.new {|str, record, tag, tag_parts, time| expand_placeholder(str, record, tag, tag_parts, time) }
47
+ PlaceholderExpander.new
48
48
  end
49
49
 
50
50
  @time_proc = # hmm, want to remove ${time} placeholder ...
@@ -60,9 +60,8 @@ module Fluent
60
60
  def emit(tag, es, chain)
61
61
  tag_parts = tag.split('.')
62
62
  es.each { |time, record|
63
- t_time = @time_proc.call(time)
64
- output_tag = @expand_placeholder_proc.call(@output_tag, record, tag, tag_parts, t_time)
65
- Engine.emit(output_tag, time, replace_record(record, tag, tag_parts, t_time))
63
+ new_tag, new_record = reform(@output_tag, record, tag, tag_parts, @time_proc.call(time))
64
+ Engine.emit(new_tag, time, new_record)
66
65
  }
67
66
  chain.next
68
67
  rescue => e
@@ -71,69 +70,85 @@ module Fluent
71
70
 
72
71
  private
73
72
 
74
- def replace_record(record, tag, tag_parts, time)
73
+ def reform(output_tag, record, tag, tag_parts, time)
74
+ @placeholder_expander.prepare_placeholders(record, tag, tag_parts, @hostname, time)
75
+ new_tag = @placeholder_expander.expand(output_tag)
76
+
75
77
  new_record = @renew_record ? {} : record.dup
76
- @map.each_pair { |k, v|
77
- new_record[k] = @expand_placeholder_proc.call(v, record, tag, tag_parts, time)
78
- }
78
+ @map.each_pair { |k, v| new_record[k] = @placeholder_expander.expand(v) }
79
79
  @remove_keys.each { |k| new_record.delete(k) } if @remove_keys
80
- new_record
80
+
81
+ [new_tag, new_record]
81
82
  end
82
83
 
83
- def expand_placeholder(str, record, tag, tag_parts, time)
84
+ class PlaceholderExpander
84
85
  # referenced https://github.com/fluent/fluent-plugin-rewrite-tag-filter, thanks!
85
- placeholders = get_placeholders(record, tag, tag_parts, time)
86
- str.gsub(/(\${[a-z_]+(\[-?[0-9]+\])?}|__[A-Z_]+__)/) do
87
- $log.warn "record_reformer: unknown placeholder `#{$1}` found in a tag `#{tag}`" unless placeholders.include?($1)
88
- placeholders[$1]
89
- end
90
- end
86
+ attr_reader :placeholders
91
87
 
92
- def get_placeholders(record, tag, tag_parts, time)
93
- placeholders = {
94
- '${time}' => time,
95
- '${tag}' => tag,
96
- '${hostname}' => @hostname,
97
- }
88
+ def prepare_placeholders(record, tag, tag_parts, hostname, time)
89
+ placeholders = {
90
+ '${time}' => time,
91
+ '${tag}' => tag,
92
+ '${hostname}' => hostname,
93
+ }
98
94
 
99
- size = tag_parts.size
100
- tag_parts.each_with_index do |t, idx|
101
- placeholders.store("${tag_parts[#{idx}]}", t)
102
- placeholders.store("${tag_parts[#{idx-size}]}", t) # support tag_parts[-1]
103
- end
104
- # tags is just for old version compatibility
105
- tag_parts.each_with_index do |t, idx|
106
- placeholders.store("${tags[#{idx}]}", t)
107
- placeholders.store("${tags[#{idx-size}]}", t) # support tags[-1]
108
- end
95
+ size = tag_parts.size
96
+ tag_parts.each_with_index do |t, idx|
97
+ placeholders.store("${tag_parts[#{idx}]}", t)
98
+ placeholders.store("${tag_parts[#{idx-size}]}", t) # support tag_parts[-1]
99
+ end
100
+ # tags is just for old version compatibility
101
+ tag_parts.each_with_index do |t, idx|
102
+ placeholders.store("${tags[#{idx}]}", t)
103
+ placeholders.store("${tags[#{idx-size}]}", t) # support tags[-1]
104
+ end
109
105
 
110
- record.each {|k, v|
111
- placeholders.store("${#{k}}", v)
112
- }
106
+ record.each {|k, v|
107
+ placeholders.store("${#{k}}", v)
108
+ }
113
109
 
114
- return placeholders
115
- end
110
+ @placeholders = placeholders
111
+ end
116
112
 
117
- # Replace placeholders in a string
118
- #
119
- # @param [String] str the string to be replaced
120
- # @param [Hash] record the record, one of information
121
- # @param [String] tag the tag
122
- # @param [Array] tag_parts the tag parts (tag splitted by .)
123
- # @param [Time] time the time
124
- def expand_ruby_placeholder(str, record, tag, tag_parts, time)
125
- struct = UndefOpenStruct.new(record)
126
- struct.tag = tag
127
- struct.tags = struct.tag_parts = tag_parts # tags is for old version compatibility
128
- struct.time = time
129
- struct.hostname = @hostname
130
- str = str.gsub(/\$\{([^}]+)\}/, '#{\1}') # ${..} => #{..}
131
- eval "\"#{str}\"", struct.instance_eval { binding }
113
+ def expand(str)
114
+ str.gsub(/(\${[a-z_]+(\[-?[0-9]+\])?}|__[A-Z_]+__)/) do
115
+ $log.warn "record_reformer: unknown placeholder `#{$1}` found in a tag `#{tag}`" unless @placeholders.include?($1)
116
+ @placeholders[$1]
117
+ end
118
+ end
132
119
  end
133
120
 
134
- class UndefOpenStruct < OpenStruct
135
- (Object.instance_methods).each do |m|
136
- undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member/
121
+ class RubyPlaceholderExpander
122
+ attr_reader :placeholders
123
+
124
+ # Get placeholders as a struct
125
+ #
126
+ # @param [Hash] record the record, one of information
127
+ # @param [String] tag the tag
128
+ # @param [Array] tag_parts the tag parts (tag splitted by .)
129
+ # @param [String] hostname the hostname
130
+ # @param [Time] time the time
131
+ def prepare_placeholders(record, tag, tag_parts, hostname, time)
132
+ struct = UndefOpenStruct.new(record)
133
+ struct.tag = tag
134
+ struct.tags = struct.tag_parts = tag_parts # tags is for old version compatibility
135
+ struct.time = time
136
+ struct.hostname = hostname
137
+ @placeholders = struct
138
+ end
139
+
140
+ # Replace placeholders in a string
141
+ #
142
+ # @param [String] str the string to be replaced
143
+ def expand(str)
144
+ str = str.gsub(/\$\{([^}]+)\}/, '#{\1}') # ${..} => #{..}
145
+ eval "\"#{str}\"", @placeholders.instance_eval { binding }
146
+ end
147
+
148
+ class UndefOpenStruct < OpenStruct
149
+ (Object.instance_methods).each do |m|
150
+ undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member/
151
+ end
137
152
  end
138
153
  end
139
154
  end
data/spec/out_record_reformer_bench.rb ADDED
@@ -0,0 +1,40 @@
1
+ # encoding: UTF-8
2
+ require_relative 'spec_helper'
3
+ require 'benchmark'
4
+ Fluent::Test.setup
5
+
6
+ def create_driver(config, tag = 'foo.bar')
7
+ Fluent::Test::OutputTestDriver.new(Fluent::RecordReformerOutput, tag).configure(config)
8
+ end
9
+
10
+ # setup
11
+ message = {'message' => "2013/01/13T07:02:11.124202 INFO GET /ping"}
12
+ time = Time.now.to_i
13
+
14
+ enable_ruby_driver = create_driver(%[
15
+ enable_ruby true
16
+ output_tag reformed.${tag}
17
+ message ${tag_parts[0]}
18
+ ])
19
+ disable_ruby_driver = create_driver(%[
20
+ enable_ruby false
21
+ output_tag reformed.${tag}
22
+ message ${tag_parts[0]}
23
+ ])
24
+
25
+ # bench
26
+ n = 1000
27
+ Benchmark.bm(7) do |x|
28
+ x.report("enable_ruby") { enable_ruby_driver.run { n.times { enable_ruby_driver.emit(message, time) } } }
29
+ x.report("disable_ruby") { disable_ruby_driver.run { n.times { disable_ruby_driver.emit(message, time) } } }
30
+ end
31
+
32
+ #BEFORE REFACTORING
33
+ # user system total real
34
+ #enable_ruby 0.310000 0.000000 0.310000 ( 0.835560)
35
+ #disable_ruby 0.150000 0.000000 0.150000 ( 0.679239)
36
+
37
+ #AFTER REFACTORING (PlaceholderParser and RubyPlaceholderParser)
38
+ # user system total real
39
+ #enable_ruby 0.290000 0.010000 0.300000 ( 0.815281)
40
+ #disable_ruby 0.060000 0.000000 0.060000 ( 0.588556)
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-record-reformer
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.0
4
+ version: 0.2.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Naotoshi Seo
@@ -80,7 +80,7 @@ dependencies:
80
80
  - - '>='
81
81
  - !ruby/object:Gem::Version
82
82
  version: '0'
83
- description: Output filter plugin for reforming each event record
83
+ description: Fluentd plugin to add or replace fields of a event record
84
84
  email: sonots@gmail.com
85
85
  executables: []
86
86
  extensions: []
@@ -96,6 +96,7 @@ files:
96
96
  - Rakefile
97
97
  - fluent-plugin-record-reformer.gemspec
98
98
  - lib/fluent/plugin/out_record_reformer.rb
99
+ - spec/out_record_reformer_bench.rb
99
100
  - spec/out_record_reformer_spec.rb
100
101
  - spec/spec_helper.rb
101
102
  homepage: https://github.com/sonots/fluent-plugin-record-reformer
@@ -121,7 +122,8 @@ rubyforge_project:
121
122
  rubygems_version: 2.0.3
122
123
  signing_key:
123
124
  specification_version: 4
124
- summary: Output filter plugin for reforming each event record
125
+ summary: Fluentd plugin to add or replace fields of a event record
125
126
  test_files:
127
+ - spec/out_record_reformer_bench.rb
126
128
  - spec/out_record_reformer_spec.rb
127
129
  - spec/spec_helper.rb