fluent-plugin-record-reformer 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +8 -2
  3. data/README.md +1 -1
  4. data/fluent-plugin-record-reformer.gemspec +2 -2
  5. data/lib/fluent/plugin/out_record_reformer.rb +72 -57
  6. data/spec/out_record_reformer_bench.rb +40 -0
  7. metadata +5 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 2166c7d160e63370e3400203a591cbb9c2de22e7
4
- data.tar.gz: e01e518088e90bdb32e4040ed5a2fbbca5ee6bf8
3
+ metadata.gz: 55ba6719a96af4f1c33c52d970579a9006ef9a7c
4
+ data.tar.gz: 15cd6f42ed2f5b0b995f20560cf9974d12e23bda
5
5
  SHA512:
6
- metadata.gz: b2e3901a0af27f800d9a81e34477409f8696b13bed9c2e47425911cd45ae0295455a5686e42969daa31a4a327c8cf779b821f0fe4a8b4625eedfe72fce58b0f9
7
- data.tar.gz: 8812a93c884c231525b95bf29b14e5f9d4381ea0f9cece5a3109b3d7439b3cfff677c99831741d7aa7f509798bc4cea68348cbe4747a15676c45c471ff26cb62
6
+ metadata.gz: 8758e09cd82e2f2fd50076811bf7a25961a2782f5ac6081185567ff8e874b1bb312aeaae546acfa17411a26f6c0af2c85ed47af99416cb7448ac4a65176e5815
7
+ data.tar.gz: 4ae98eedf0aafea25057b67a9d2f2f189f54c1063d00ecd70f90075c317171089e614bc3f00115f027037ca5a23eee51c0c7b3c75f6f8b150174792f1c0cf972
data/CHANGELOG.md CHANGED
@@ -1,10 +1,16 @@
1
+ ## 0.2.1 (2014/01/15)
2
+
3
+ Enhancement:
4
+
5
+ * Speed up
6
+
1
7
  ## 0.2.0 (2014/01/15)
2
8
 
3
9
  Enhancement:
4
10
 
5
- * Support a record directive
11
+ * Support a `record` directive
6
12
  * Add `remove_keys` option
7
- * Add `extends` option
13
+ * Add `renew_record` option
8
14
  * Add `enable_ruby` option
9
15
 
10
16
  ## 0.1.1 (2013/11/21)
data/README.md CHANGED
@@ -2,7 +2,7 @@
2
2
 
3
3
  [![Build Status](https://secure.travis-ci.org/sonots/fluent-plugin-record-reformer.png?branch=master)](http://travis-ci.org/sonots/fluent-plugin-record-reformer)
4
4
 
5
- Fluentd pluging to add or replace fields of a event record
5
+ Fluentd plugin to add or replace fields of a event record
6
6
 
7
7
  ## Installation
8
8
 
data/fluent-plugin-record-reformer.gemspec CHANGED
@@ -3,11 +3,11 @@ $:.push File.expand_path('../lib', __FILE__)
3
3
 
4
4
  Gem::Specification.new do |gem|
5
5
  gem.name = "fluent-plugin-record-reformer"
6
- gem.version = "0.2.0"
6
+ gem.version = "0.2.1"
7
7
  gem.authors = ["Naotoshi Seo"]
8
8
  gem.email = "sonots@gmail.com"
9
9
  gem.homepage = "https://github.com/sonots/fluent-plugin-record-reformer"
10
- gem.description = "Output filter plugin for reforming each event record"
10
+ gem.description = "Fluentd plugin to add or replace fields of a event record"
11
11
  gem.summary = gem.description
12
12
  gem.licenses = ["MIT"]
13
13
  gem.has_rdoc = false
data/lib/fluent/plugin/out_record_reformer.rb CHANGED
@@ -40,11 +40,11 @@ module Fluent
40
40
  @remove_keys = @remove_keys.split(',')
41
41
  end
42
42
 
43
- @expand_placeholder_proc =
43
+ @placeholder_expander =
44
44
  if @enable_ruby
45
- Proc.new {|str, record, tag, tag_parts, time| expand_ruby_placeholder(str, record, tag, tag_parts, time) }
45
+ RubyPlaceholderExpander.new
46
46
  else
47
- Proc.new {|str, record, tag, tag_parts, time| expand_placeholder(str, record, tag, tag_parts, time) }
47
+ PlaceholderExpander.new
48
48
  end
49
49
 
50
50
  @time_proc = # hmm, want to remove ${time} placeholder ...
@@ -60,9 +60,8 @@ module Fluent
60
60
  def emit(tag, es, chain)
61
61
  tag_parts = tag.split('.')
62
62
  es.each { |time, record|
63
- t_time = @time_proc.call(time)
64
- output_tag = @expand_placeholder_proc.call(@output_tag, record, tag, tag_parts, t_time)
65
- Engine.emit(output_tag, time, replace_record(record, tag, tag_parts, t_time))
63
+ new_tag, new_record = reform(@output_tag, record, tag, tag_parts, @time_proc.call(time))
64
+ Engine.emit(new_tag, time, new_record)
66
65
  }
67
66
  chain.next
68
67
  rescue => e
@@ -71,69 +70,85 @@ module Fluent
71
70
 
72
71
  private
73
72
 
74
- def replace_record(record, tag, tag_parts, time)
73
+ def reform(output_tag, record, tag, tag_parts, time)
74
+ @placeholder_expander.prepare_placeholders(record, tag, tag_parts, @hostname, time)
75
+ new_tag = @placeholder_expander.expand(output_tag)
76
+
75
77
  new_record = @renew_record ? {} : record.dup
76
- @map.each_pair { |k, v|
77
- new_record[k] = @expand_placeholder_proc.call(v, record, tag, tag_parts, time)
78
- }
78
+ @map.each_pair { |k, v| new_record[k] = @placeholder_expander.expand(v) }
79
79
  @remove_keys.each { |k| new_record.delete(k) } if @remove_keys
80
- new_record
80
+
81
+ [new_tag, new_record]
81
82
  end
82
83
 
83
- def expand_placeholder(str, record, tag, tag_parts, time)
84
+ class PlaceholderExpander
84
85
  # referenced https://github.com/fluent/fluent-plugin-rewrite-tag-filter, thanks!
85
- placeholders = get_placeholders(record, tag, tag_parts, time)
86
- str.gsub(/(\${[a-z_]+(\[-?[0-9]+\])?}|__[A-Z_]+__)/) do
87
- $log.warn "record_reformer: unknown placeholder `#{$1}` found in a tag `#{tag}`" unless placeholders.include?($1)
88
- placeholders[$1]
89
- end
90
- end
86
+ attr_reader :placeholders
91
87
 
92
- def get_placeholders(record, tag, tag_parts, time)
93
- placeholders = {
94
- '${time}' => time,
95
- '${tag}' => tag,
96
- '${hostname}' => @hostname,
97
- }
88
+ def prepare_placeholders(record, tag, tag_parts, hostname, time)
89
+ placeholders = {
90
+ '${time}' => time,
91
+ '${tag}' => tag,
92
+ '${hostname}' => hostname,
93
+ }
98
94
 
99
- size = tag_parts.size
100
- tag_parts.each_with_index do |t, idx|
101
- placeholders.store("${tag_parts[#{idx}]}", t)
102
- placeholders.store("${tag_parts[#{idx-size}]}", t) # support tag_parts[-1]
103
- end
104
- # tags is just for old version compatibility
105
- tag_parts.each_with_index do |t, idx|
106
- placeholders.store("${tags[#{idx}]}", t)
107
- placeholders.store("${tags[#{idx-size}]}", t) # support tags[-1]
108
- end
95
+ size = tag_parts.size
96
+ tag_parts.each_with_index do |t, idx|
97
+ placeholders.store("${tag_parts[#{idx}]}", t)
98
+ placeholders.store("${tag_parts[#{idx-size}]}", t) # support tag_parts[-1]
99
+ end
100
+ # tags is just for old version compatibility
101
+ tag_parts.each_with_index do |t, idx|
102
+ placeholders.store("${tags[#{idx}]}", t)
103
+ placeholders.store("${tags[#{idx-size}]}", t) # support tags[-1]
104
+ end
109
105
 
110
- record.each {|k, v|
111
- placeholders.store("${#{k}}", v)
112
- }
106
+ record.each {|k, v|
107
+ placeholders.store("${#{k}}", v)
108
+ }
113
109
 
114
- return placeholders
115
- end
110
+ @placeholders = placeholders
111
+ end
116
112
 
117
- # Replace placeholders in a string
118
- #
119
- # @param [String] str the string to be replaced
120
- # @param [Hash] record the record, one of information
121
- # @param [String] tag the tag
122
- # @param [Array] tag_parts the tag parts (tag splitted by .)
123
- # @param [Time] time the time
124
- def expand_ruby_placeholder(str, record, tag, tag_parts, time)
125
- struct = UndefOpenStruct.new(record)
126
- struct.tag = tag
127
- struct.tags = struct.tag_parts = tag_parts # tags is for old version compatibility
128
- struct.time = time
129
- struct.hostname = @hostname
130
- str = str.gsub(/\$\{([^}]+)\}/, '#{\1}') # ${..} => #{..}
131
- eval "\"#{str}\"", struct.instance_eval { binding }
113
+ def expand(str)
114
+ str.gsub(/(\${[a-z_]+(\[-?[0-9]+\])?}|__[A-Z_]+__)/) do
115
+ $log.warn "record_reformer: unknown placeholder `#{$1}` found in a tag `#{tag}`" unless @placeholders.include?($1)
116
+ @placeholders[$1]
117
+ end
118
+ end
132
119
  end
133
120
 
134
- class UndefOpenStruct < OpenStruct
135
- (Object.instance_methods).each do |m|
136
- undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member/
121
+ class RubyPlaceholderExpander
122
+ attr_reader :placeholders
123
+
124
+ # Get placeholders as a struct
125
+ #
126
+ # @param [Hash] record the record, one of information
127
+ # @param [String] tag the tag
128
+ # @param [Array] tag_parts the tag parts (tag splitted by .)
129
+ # @param [String] hostname the hostname
130
+ # @param [Time] time the time
131
+ def prepare_placeholders(record, tag, tag_parts, hostname, time)
132
+ struct = UndefOpenStruct.new(record)
133
+ struct.tag = tag
134
+ struct.tags = struct.tag_parts = tag_parts # tags is for old version compatibility
135
+ struct.time = time
136
+ struct.hostname = hostname
137
+ @placeholders = struct
138
+ end
139
+
140
+ # Replace placeholders in a string
141
+ #
142
+ # @param [String] str the string to be replaced
143
+ def expand(str)
144
+ str = str.gsub(/\$\{([^}]+)\}/, '#{\1}') # ${..} => #{..}
145
+ eval "\"#{str}\"", @placeholders.instance_eval { binding }
146
+ end
147
+
148
+ class UndefOpenStruct < OpenStruct
149
+ (Object.instance_methods).each do |m|
150
+ undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member/
151
+ end
137
152
  end
138
153
  end
139
154
  end
data/spec/out_record_reformer_bench.rb ADDED
@@ -0,0 +1,40 @@
1
+ # encoding: UTF-8
2
+ require_relative 'spec_helper'
3
+ require 'benchmark'
4
+ Fluent::Test.setup
5
+
6
+ def create_driver(config, tag = 'foo.bar')
7
+ Fluent::Test::OutputTestDriver.new(Fluent::RecordReformerOutput, tag).configure(config)
8
+ end
9
+
10
+ # setup
11
+ message = {'message' => "2013/01/13T07:02:11.124202 INFO GET /ping"}
12
+ time = Time.now.to_i
13
+
14
+ enable_ruby_driver = create_driver(%[
15
+ enable_ruby true
16
+ output_tag reformed.${tag}
17
+ message ${tag_parts[0]}
18
+ ])
19
+ disable_ruby_driver = create_driver(%[
20
+ enable_ruby false
21
+ output_tag reformed.${tag}
22
+ message ${tag_parts[0]}
23
+ ])
24
+
25
+ # bench
26
+ n = 1000
27
+ Benchmark.bm(7) do |x|
28
+ x.report("enable_ruby") { enable_ruby_driver.run { n.times { enable_ruby_driver.emit(message, time) } } }
29
+ x.report("disable_ruby") { disable_ruby_driver.run { n.times { disable_ruby_driver.emit(message, time) } } }
30
+ end
31
+
32
+ #BEFORE REFACTORING
33
+ # user system total real
34
+ #enable_ruby 0.310000 0.000000 0.310000 ( 0.835560)
35
+ #disable_ruby 0.150000 0.000000 0.150000 ( 0.679239)
36
+
37
+ #AFTER REFACTORING (PlaceholderParser and RubyPlaceholderParser)
38
+ # user system total real
39
+ #enable_ruby 0.290000 0.010000 0.300000 ( 0.815281)
40
+ #disable_ruby 0.060000 0.000000 0.060000 ( 0.588556)
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-record-reformer
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.0
4
+ version: 0.2.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Naotoshi Seo
@@ -80,7 +80,7 @@ dependencies:
80
80
  - - '>='
81
81
  - !ruby/object:Gem::Version
82
82
  version: '0'
83
- description: Output filter plugin for reforming each event record
83
+ description: Fluentd plugin to add or replace fields of a event record
84
84
  email: sonots@gmail.com
85
85
  executables: []
86
86
  extensions: []
@@ -96,6 +96,7 @@ files:
96
96
  - Rakefile
97
97
  - fluent-plugin-record-reformer.gemspec
98
98
  - lib/fluent/plugin/out_record_reformer.rb
99
+ - spec/out_record_reformer_bench.rb
99
100
  - spec/out_record_reformer_spec.rb
100
101
  - spec/spec_helper.rb
101
102
  homepage: https://github.com/sonots/fluent-plugin-record-reformer
@@ -121,7 +122,8 @@ rubyforge_project:
121
122
  rubygems_version: 2.0.3
122
123
  signing_key:
123
124
  specification_version: 4
124
- summary: Output filter plugin for reforming each event record
125
+ summary: Fluentd plugin to add or replace fields of a event record
125
126
  test_files:
127
+ - spec/out_record_reformer_bench.rb
126
128
  - spec/out_record_reformer_spec.rb
127
129
  - spec/spec_helper.rb