fluent-plugin-record-reformer 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -2
- data/README.md +1 -1
- data/fluent-plugin-record-reformer.gemspec +2 -2
- data/lib/fluent/plugin/out_record_reformer.rb +72 -57
- data/spec/out_record_reformer_bench.rb +40 -0
- metadata +5 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 55ba6719a96af4f1c33c52d970579a9006ef9a7c
|
4
|
+
data.tar.gz: 15cd6f42ed2f5b0b995f20560cf9974d12e23bda
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8758e09cd82e2f2fd50076811bf7a25961a2782f5ac6081185567ff8e874b1bb312aeaae546acfa17411a26f6c0af2c85ed47af99416cb7448ac4a65176e5815
|
7
|
+
data.tar.gz: 4ae98eedf0aafea25057b67a9d2f2f189f54c1063d00ecd70f90075c317171089e614bc3f00115f027037ca5a23eee51c0c7b3c75f6f8b150174792f1c0cf972
|
data/CHANGELOG.md
CHANGED
@@ -1,10 +1,16 @@
|
|
1
|
+
## 0.2.1 (2014/01/15)
|
2
|
+
|
3
|
+
Enhancement:
|
4
|
+
|
5
|
+
* Speed up
|
6
|
+
|
1
7
|
## 0.2.0 (2014/01/15)
|
2
8
|
|
3
9
|
Enhancement:
|
4
10
|
|
5
|
-
* Support a record directive
|
11
|
+
* Support a `record` directive
|
6
12
|
* Add `remove_keys` option
|
7
|
-
* Add `
|
13
|
+
* Add `renew_record` option
|
8
14
|
* Add `enable_ruby` option
|
9
15
|
|
10
16
|
## 0.1.1 (2013/11/21)
|
data/README.md
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
[](http://travis-ci.org/sonots/fluent-plugin-record-reformer)
|
4
4
|
|
5
|
-
Fluentd
|
5
|
+
Fluentd plugin to add or replace fields of a event record
|
6
6
|
|
7
7
|
## Installation
|
8
8
|
|
@@ -3,11 +3,11 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |gem|
|
5
5
|
gem.name = "fluent-plugin-record-reformer"
|
6
|
-
gem.version = "0.2.
|
6
|
+
gem.version = "0.2.1"
|
7
7
|
gem.authors = ["Naotoshi Seo"]
|
8
8
|
gem.email = "sonots@gmail.com"
|
9
9
|
gem.homepage = "https://github.com/sonots/fluent-plugin-record-reformer"
|
10
|
-
gem.description = "
|
10
|
+
gem.description = "Fluentd plugin to add or replace fields of a event record"
|
11
11
|
gem.summary = gem.description
|
12
12
|
gem.licenses = ["MIT"]
|
13
13
|
gem.has_rdoc = false
|
@@ -40,11 +40,11 @@ module Fluent
|
|
40
40
|
@remove_keys = @remove_keys.split(',')
|
41
41
|
end
|
42
42
|
|
43
|
-
@
|
43
|
+
@placeholder_expander =
|
44
44
|
if @enable_ruby
|
45
|
-
|
45
|
+
RubyPlaceholderExpander.new
|
46
46
|
else
|
47
|
-
|
47
|
+
PlaceholderExpander.new
|
48
48
|
end
|
49
49
|
|
50
50
|
@time_proc = # hmm, want to remove ${time} placeholder ...
|
@@ -60,9 +60,8 @@ module Fluent
|
|
60
60
|
def emit(tag, es, chain)
|
61
61
|
tag_parts = tag.split('.')
|
62
62
|
es.each { |time, record|
|
63
|
-
|
64
|
-
|
65
|
-
Engine.emit(output_tag, time, replace_record(record, tag, tag_parts, t_time))
|
63
|
+
new_tag, new_record = reform(@output_tag, record, tag, tag_parts, @time_proc.call(time))
|
64
|
+
Engine.emit(new_tag, time, new_record)
|
66
65
|
}
|
67
66
|
chain.next
|
68
67
|
rescue => e
|
@@ -71,69 +70,85 @@ module Fluent
|
|
71
70
|
|
72
71
|
private
|
73
72
|
|
74
|
-
def
|
73
|
+
def reform(output_tag, record, tag, tag_parts, time)
|
74
|
+
@placeholder_expander.prepare_placeholders(record, tag, tag_parts, @hostname, time)
|
75
|
+
new_tag = @placeholder_expander.expand(output_tag)
|
76
|
+
|
75
77
|
new_record = @renew_record ? {} : record.dup
|
76
|
-
@map.each_pair { |k, v|
|
77
|
-
new_record[k] = @expand_placeholder_proc.call(v, record, tag, tag_parts, time)
|
78
|
-
}
|
78
|
+
@map.each_pair { |k, v| new_record[k] = @placeholder_expander.expand(v) }
|
79
79
|
@remove_keys.each { |k| new_record.delete(k) } if @remove_keys
|
80
|
-
|
80
|
+
|
81
|
+
[new_tag, new_record]
|
81
82
|
end
|
82
83
|
|
83
|
-
|
84
|
+
class PlaceholderExpander
|
84
85
|
# referenced https://github.com/fluent/fluent-plugin-rewrite-tag-filter, thanks!
|
85
|
-
placeholders
|
86
|
-
str.gsub(/(\${[a-z_]+(\[-?[0-9]+\])?}|__[A-Z_]+__)/) do
|
87
|
-
$log.warn "record_reformer: unknown placeholder `#{$1}` found in a tag `#{tag}`" unless placeholders.include?($1)
|
88
|
-
placeholders[$1]
|
89
|
-
end
|
90
|
-
end
|
86
|
+
attr_reader :placeholders
|
91
87
|
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
88
|
+
def prepare_placeholders(record, tag, tag_parts, hostname, time)
|
89
|
+
placeholders = {
|
90
|
+
'${time}' => time,
|
91
|
+
'${tag}' => tag,
|
92
|
+
'${hostname}' => hostname,
|
93
|
+
}
|
98
94
|
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
95
|
+
size = tag_parts.size
|
96
|
+
tag_parts.each_with_index do |t, idx|
|
97
|
+
placeholders.store("${tag_parts[#{idx}]}", t)
|
98
|
+
placeholders.store("${tag_parts[#{idx-size}]}", t) # support tag_parts[-1]
|
99
|
+
end
|
100
|
+
# tags is just for old version compatibility
|
101
|
+
tag_parts.each_with_index do |t, idx|
|
102
|
+
placeholders.store("${tags[#{idx}]}", t)
|
103
|
+
placeholders.store("${tags[#{idx-size}]}", t) # support tags[-1]
|
104
|
+
end
|
109
105
|
|
110
|
-
|
111
|
-
|
112
|
-
|
106
|
+
record.each {|k, v|
|
107
|
+
placeholders.store("${#{k}}", v)
|
108
|
+
}
|
113
109
|
|
114
|
-
|
115
|
-
|
110
|
+
@placeholders = placeholders
|
111
|
+
end
|
116
112
|
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
# @param [Time] time the time
|
124
|
-
def expand_ruby_placeholder(str, record, tag, tag_parts, time)
|
125
|
-
struct = UndefOpenStruct.new(record)
|
126
|
-
struct.tag = tag
|
127
|
-
struct.tags = struct.tag_parts = tag_parts # tags is for old version compatibility
|
128
|
-
struct.time = time
|
129
|
-
struct.hostname = @hostname
|
130
|
-
str = str.gsub(/\$\{([^}]+)\}/, '#{\1}') # ${..} => #{..}
|
131
|
-
eval "\"#{str}\"", struct.instance_eval { binding }
|
113
|
+
def expand(str)
|
114
|
+
str.gsub(/(\${[a-z_]+(\[-?[0-9]+\])?}|__[A-Z_]+__)/) do
|
115
|
+
$log.warn "record_reformer: unknown placeholder `#{$1}` found in a tag `#{tag}`" unless @placeholders.include?($1)
|
116
|
+
@placeholders[$1]
|
117
|
+
end
|
118
|
+
end
|
132
119
|
end
|
133
120
|
|
134
|
-
class
|
135
|
-
|
136
|
-
|
121
|
+
class RubyPlaceholderExpander
|
122
|
+
attr_reader :placeholders
|
123
|
+
|
124
|
+
# Get placeholders as a struct
|
125
|
+
#
|
126
|
+
# @param [Hash] record the record, one of information
|
127
|
+
# @param [String] tag the tag
|
128
|
+
# @param [Array] tag_parts the tag parts (tag splitted by .)
|
129
|
+
# @param [String] hostname the hostname
|
130
|
+
# @param [Time] time the time
|
131
|
+
def prepare_placeholders(record, tag, tag_parts, hostname, time)
|
132
|
+
struct = UndefOpenStruct.new(record)
|
133
|
+
struct.tag = tag
|
134
|
+
struct.tags = struct.tag_parts = tag_parts # tags is for old version compatibility
|
135
|
+
struct.time = time
|
136
|
+
struct.hostname = hostname
|
137
|
+
@placeholders = struct
|
138
|
+
end
|
139
|
+
|
140
|
+
# Replace placeholders in a string
|
141
|
+
#
|
142
|
+
# @param [String] str the string to be replaced
|
143
|
+
def expand(str)
|
144
|
+
str = str.gsub(/\$\{([^}]+)\}/, '#{\1}') # ${..} => #{..}
|
145
|
+
eval "\"#{str}\"", @placeholders.instance_eval { binding }
|
146
|
+
end
|
147
|
+
|
148
|
+
class UndefOpenStruct < OpenStruct
|
149
|
+
(Object.instance_methods).each do |m|
|
150
|
+
undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member/
|
151
|
+
end
|
137
152
|
end
|
138
153
|
end
|
139
154
|
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
require_relative 'spec_helper'
|
3
|
+
require 'benchmark'
|
4
|
+
Fluent::Test.setup
|
5
|
+
|
6
|
+
def create_driver(config, tag = 'foo.bar')
|
7
|
+
Fluent::Test::OutputTestDriver.new(Fluent::RecordReformerOutput, tag).configure(config)
|
8
|
+
end
|
9
|
+
|
10
|
+
# setup
|
11
|
+
message = {'message' => "2013/01/13T07:02:11.124202 INFO GET /ping"}
|
12
|
+
time = Time.now.to_i
|
13
|
+
|
14
|
+
enable_ruby_driver = create_driver(%[
|
15
|
+
enable_ruby true
|
16
|
+
output_tag reformed.${tag}
|
17
|
+
message ${tag_parts[0]}
|
18
|
+
])
|
19
|
+
disable_ruby_driver = create_driver(%[
|
20
|
+
enable_ruby false
|
21
|
+
output_tag reformed.${tag}
|
22
|
+
message ${tag_parts[0]}
|
23
|
+
])
|
24
|
+
|
25
|
+
# bench
|
26
|
+
n = 1000
|
27
|
+
Benchmark.bm(7) do |x|
|
28
|
+
x.report("enable_ruby") { enable_ruby_driver.run { n.times { enable_ruby_driver.emit(message, time) } } }
|
29
|
+
x.report("disable_ruby") { disable_ruby_driver.run { n.times { disable_ruby_driver.emit(message, time) } } }
|
30
|
+
end
|
31
|
+
|
32
|
+
#BEFORE REFACTORING
|
33
|
+
# user system total real
|
34
|
+
#enable_ruby 0.310000 0.000000 0.310000 ( 0.835560)
|
35
|
+
#disable_ruby 0.150000 0.000000 0.150000 ( 0.679239)
|
36
|
+
|
37
|
+
#AFTER REFACTORING (PlaceholderParser and RubyPlaceholderParser)
|
38
|
+
# user system total real
|
39
|
+
#enable_ruby 0.290000 0.010000 0.300000 ( 0.815281)
|
40
|
+
#disable_ruby 0.060000 0.000000 0.060000 ( 0.588556)
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-record-reformer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Naotoshi Seo
|
@@ -80,7 +80,7 @@ dependencies:
|
|
80
80
|
- - '>='
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: '0'
|
83
|
-
description:
|
83
|
+
description: Fluentd plugin to add or replace fields of a event record
|
84
84
|
email: sonots@gmail.com
|
85
85
|
executables: []
|
86
86
|
extensions: []
|
@@ -96,6 +96,7 @@ files:
|
|
96
96
|
- Rakefile
|
97
97
|
- fluent-plugin-record-reformer.gemspec
|
98
98
|
- lib/fluent/plugin/out_record_reformer.rb
|
99
|
+
- spec/out_record_reformer_bench.rb
|
99
100
|
- spec/out_record_reformer_spec.rb
|
100
101
|
- spec/spec_helper.rb
|
101
102
|
homepage: https://github.com/sonots/fluent-plugin-record-reformer
|
@@ -121,7 +122,8 @@ rubyforge_project:
|
|
121
122
|
rubygems_version: 2.0.3
|
122
123
|
signing_key:
|
123
124
|
specification_version: 4
|
124
|
-
summary:
|
125
|
+
summary: Fluentd plugin to add or replace fields of a event record
|
125
126
|
test_files:
|
127
|
+
- spec/out_record_reformer_bench.rb
|
126
128
|
- spec/out_record_reformer_spec.rb
|
127
129
|
- spec/spec_helper.rb
|