fluent-plugin-yohoushi 0.0.3 → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +7 -0
- data/README.md +28 -5
- data/fluent-plugin-yohoushi.gemspec +2 -2
- data/lib/fluent/plugin/out_yohoushi.rb +105 -15
- data/spec/out_yohoushi_spec.rb +50 -10
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4e938d9760ec8f7885633d08d9909f17274b9b42
|
4
|
+
data.tar.gz: ea2a1c8be04c55d11c0b8a15bee8712fa8166b7e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 57efe427de701f149cc1932cf0daab759c36ae18021a15f5f1a7a52fc4e192bc08225d942eae37f0a6920d6ce888ed09be649c267b958f7a775be193415cae84
|
7
|
+
data.tar.gz: 2410dd562c892fee6ab12ec50c9f9c46542fd4f900ff0d3dde090ff433451e21372258a131516b208807b9cc1ace7d85e769040080daf369bd60d521c9a95966
|
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
@@ -66,6 +66,10 @@ then fluent-plugin-yohoushi posts data to yohoshi similarly like
|
|
66
66
|
|
67
67
|
The graph mode (either of gauge, count, modified, or derive). Just same as mode of GrowthForecast POST parameter. Default is gauge.
|
68
68
|
|
69
|
+
- enable\_ruby *bool*
|
70
|
+
|
71
|
+
Enable to use ruby codes in placeholders. See Placeholders section. Default is true (just for lower version compatibility).
|
72
|
+
|
69
73
|
### Placeholders
|
70
74
|
|
71
75
|
The keys of input json are available as placeholders. In the above example,
|
@@ -76,17 +80,36 @@ The keys of input json are available as placeholders. In the above example,
|
|
76
80
|
shall be available. In addition, following placeholders are reserved:
|
77
81
|
|
78
82
|
* ${hostname} hostname
|
79
|
-
* ${tag} input tag
|
80
|
-
* ${tag\_parts} input tag splitted by '.'
|
81
|
-
* ${tags} input tag splitted by '.' (obsolete)
|
82
83
|
* ${time} time of the event
|
83
84
|
* ${key} the matched key value with `key_pattern` or `key1`, `key2`, ...
|
85
|
+
* ${tag} input tag
|
86
|
+
* ${tags[N]} (Obsolete. Use tag\_parts) Input tag splitted by '.'
|
87
|
+
* ${tag\_parts[N]} Input tag splitted by '.' indexed with N such as `${tag_parts[0]}`, `${tag_parts[-1]}`.
|
88
|
+
* ${tag\_prefix[N]} Tag parts before and on the index N. For example,
|
84
89
|
|
85
|
-
|
90
|
+
Input tag: prefix.test.tag.suffix
|
91
|
+
|
92
|
+
${tag_prefix[0]} => prefix
|
93
|
+
${tag_prefix[1]} => prefix.test
|
94
|
+
${tag_prefix[-2]} => prefix.test.tag
|
95
|
+
${tag_prefix[-1]} => prefix.test.tag.suffix
|
96
|
+
|
97
|
+
* ${tag\_suffix[N]} Tag parts after and on the index N. For example,
|
98
|
+
|
99
|
+
Input tag: prefix.test.tag.suffix
|
100
|
+
|
101
|
+
${tag_suffix[0]} => prefix.test.tag.suffix
|
102
|
+
${tag_suffix[1]} => test.tag.suffix
|
103
|
+
${tag_suffix[-2]} => tag.suffix
|
104
|
+
${tag_suffix[-1]} => suffix
|
105
|
+
|
106
|
+
It is also possible to write a ruby code in placeholders if you set `enable_ruby true` option, so you may write some codes as
|
86
107
|
|
87
108
|
* ${time.strftime('%Y-%m-%dT%H:%M:%S%z')}
|
88
109
|
* ${tag\_parts.last}
|
89
110
|
|
111
|
+
but, please note that enabling ruby codes is not encouraged by security reasons and also in terms of the performance.
|
112
|
+
|
90
113
|
## ChangeLog
|
91
114
|
|
92
115
|
See [CHANGELOG.md](CHANGELOG.md) for details.
|
@@ -101,5 +124,5 @@ See [CHANGELOG.md](CHANGELOG.md) for details.
|
|
101
124
|
|
102
125
|
## Copyright
|
103
126
|
|
104
|
-
Copyright (c) 2013 Naotoshi
|
127
|
+
Copyright (c) 2013 Naotoshi Seo. See [LICENSE](LICENSE) for details.
|
105
128
|
|
@@ -3,11 +3,11 @@ $:.push File.expand_path("../lib", __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = "fluent-plugin-yohoushi"
|
6
|
-
s.version = "0.0.
|
6
|
+
s.version = "0.0.4"
|
7
7
|
s.authors = ["Naotoshi Seo"]
|
8
8
|
s.email = ["sonots@gmail.com"]
|
9
9
|
s.homepage = "https://github.com/sonots/fluent-plugin-yohoushi"
|
10
|
-
s.summary = "
|
10
|
+
s.summary = "Fluentd plugin to post data to yohoushi"
|
11
11
|
s.description = s.summary
|
12
12
|
s.licenses = ["MIT"]
|
13
13
|
|
@@ -30,6 +30,7 @@ class Fluent::YohoushiOutput < Fluent::Output
|
|
30
30
|
raise Fluent::ConfigError, "stdout output output_type should be `gauge`, `count`, `modified`, or `derive`"
|
31
31
|
end
|
32
32
|
end
|
33
|
+
config_param :enable_ruby, :bool, :default => true # true for lower version compatibility
|
33
34
|
|
34
35
|
# for test
|
35
36
|
attr_reader :client
|
@@ -70,6 +71,17 @@ class Fluent::YohoushiOutput < Fluent::Output
|
|
70
71
|
end
|
71
72
|
raise Fluent::ConfigError, "Either of `key_pattern` or `key1` must be specified" if (@key_pattern.nil? and @keys.empty?)
|
72
73
|
|
74
|
+
@placeholder_expander =
|
75
|
+
if @enable_ruby
|
76
|
+
# require utilities which would be used in ruby placeholders
|
77
|
+
require 'pathname'
|
78
|
+
require 'uri'
|
79
|
+
require 'cgi'
|
80
|
+
RubyPlaceholderExpander.new
|
81
|
+
else
|
82
|
+
PlaceholderExpander.new
|
83
|
+
end
|
84
|
+
|
73
85
|
@hostname = Socket.gethostname
|
74
86
|
rescue => e
|
75
87
|
raise Fluent::ConfigError, "#{e.class} #{e.message} #{e.backtrace.first}"
|
@@ -95,11 +107,22 @@ class Fluent::YohoushiOutput < Fluent::Output
|
|
95
107
|
|
96
108
|
def emit(tag, es, chain)
|
97
109
|
tag_parts = tag.split('.')
|
110
|
+
tag_prefix = tag_prefix(tag_parts)
|
111
|
+
tag_suffix = tag_suffix(tag_parts)
|
112
|
+
placeholders = {
|
113
|
+
'tag' => tag,
|
114
|
+
'tags' => tag_parts, # for lower compatibility
|
115
|
+
'tag_parts' => tag_parts,
|
116
|
+
'tag_prefix' => tag_prefix,
|
117
|
+
'tag_suffix' => tag_suffix,
|
118
|
+
'hostname' => @hostname,
|
119
|
+
}
|
98
120
|
if @key_pattern
|
99
121
|
es.each do |time, record|
|
100
122
|
record.each do |key, value|
|
101
123
|
next unless key =~ @key_pattern
|
102
|
-
|
124
|
+
placeholders['key'] = key
|
125
|
+
path = expand_placeholder(@key_pattern_path, time, record, placeholders)
|
103
126
|
post(path, value)
|
104
127
|
end
|
105
128
|
end
|
@@ -107,7 +130,8 @@ class Fluent::YohoushiOutput < Fluent::Output
|
|
107
130
|
es.each do |time, record|
|
108
131
|
@keys.each do |key, path|
|
109
132
|
next unless value = record[key]
|
110
|
-
|
133
|
+
placeholders['key'] = key
|
134
|
+
path = expand_placeholder(path, time, record, placeholders)
|
111
135
|
post(path, value)
|
112
136
|
end
|
113
137
|
end
|
@@ -118,21 +142,87 @@ class Fluent::YohoushiOutput < Fluent::Output
|
|
118
142
|
$log.warn "out_yohoushi: #{e.class} #{e.message} #{e.backtrace.first}"
|
119
143
|
end
|
120
144
|
|
121
|
-
def expand_placeholder(
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
145
|
+
def expand_placeholder(value, time, record, opts)
|
146
|
+
@placeholder_expander.prepare_placeholders(time, record, opts)
|
147
|
+
@placeholder_expander.expand(value)
|
148
|
+
end
|
149
|
+
|
150
|
+
def tag_prefix(tag_parts)
|
151
|
+
return [] if tag_parts.empty?
|
152
|
+
tag_prefix = [tag_parts.first]
|
153
|
+
1.upto(tag_parts.size-1).each do |i|
|
154
|
+
tag_prefix[i] = "#{tag_prefix[i-1]}.#{tag_parts[i]}"
|
155
|
+
end
|
156
|
+
tag_prefix
|
157
|
+
end
|
158
|
+
|
159
|
+
def tag_suffix(tag_parts)
|
160
|
+
return [] if tag_parts.empty?
|
161
|
+
rev_tag_parts = tag_parts.reverse
|
162
|
+
rev_tag_suffix = [rev_tag_parts.first]
|
163
|
+
1.upto(tag_parts.size-1).each do |i|
|
164
|
+
rev_tag_suffix[i] = "#{rev_tag_parts[i]}.#{rev_tag_suffix[i-1]}"
|
165
|
+
end
|
166
|
+
rev_tag_suffix.reverse
|
167
|
+
end
|
168
|
+
|
169
|
+
class PlaceholderExpander
|
170
|
+
attr_reader :placeholders
|
171
|
+
|
172
|
+
def prepare_placeholders(time, record, opts)
|
173
|
+
placeholders = { '${time}' => time.to_s }
|
174
|
+
record.each {|key, val| placeholders.store("${#{key}}", val) }
|
175
|
+
|
176
|
+
opts.each do |key, val|
|
177
|
+
if val.kind_of?(Array)
|
178
|
+
size = val.size
|
179
|
+
val.each_with_index { |t, idx|
|
180
|
+
placeholders.store("${#{key}[#{idx}]}", t)
|
181
|
+
placeholders.store("${#{key}[#{idx-size}]}", t) # support [-1]
|
182
|
+
}
|
183
|
+
else # string, interger, float, and others?
|
184
|
+
placeholders["${#{key}}"] = val
|
185
|
+
end
|
186
|
+
end
|
187
|
+
|
188
|
+
@placeholders = placeholders
|
189
|
+
end
|
190
|
+
|
191
|
+
def expand(str)
|
192
|
+
str.gsub(/(\${[a-z_]+(\[-?[0-9]+\])?}|__[A-Z_]+__)/) {
|
193
|
+
$log.warn "record_reformer: unknown placeholder `#{$1}` found" unless @placeholders.include?($1)
|
194
|
+
@placeholders[$1]
|
195
|
+
}
|
196
|
+
end
|
131
197
|
end
|
132
198
|
|
133
|
-
class
|
134
|
-
|
135
|
-
|
199
|
+
class RubyPlaceholderExpander
|
200
|
+
attr_reader :placeholders
|
201
|
+
|
202
|
+
# Get placeholders as a struct
|
203
|
+
#
|
204
|
+
# @param [Time] time the time
|
205
|
+
# @param [Hash] record the record
|
206
|
+
# @param [Hash] opts others
|
207
|
+
def prepare_placeholders(time, record, opts)
|
208
|
+
struct = UndefOpenStruct.new(record)
|
209
|
+
struct.time = time
|
210
|
+
opts.each {|key, val| struct.__send__("#{key}=", val) }
|
211
|
+
@placeholders = struct
|
212
|
+
end
|
213
|
+
|
214
|
+
# Replace placeholders in a string
|
215
|
+
#
|
216
|
+
# @param [String] str the string to be replaced
|
217
|
+
def expand(str)
|
218
|
+
str = str.gsub(/\$\{([^}]+)\}/, '#{\1}') # ${..} => #{..}
|
219
|
+
eval "\"#{str}\"", @placeholders.instance_eval { binding }
|
220
|
+
end
|
221
|
+
|
222
|
+
class UndefOpenStruct < OpenStruct
|
223
|
+
(Object.instance_methods).each do |m|
|
224
|
+
undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member/
|
225
|
+
end
|
136
226
|
end
|
137
227
|
end
|
138
228
|
end
|
data/spec/out_yohoushi_spec.rb
CHANGED
@@ -139,21 +139,61 @@ describe Fluent::YohoushiOutput do
|
|
139
139
|
end
|
140
140
|
end
|
141
141
|
|
142
|
-
describe 'expand_placeholder' do
|
143
|
-
let(:config) { %[mapping1 / http://foo
|
142
|
+
describe 'expand_placeholder with enable_ruby true' do
|
143
|
+
let(:config) { %[mapping1 / http://foo] }
|
144
144
|
let(:tag) { 'fluent.error' }
|
145
|
-
let(:record) { { 'foo_count' => "1", 'bar_count' => "1" } }
|
146
|
-
let(:tag_parts) { tag.split('.') }
|
147
145
|
let(:time) { Time.now.to_i }
|
146
|
+
let(:record) { { 'foo_count' => "1" } }
|
147
|
+
let(:emit) { driver.run { driver.emit(record, time) } }
|
148
|
+
let(:expected_path) { '/fluent/error/fluent.error/1/foo_count' }
|
149
|
+
let(:expected_value) { '1' }
|
150
|
+
before { driver.instance.should_receive(:post).with(expected_path, expected_value) }
|
148
151
|
|
149
|
-
context '
|
150
|
-
let(:
|
151
|
-
|
152
|
+
context 'keyN' do
|
153
|
+
let(:config) {%[
|
154
|
+
mapping1 / http://foobar
|
155
|
+
key1 foo_count /${tags.first}/${tag_parts.last}/${tag_prefix[1]}/${foo_count}/${CGI.unescape(key)}
|
156
|
+
enable_ruby true
|
157
|
+
]}
|
158
|
+
it { emit }
|
159
|
+
end
|
160
|
+
|
161
|
+
context 'key_pattern' do
|
162
|
+
let(:config) {%[
|
163
|
+
mapping1 / http://foobar
|
164
|
+
key_pattern _count$ /${tags.first}/${tag_parts.last}/${tag_prefix[1]}/${foo_count}/${URI.unescape(key)}
|
165
|
+
enable_ruby true
|
166
|
+
]}
|
167
|
+
it { emit }
|
168
|
+
end
|
169
|
+
end
|
170
|
+
|
171
|
+
describe 'expand_placeholder with enable_ruby false' do
|
172
|
+
let(:config) { %[mapping1 / http://foo] }
|
173
|
+
let(:tag) { 'fluent.error' }
|
174
|
+
let(:time) { Time.now.to_i }
|
175
|
+
let(:record) { { 'foo_count' => "1" } }
|
176
|
+
let(:emit) { driver.run { driver.emit(record, time) } }
|
177
|
+
let(:expected_path) { '/fluent/error/fluent.error/1/foo_count' }
|
178
|
+
let(:expected_value) { '1' }
|
179
|
+
before { driver.instance.should_receive(:post).with(expected_path, expected_value) }
|
180
|
+
|
181
|
+
context 'keyN' do
|
182
|
+
let(:config) {%[
|
183
|
+
mapping1 / http://foobar
|
184
|
+
key1 foo_count /${tags[0]}/${tag_parts[-1]}/${tag_prefix[1]}/${foo_count}/${key}
|
185
|
+
enable_ruby false
|
186
|
+
]}
|
187
|
+
it { emit }
|
152
188
|
end
|
153
189
|
|
154
|
-
context '
|
155
|
-
let(:
|
156
|
-
|
190
|
+
context 'key_pattern' do
|
191
|
+
let(:config) {%[
|
192
|
+
mapping1 / http://foobar
|
193
|
+
key_pattern _count$ /${tags[0]}/${tag_parts[-1]}/${tag_prefix[1]}/${foo_count}/${key}
|
194
|
+
enable_ruby false
|
195
|
+
]}
|
196
|
+
it { emit }
|
157
197
|
end
|
158
198
|
end
|
159
199
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-yohoushi
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Naotoshi Seo
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2014-01-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -108,7 +108,7 @@ dependencies:
|
|
108
108
|
- - '>='
|
109
109
|
- !ruby/object:Gem::Version
|
110
110
|
version: '0'
|
111
|
-
description:
|
111
|
+
description: Fluentd plugin to post data to yohoushi
|
112
112
|
email:
|
113
113
|
- sonots@gmail.com
|
114
114
|
executables: []
|
@@ -151,7 +151,7 @@ rubyforge_project: fluent-plugin-yohoushi
|
|
151
151
|
rubygems_version: 2.0.3
|
152
152
|
signing_key:
|
153
153
|
specification_version: 4
|
154
|
-
summary:
|
154
|
+
summary: Fluentd plugin to post data to yohoushi
|
155
155
|
test_files:
|
156
156
|
- spec/out_yohoushi_spec.rb
|
157
157
|
- spec/spec_helper.rb
|