fluent-plugin-hoop 0.1.3 → 0.1.4
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile +3 -3
- data/Rakefile +0 -8
- data/VERSION +1 -1
- data/fluent-plugin-hoop.gemspec +11 -8
- data/lib/fluent/plugin/out_hoop.rb +4 -220
- data/test/plugin/test_out_hoop.rb +1 -1
- metadata +33 -17
data/Gemfile
CHANGED
@@ -7,10 +7,10 @@ source "http://rubygems.org"
|
|
7
7
|
# Include everything needed to run rake, tests, features, etc.
|
8
8
|
group :development do
|
9
9
|
gem "shoulda", ">= 0"
|
10
|
-
gem "bundler"
|
10
|
+
gem "bundler"
|
11
11
|
gem "jeweler", "~> 1.6.4"
|
12
|
-
gem "rcov", ">= 0"
|
13
12
|
end
|
14
13
|
|
15
14
|
gem "rdoc"
|
16
|
-
|
15
|
+
gem "fluentd"
|
16
|
+
gem "fluent-mixin-plaintextformatter"
|
data/Rakefile
CHANGED
@@ -43,14 +43,6 @@ Rake::TestTask.new(:test) do |test|
|
|
43
43
|
test.verbose = true
|
44
44
|
end
|
45
45
|
|
46
|
-
require 'rcov/rcovtask'
|
47
|
-
Rcov::RcovTask.new do |test|
|
48
|
-
test.libs << 'test'
|
49
|
-
test.pattern = 'test/**/test_*.rb'
|
50
|
-
test.verbose = true
|
51
|
-
test.rcov_opts << '--exclude "gems/*"'
|
52
|
-
end
|
53
|
-
|
54
46
|
task :default => :test
|
55
47
|
|
56
48
|
require 'rdoc/task'
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.1.
|
1
|
+
0.1.4
|
data/fluent-plugin-hoop.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = "fluent-plugin-hoop"
|
8
|
-
s.version = "0.1.
|
8
|
+
s.version = "0.1.4"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["TAGOMORI Satoshi"]
|
12
|
-
s.date = "2012-
|
12
|
+
s.date = "2012-07-13"
|
13
13
|
s.description = "This plugin doesn't support Apache Hadoop's HttpFs. See fluent-plugin-webhdfs."
|
14
14
|
s.email = "tagomoris@gmail.com"
|
15
15
|
s.extra_rdoc_files = [
|
@@ -44,29 +44,32 @@ Gem::Specification.new do |s|
|
|
44
44
|
|
45
45
|
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
46
46
|
s.add_runtime_dependency(%q<rdoc>, [">= 0"])
|
47
|
+
s.add_runtime_dependency(%q<fluentd>, [">= 0"])
|
48
|
+
s.add_runtime_dependency(%q<fluent-mixin-plaintextformatter>, [">= 0"])
|
47
49
|
s.add_development_dependency(%q<shoulda>, [">= 0"])
|
48
|
-
s.add_development_dependency(%q<bundler>, ["
|
50
|
+
s.add_development_dependency(%q<bundler>, [">= 0"])
|
49
51
|
s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
|
50
|
-
s.add_development_dependency(%q<rcov>, [">= 0"])
|
51
52
|
s.add_runtime_dependency(%q<fluentd>, ["~> 0.10.8"])
|
52
53
|
s.add_development_dependency(%q<rake>, [">= 0.9.2"])
|
53
54
|
s.add_development_dependency(%q<simplecov>, [">= 0.5.4"])
|
54
55
|
else
|
55
56
|
s.add_dependency(%q<rdoc>, [">= 0"])
|
57
|
+
s.add_dependency(%q<fluentd>, [">= 0"])
|
58
|
+
s.add_dependency(%q<fluent-mixin-plaintextformatter>, [">= 0"])
|
56
59
|
s.add_dependency(%q<shoulda>, [">= 0"])
|
57
|
-
s.add_dependency(%q<bundler>, ["
|
60
|
+
s.add_dependency(%q<bundler>, [">= 0"])
|
58
61
|
s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
|
59
|
-
s.add_dependency(%q<rcov>, [">= 0"])
|
60
62
|
s.add_dependency(%q<fluentd>, ["~> 0.10.8"])
|
61
63
|
s.add_dependency(%q<rake>, [">= 0.9.2"])
|
62
64
|
s.add_dependency(%q<simplecov>, [">= 0.5.4"])
|
63
65
|
end
|
64
66
|
else
|
65
67
|
s.add_dependency(%q<rdoc>, [">= 0"])
|
68
|
+
s.add_dependency(%q<fluentd>, [">= 0"])
|
69
|
+
s.add_dependency(%q<fluent-mixin-plaintextformatter>, [">= 0"])
|
66
70
|
s.add_dependency(%q<shoulda>, [">= 0"])
|
67
|
-
s.add_dependency(%q<bundler>, ["
|
71
|
+
s.add_dependency(%q<bundler>, [">= 0"])
|
68
72
|
s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
|
69
|
-
s.add_dependency(%q<rcov>, [">= 0"])
|
70
73
|
s.add_dependency(%q<fluentd>, ["~> 0.10.8"])
|
71
74
|
s.add_dependency(%q<rake>, [">= 0.9.2"])
|
72
75
|
s.add_dependency(%q<simplecov>, [">= 0.5.4"])
|
@@ -1,224 +1,16 @@
|
|
1
|
-
|
2
|
-
module FluentExt::PlainTextFormatterMixin
|
3
|
-
# config_param :output_data_type, :string, :default => 'json' # or 'attr:field' or 'attr:field1,field2,field3(...)'
|
4
|
-
|
5
|
-
attr_accessor :output_include_time, :output_include_tag, :output_data_type
|
6
|
-
attr_accessor :add_newline, :field_separator
|
7
|
-
attr_accessor :remove_prefix, :default_tag
|
8
|
-
|
9
|
-
def configure(conf)
|
10
|
-
super
|
11
|
-
|
12
|
-
@output_include_time = Fluent::Config.bool_value(conf['output_include_time'])
|
13
|
-
@output_include_time = true if @output_include_time.nil?
|
14
|
-
|
15
|
-
@output_include_tag = Fluent::Config.bool_value(conf['output_include_tag'])
|
16
|
-
@output_include_tag = true if @output_include_tag.nil?
|
17
|
-
|
18
|
-
@output_data_type = conf['output_data_type']
|
19
|
-
@output_data_type = 'json' if @output_data_type.nil?
|
20
|
-
|
21
|
-
@field_separator = case conf['field_separator']
|
22
|
-
when 'SPACE' then ' '
|
23
|
-
when 'COMMA' then ','
|
24
|
-
else "\t"
|
25
|
-
end
|
26
|
-
@add_newline = Fluent::Config.bool_value(conf['add_newline'])
|
27
|
-
if @add_newline.nil?
|
28
|
-
@add_newline = true
|
29
|
-
end
|
30
|
-
|
31
|
-
@remove_prefix = conf['remove_prefix']
|
32
|
-
if @remove_prefix
|
33
|
-
@removed_prefix_string = @remove_prefix + '.'
|
34
|
-
@removed_length = @removed_prefix_string.length
|
35
|
-
end
|
36
|
-
if @output_include_tag and @remove_prefix and @remove_prefix.length > 0
|
37
|
-
@default_tag = conf['default_tag']
|
38
|
-
if @default_tag.nil? or @default_tag.length < 1
|
39
|
-
raise Fluent::ConfigError, "Missing 'default_tag' with output_include_tag and remove_prefix."
|
40
|
-
end
|
41
|
-
end
|
42
|
-
|
43
|
-
# default timezone: utc
|
44
|
-
if conf['localtime'].nil? and conf['utc'].nil?
|
45
|
-
@utc = true
|
46
|
-
@localtime = false
|
47
|
-
elsif not @localtime and not @utc
|
48
|
-
@utc = true
|
49
|
-
@localtime = false
|
50
|
-
end
|
51
|
-
# mix-in default time formatter (or you can overwrite @timef on your own configure)
|
52
|
-
@timef = @output_include_time ? Fluent::TimeFormatter.new(@time_format, @localtime) : nil
|
53
|
-
|
54
|
-
@custom_attributes = []
|
55
|
-
if @output_data_type == 'json'
|
56
|
-
self.instance_eval {
|
57
|
-
def stringify_record(record)
|
58
|
-
record.to_json
|
59
|
-
end
|
60
|
-
}
|
61
|
-
elsif @output_data_type =~ /^attr:(.*)$/
|
62
|
-
@custom_attributes = $1.split(',')
|
63
|
-
if @custom_attributes.size > 1
|
64
|
-
self.instance_eval {
|
65
|
-
def stringify_record(record)
|
66
|
-
@custom_attributes.map{|attr| (record[attr] || 'NULL').to_s}.join(@field_separator)
|
67
|
-
end
|
68
|
-
}
|
69
|
-
elsif @custom_attributes.size == 1
|
70
|
-
self.instance_eval {
|
71
|
-
def stringify_record(record)
|
72
|
-
(record[@custom_attributes[0]] || 'NULL').to_s
|
73
|
-
end
|
74
|
-
}
|
75
|
-
else
|
76
|
-
raise Fluent::ConfigError, "Invalid attributes specification: '#{@output_data_type}', needs one or more attributes."
|
77
|
-
end
|
78
|
-
else
|
79
|
-
raise Fluent::ConfigError, "Invalid output_data_type: '#{@output_data_type}'. specify 'json' or 'attr:ATTRIBUTE_NAME' or 'attr:ATTR1,ATTR2,...'"
|
80
|
-
end
|
81
|
-
|
82
|
-
if @output_include_time and @output_include_tag
|
83
|
-
if @add_newline and @remove_prefix
|
84
|
-
self.instance_eval {
|
85
|
-
def format(tag,time,record)
|
86
|
-
if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
|
87
|
-
tag == @remove_prefix
|
88
|
-
tag = tag[@removed_length..-1] || @default_tag
|
89
|
-
end
|
90
|
-
@timef.format(time) + @field_separator + tag + @field_separator + stringify_record(record) + "\n"
|
91
|
-
end
|
92
|
-
}
|
93
|
-
elsif @add_newline
|
94
|
-
self.instance_eval {
|
95
|
-
def format(tag,time,record)
|
96
|
-
@timef.format(time) + @field_separator + tag + @field_separator + stringify_record(record) + "\n"
|
97
|
-
end
|
98
|
-
}
|
99
|
-
elsif @remove_prefix
|
100
|
-
self.instance_eval {
|
101
|
-
def format(tag,time,record)
|
102
|
-
if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
|
103
|
-
tag == @remove_prefix
|
104
|
-
tag = tag[@removed_length..-1] || @default_tag
|
105
|
-
end
|
106
|
-
@timef.format(time) + @field_separator + tag + @field_separator + stringify_record(record)
|
107
|
-
end
|
108
|
-
}
|
109
|
-
else
|
110
|
-
self.instance_eval {
|
111
|
-
def format(tag,time,record)
|
112
|
-
@timef.format(time) + @field_separator + tag + @field_separator + stringify_record(record)
|
113
|
-
end
|
114
|
-
}
|
115
|
-
end
|
116
|
-
elsif @output_include_time
|
117
|
-
if @add_newline
|
118
|
-
self.instance_eval {
|
119
|
-
def format(tag,time,record);
|
120
|
-
@timef.format(time) + @field_separator + stringify_record(record) + "\n"
|
121
|
-
end
|
122
|
-
}
|
123
|
-
else
|
124
|
-
self.instance_eval {
|
125
|
-
def format(tag,time,record);
|
126
|
-
@timef.format(time) + @field_separator + stringify_record(record)
|
127
|
-
end
|
128
|
-
}
|
129
|
-
end
|
130
|
-
elsif @output_include_tag
|
131
|
-
if @add_newline and @remove_prefix
|
132
|
-
self.instance_eval {
|
133
|
-
def format(tag,time,record)
|
134
|
-
if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
|
135
|
-
tag == @remove_prefix
|
136
|
-
tag = tag[@removed_length..-1] || @default_tag
|
137
|
-
end
|
138
|
-
tag + @field_separator + stringify_record(record) + "\n"
|
139
|
-
end
|
140
|
-
}
|
141
|
-
elsif @add_newline
|
142
|
-
self.instance_eval {
|
143
|
-
def format(tag,time,record)
|
144
|
-
tag + @field_separator + stringify_record(record) + "\n"
|
145
|
-
end
|
146
|
-
}
|
147
|
-
elsif @remove_prefix
|
148
|
-
self.instance_eval {
|
149
|
-
def format(tag,time,record)
|
150
|
-
if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
|
151
|
-
tag == @remove_prefix
|
152
|
-
tag = tag[@removed_length..-1] || @default_tag
|
153
|
-
end
|
154
|
-
tag + @field_separator + stringify_record(record)
|
155
|
-
end
|
156
|
-
}
|
157
|
-
else
|
158
|
-
self.instance_eval {
|
159
|
-
def format(tag,time,record)
|
160
|
-
tag + @field_separator + stringify_record(record)
|
161
|
-
end
|
162
|
-
}
|
163
|
-
end
|
164
|
-
else # without time, tag
|
165
|
-
if @add_newline
|
166
|
-
self.instance_eval {
|
167
|
-
def format(tag,time,record);
|
168
|
-
stringify_record(record) + "\n"
|
169
|
-
end
|
170
|
-
}
|
171
|
-
else
|
172
|
-
self.instance_eval {
|
173
|
-
def format(tag,time,record);
|
174
|
-
stringify_record(record)
|
175
|
-
end
|
176
|
-
}
|
177
|
-
end
|
178
|
-
end
|
179
|
-
end
|
180
|
-
|
181
|
-
def stringify_record(record)
|
182
|
-
record.to_json
|
183
|
-
end
|
184
|
-
|
185
|
-
def format(tag, time, record)
|
186
|
-
if tag == @remove_prefix or (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length)
|
187
|
-
tag = tag[@removed_length..-1] || @default_tag
|
188
|
-
end
|
189
|
-
time_str = if @output_include_time
|
190
|
-
@timef.format(time) + @field_separator
|
191
|
-
else
|
192
|
-
''
|
193
|
-
end
|
194
|
-
tag_str = if @output_include_tag
|
195
|
-
tag + @field_separator
|
196
|
-
else
|
197
|
-
''
|
198
|
-
end
|
199
|
-
time_str + tag_str + stringify_record(record) + "\n"
|
200
|
-
end
|
201
|
-
|
202
|
-
end
|
1
|
+
require 'fluent/mixin/plaintextformatter'
|
203
2
|
|
204
3
|
class Fluent::HoopOutput < Fluent::TimeSlicedOutput
|
205
4
|
Fluent::Plugin.register_output('hoop', self)
|
206
5
|
|
207
6
|
config_set_default :buffer_type, 'memory'
|
208
7
|
config_set_default :time_slice_format, '%Y%m%d' # %Y%m%d%H
|
209
|
-
# config_param :tag_format, :string, :default => 'all' # or 'last'(last.part.of.tag => tag) or 'none'
|
210
8
|
|
211
9
|
config_param :hoop_server, :string # host:port
|
212
10
|
config_param :path, :string # /path/pattern/to/hdfs/file can use %Y %m %d %H %M %S and %T(tag, not-supported-yet)
|
213
11
|
config_param :username, :string # hoop pseudo username
|
214
12
|
|
215
|
-
include
|
216
|
-
config_set_default :output_include_time, true
|
217
|
-
config_set_default :output_include_tag, true
|
218
|
-
config_set_default :output_data_type, 'json'
|
219
|
-
config_set_default :field_separator, "\t"
|
220
|
-
config_set_default :add_newline, true
|
221
|
-
config_set_default :remove_prefix, nil
|
13
|
+
include Fluent::Mixin::PlainTextFormatter
|
222
14
|
|
223
15
|
def initialize
|
224
16
|
super
|
@@ -249,12 +41,6 @@ class Fluent::HoopOutput < Fluent::TimeSlicedOutput
|
|
249
41
|
end
|
250
42
|
@conn = nil
|
251
43
|
@header = {'Content-Type' => 'application/octet-stream'}
|
252
|
-
|
253
|
-
@f_separator = case @field_separator
|
254
|
-
when 'SPACE' then ' '
|
255
|
-
when 'COMMA' then ','
|
256
|
-
else "\t"
|
257
|
-
end
|
258
44
|
end
|
259
45
|
|
260
46
|
def start
|
@@ -286,10 +72,8 @@ class Fluent::HoopOutput < Fluent::TimeSlicedOutput
|
|
286
72
|
record.to_json
|
287
73
|
end
|
288
74
|
|
289
|
-
def format(tag, time, record)
|
290
|
-
|
291
|
-
time_str + @f_separator + tag + @f_separator + record_to_string(record) + @line_end
|
292
|
-
end
|
75
|
+
# def format(tag, time, record)
|
76
|
+
# end
|
293
77
|
|
294
78
|
def path_format(chunk_key)
|
295
79
|
Time.strptime(chunk_key, @time_slice_format).strftime(@path)
|
@@ -80,7 +80,7 @@ class HoopOutputTest < Test::Unit::TestCase
|
|
80
80
|
assert_equal true, d.instance.output_include_tag
|
81
81
|
assert_equal 'json', d.instance.output_data_type
|
82
82
|
assert_equal true, d.instance.add_newline
|
83
|
-
assert_equal "
|
83
|
+
assert_equal "TAB", d.instance.field_separator
|
84
84
|
|
85
85
|
assert_nil d.instance.remove_prefix
|
86
86
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-hoop
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.4
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-
|
12
|
+
date: 2012-07-13 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rdoc
|
@@ -28,14 +28,14 @@ dependencies:
|
|
28
28
|
- !ruby/object:Gem::Version
|
29
29
|
version: '0'
|
30
30
|
- !ruby/object:Gem::Dependency
|
31
|
-
name:
|
31
|
+
name: fluentd
|
32
32
|
requirement: !ruby/object:Gem::Requirement
|
33
33
|
none: false
|
34
34
|
requirements:
|
35
35
|
- - ! '>='
|
36
36
|
- !ruby/object:Gem::Version
|
37
37
|
version: '0'
|
38
|
-
type: :
|
38
|
+
type: :runtime
|
39
39
|
prerelease: false
|
40
40
|
version_requirements: !ruby/object:Gem::Requirement
|
41
41
|
none: false
|
@@ -44,39 +44,39 @@ dependencies:
|
|
44
44
|
- !ruby/object:Gem::Version
|
45
45
|
version: '0'
|
46
46
|
- !ruby/object:Gem::Dependency
|
47
|
-
name:
|
47
|
+
name: fluent-mixin-plaintextformatter
|
48
48
|
requirement: !ruby/object:Gem::Requirement
|
49
49
|
none: false
|
50
50
|
requirements:
|
51
|
-
- -
|
51
|
+
- - ! '>='
|
52
52
|
- !ruby/object:Gem::Version
|
53
|
-
version:
|
54
|
-
type: :
|
53
|
+
version: '0'
|
54
|
+
type: :runtime
|
55
55
|
prerelease: false
|
56
56
|
version_requirements: !ruby/object:Gem::Requirement
|
57
57
|
none: false
|
58
58
|
requirements:
|
59
|
-
- -
|
59
|
+
- - ! '>='
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version:
|
61
|
+
version: '0'
|
62
62
|
- !ruby/object:Gem::Dependency
|
63
|
-
name:
|
63
|
+
name: shoulda
|
64
64
|
requirement: !ruby/object:Gem::Requirement
|
65
65
|
none: false
|
66
66
|
requirements:
|
67
|
-
- -
|
67
|
+
- - ! '>='
|
68
68
|
- !ruby/object:Gem::Version
|
69
|
-
version:
|
69
|
+
version: '0'
|
70
70
|
type: :development
|
71
71
|
prerelease: false
|
72
72
|
version_requirements: !ruby/object:Gem::Requirement
|
73
73
|
none: false
|
74
74
|
requirements:
|
75
|
-
- -
|
75
|
+
- - ! '>='
|
76
76
|
- !ruby/object:Gem::Version
|
77
|
-
version:
|
77
|
+
version: '0'
|
78
78
|
- !ruby/object:Gem::Dependency
|
79
|
-
name:
|
79
|
+
name: bundler
|
80
80
|
requirement: !ruby/object:Gem::Requirement
|
81
81
|
none: false
|
82
82
|
requirements:
|
@@ -91,6 +91,22 @@ dependencies:
|
|
91
91
|
- - ! '>='
|
92
92
|
- !ruby/object:Gem::Version
|
93
93
|
version: '0'
|
94
|
+
- !ruby/object:Gem::Dependency
|
95
|
+
name: jeweler
|
96
|
+
requirement: !ruby/object:Gem::Requirement
|
97
|
+
none: false
|
98
|
+
requirements:
|
99
|
+
- - ~>
|
100
|
+
- !ruby/object:Gem::Version
|
101
|
+
version: 1.6.4
|
102
|
+
type: :development
|
103
|
+
prerelease: false
|
104
|
+
version_requirements: !ruby/object:Gem::Requirement
|
105
|
+
none: false
|
106
|
+
requirements:
|
107
|
+
- - ~>
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: 1.6.4
|
94
110
|
- !ruby/object:Gem::Dependency
|
95
111
|
name: fluentd
|
96
112
|
requirement: !ruby/object:Gem::Requirement
|
@@ -176,7 +192,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
176
192
|
version: '0'
|
177
193
|
segments:
|
178
194
|
- 0
|
179
|
-
hash:
|
195
|
+
hash: 3002117588633528194
|
180
196
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
181
197
|
none: false
|
182
198
|
requirements:
|