fluent-plugin-hoop 0.1.3 → 0.1.4

Sign up to get free protection for your applications and to get access to all the features.
data/Gemfile CHANGED
@@ -7,10 +7,10 @@ source "http://rubygems.org"
7
7
  # Include everything needed to run rake, tests, features, etc.
8
8
  group :development do
9
9
  gem "shoulda", ">= 0"
10
- gem "bundler", "~> 1.0.0"
10
+ gem "bundler"
11
11
  gem "jeweler", "~> 1.6.4"
12
- gem "rcov", ">= 0"
13
12
  end
14
13
 
15
14
  gem "rdoc"
16
-
15
+ gem "fluentd"
16
+ gem "fluent-mixin-plaintextformatter"
data/Rakefile CHANGED
@@ -43,14 +43,6 @@ Rake::TestTask.new(:test) do |test|
43
43
  test.verbose = true
44
44
  end
45
45
 
46
- require 'rcov/rcovtask'
47
- Rcov::RcovTask.new do |test|
48
- test.libs << 'test'
49
- test.pattern = 'test/**/test_*.rb'
50
- test.verbose = true
51
- test.rcov_opts << '--exclude "gems/*"'
52
- end
53
-
54
46
  task :default => :test
55
47
 
56
48
  require 'rdoc/task'
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.1.3
1
+ 0.1.4
@@ -5,11 +5,11 @@
5
5
 
6
6
  Gem::Specification.new do |s|
7
7
  s.name = "fluent-plugin-hoop"
8
- s.version = "0.1.3"
8
+ s.version = "0.1.4"
9
9
 
10
10
  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
11
11
  s.authors = ["TAGOMORI Satoshi"]
12
- s.date = "2012-06-13"
12
+ s.date = "2012-07-13"
13
13
  s.description = "This plugin doesn't support Apache Hadoop's HttpFs. See fluent-plugin-webhdfs."
14
14
  s.email = "tagomoris@gmail.com"
15
15
  s.extra_rdoc_files = [
@@ -44,29 +44,32 @@ Gem::Specification.new do |s|
44
44
 
45
45
  if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
46
46
  s.add_runtime_dependency(%q<rdoc>, [">= 0"])
47
+ s.add_runtime_dependency(%q<fluentd>, [">= 0"])
48
+ s.add_runtime_dependency(%q<fluent-mixin-plaintextformatter>, [">= 0"])
47
49
  s.add_development_dependency(%q<shoulda>, [">= 0"])
48
- s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
50
+ s.add_development_dependency(%q<bundler>, [">= 0"])
49
51
  s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
50
- s.add_development_dependency(%q<rcov>, [">= 0"])
51
52
  s.add_runtime_dependency(%q<fluentd>, ["~> 0.10.8"])
52
53
  s.add_development_dependency(%q<rake>, [">= 0.9.2"])
53
54
  s.add_development_dependency(%q<simplecov>, [">= 0.5.4"])
54
55
  else
55
56
  s.add_dependency(%q<rdoc>, [">= 0"])
57
+ s.add_dependency(%q<fluentd>, [">= 0"])
58
+ s.add_dependency(%q<fluent-mixin-plaintextformatter>, [">= 0"])
56
59
  s.add_dependency(%q<shoulda>, [">= 0"])
57
- s.add_dependency(%q<bundler>, ["~> 1.0.0"])
60
+ s.add_dependency(%q<bundler>, [">= 0"])
58
61
  s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
59
- s.add_dependency(%q<rcov>, [">= 0"])
60
62
  s.add_dependency(%q<fluentd>, ["~> 0.10.8"])
61
63
  s.add_dependency(%q<rake>, [">= 0.9.2"])
62
64
  s.add_dependency(%q<simplecov>, [">= 0.5.4"])
63
65
  end
64
66
  else
65
67
  s.add_dependency(%q<rdoc>, [">= 0"])
68
+ s.add_dependency(%q<fluentd>, [">= 0"])
69
+ s.add_dependency(%q<fluent-mixin-plaintextformatter>, [">= 0"])
66
70
  s.add_dependency(%q<shoulda>, [">= 0"])
67
- s.add_dependency(%q<bundler>, ["~> 1.0.0"])
71
+ s.add_dependency(%q<bundler>, [">= 0"])
68
72
  s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
69
- s.add_dependency(%q<rcov>, [">= 0"])
70
73
  s.add_dependency(%q<fluentd>, ["~> 0.10.8"])
71
74
  s.add_dependency(%q<rake>, [">= 0.9.2"])
72
75
  s.add_dependency(%q<simplecov>, [">= 0.5.4"])
@@ -1,224 +1,16 @@
1
- module FluentExt; end
2
- module FluentExt::PlainTextFormatterMixin
3
- # config_param :output_data_type, :string, :default => 'json' # or 'attr:field' or 'attr:field1,field2,field3(...)'
4
-
5
- attr_accessor :output_include_time, :output_include_tag, :output_data_type
6
- attr_accessor :add_newline, :field_separator
7
- attr_accessor :remove_prefix, :default_tag
8
-
9
- def configure(conf)
10
- super
11
-
12
- @output_include_time = Fluent::Config.bool_value(conf['output_include_time'])
13
- @output_include_time = true if @output_include_time.nil?
14
-
15
- @output_include_tag = Fluent::Config.bool_value(conf['output_include_tag'])
16
- @output_include_tag = true if @output_include_tag.nil?
17
-
18
- @output_data_type = conf['output_data_type']
19
- @output_data_type = 'json' if @output_data_type.nil?
20
-
21
- @field_separator = case conf['field_separator']
22
- when 'SPACE' then ' '
23
- when 'COMMA' then ','
24
- else "\t"
25
- end
26
- @add_newline = Fluent::Config.bool_value(conf['add_newline'])
27
- if @add_newline.nil?
28
- @add_newline = true
29
- end
30
-
31
- @remove_prefix = conf['remove_prefix']
32
- if @remove_prefix
33
- @removed_prefix_string = @remove_prefix + '.'
34
- @removed_length = @removed_prefix_string.length
35
- end
36
- if @output_include_tag and @remove_prefix and @remove_prefix.length > 0
37
- @default_tag = conf['default_tag']
38
- if @default_tag.nil? or @default_tag.length < 1
39
- raise Fluent::ConfigError, "Missing 'default_tag' with output_include_tag and remove_prefix."
40
- end
41
- end
42
-
43
- # default timezone: utc
44
- if conf['localtime'].nil? and conf['utc'].nil?
45
- @utc = true
46
- @localtime = false
47
- elsif not @localtime and not @utc
48
- @utc = true
49
- @localtime = false
50
- end
51
- # mix-in default time formatter (or you can overwrite @timef on your own configure)
52
- @timef = @output_include_time ? Fluent::TimeFormatter.new(@time_format, @localtime) : nil
53
-
54
- @custom_attributes = []
55
- if @output_data_type == 'json'
56
- self.instance_eval {
57
- def stringify_record(record)
58
- record.to_json
59
- end
60
- }
61
- elsif @output_data_type =~ /^attr:(.*)$/
62
- @custom_attributes = $1.split(',')
63
- if @custom_attributes.size > 1
64
- self.instance_eval {
65
- def stringify_record(record)
66
- @custom_attributes.map{|attr| (record[attr] || 'NULL').to_s}.join(@field_separator)
67
- end
68
- }
69
- elsif @custom_attributes.size == 1
70
- self.instance_eval {
71
- def stringify_record(record)
72
- (record[@custom_attributes[0]] || 'NULL').to_s
73
- end
74
- }
75
- else
76
- raise Fluent::ConfigError, "Invalid attributes specification: '#{@output_data_type}', needs one or more attributes."
77
- end
78
- else
79
- raise Fluent::ConfigError, "Invalid output_data_type: '#{@output_data_type}'. specify 'json' or 'attr:ATTRIBUTE_NAME' or 'attr:ATTR1,ATTR2,...'"
80
- end
81
-
82
- if @output_include_time and @output_include_tag
83
- if @add_newline and @remove_prefix
84
- self.instance_eval {
85
- def format(tag,time,record)
86
- if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
87
- tag == @remove_prefix
88
- tag = tag[@removed_length..-1] || @default_tag
89
- end
90
- @timef.format(time) + @field_separator + tag + @field_separator + stringify_record(record) + "\n"
91
- end
92
- }
93
- elsif @add_newline
94
- self.instance_eval {
95
- def format(tag,time,record)
96
- @timef.format(time) + @field_separator + tag + @field_separator + stringify_record(record) + "\n"
97
- end
98
- }
99
- elsif @remove_prefix
100
- self.instance_eval {
101
- def format(tag,time,record)
102
- if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
103
- tag == @remove_prefix
104
- tag = tag[@removed_length..-1] || @default_tag
105
- end
106
- @timef.format(time) + @field_separator + tag + @field_separator + stringify_record(record)
107
- end
108
- }
109
- else
110
- self.instance_eval {
111
- def format(tag,time,record)
112
- @timef.format(time) + @field_separator + tag + @field_separator + stringify_record(record)
113
- end
114
- }
115
- end
116
- elsif @output_include_time
117
- if @add_newline
118
- self.instance_eval {
119
- def format(tag,time,record);
120
- @timef.format(time) + @field_separator + stringify_record(record) + "\n"
121
- end
122
- }
123
- else
124
- self.instance_eval {
125
- def format(tag,time,record);
126
- @timef.format(time) + @field_separator + stringify_record(record)
127
- end
128
- }
129
- end
130
- elsif @output_include_tag
131
- if @add_newline and @remove_prefix
132
- self.instance_eval {
133
- def format(tag,time,record)
134
- if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
135
- tag == @remove_prefix
136
- tag = tag[@removed_length..-1] || @default_tag
137
- end
138
- tag + @field_separator + stringify_record(record) + "\n"
139
- end
140
- }
141
- elsif @add_newline
142
- self.instance_eval {
143
- def format(tag,time,record)
144
- tag + @field_separator + stringify_record(record) + "\n"
145
- end
146
- }
147
- elsif @remove_prefix
148
- self.instance_eval {
149
- def format(tag,time,record)
150
- if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
151
- tag == @remove_prefix
152
- tag = tag[@removed_length..-1] || @default_tag
153
- end
154
- tag + @field_separator + stringify_record(record)
155
- end
156
- }
157
- else
158
- self.instance_eval {
159
- def format(tag,time,record)
160
- tag + @field_separator + stringify_record(record)
161
- end
162
- }
163
- end
164
- else # without time, tag
165
- if @add_newline
166
- self.instance_eval {
167
- def format(tag,time,record);
168
- stringify_record(record) + "\n"
169
- end
170
- }
171
- else
172
- self.instance_eval {
173
- def format(tag,time,record);
174
- stringify_record(record)
175
- end
176
- }
177
- end
178
- end
179
- end
180
-
181
- def stringify_record(record)
182
- record.to_json
183
- end
184
-
185
- def format(tag, time, record)
186
- if tag == @remove_prefix or (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length)
187
- tag = tag[@removed_length..-1] || @default_tag
188
- end
189
- time_str = if @output_include_time
190
- @timef.format(time) + @field_separator
191
- else
192
- ''
193
- end
194
- tag_str = if @output_include_tag
195
- tag + @field_separator
196
- else
197
- ''
198
- end
199
- time_str + tag_str + stringify_record(record) + "\n"
200
- end
201
-
202
- end
1
+ require 'fluent/mixin/plaintextformatter'
203
2
 
204
3
  class Fluent::HoopOutput < Fluent::TimeSlicedOutput
205
4
  Fluent::Plugin.register_output('hoop', self)
206
5
 
207
6
  config_set_default :buffer_type, 'memory'
208
7
  config_set_default :time_slice_format, '%Y%m%d' # %Y%m%d%H
209
- # config_param :tag_format, :string, :default => 'all' # or 'last'(last.part.of.tag => tag) or 'none'
210
8
 
211
9
  config_param :hoop_server, :string # host:port
212
10
  config_param :path, :string # /path/pattern/to/hdfs/file can use %Y %m %d %H %M %S and %T(tag, not-supported-yet)
213
11
  config_param :username, :string # hoop pseudo username
214
12
 
215
- include FluentExt::PlainTextFormatterMixin
216
- config_set_default :output_include_time, true
217
- config_set_default :output_include_tag, true
218
- config_set_default :output_data_type, 'json'
219
- config_set_default :field_separator, "\t"
220
- config_set_default :add_newline, true
221
- config_set_default :remove_prefix, nil
13
+ include Fluent::Mixin::PlainTextFormatter
222
14
 
223
15
  def initialize
224
16
  super
@@ -249,12 +41,6 @@ class Fluent::HoopOutput < Fluent::TimeSlicedOutput
249
41
  end
250
42
  @conn = nil
251
43
  @header = {'Content-Type' => 'application/octet-stream'}
252
-
253
- @f_separator = case @field_separator
254
- when 'SPACE' then ' '
255
- when 'COMMA' then ','
256
- else "\t"
257
- end
258
44
  end
259
45
 
260
46
  def start
@@ -286,10 +72,8 @@ class Fluent::HoopOutput < Fluent::TimeSlicedOutput
286
72
  record.to_json
287
73
  end
288
74
 
289
- def format(tag, time, record)
290
- time_str = @timef.format(time)
291
- time_str + @f_separator + tag + @f_separator + record_to_string(record) + @line_end
292
- end
75
+ # def format(tag, time, record)
76
+ # end
293
77
 
294
78
  def path_format(chunk_key)
295
79
  Time.strptime(chunk_key, @time_slice_format).strftime(@path)
@@ -80,7 +80,7 @@ class HoopOutputTest < Test::Unit::TestCase
80
80
  assert_equal true, d.instance.output_include_tag
81
81
  assert_equal 'json', d.instance.output_data_type
82
82
  assert_equal true, d.instance.add_newline
83
- assert_equal "\t", d.instance.field_separator
83
+ assert_equal "TAB", d.instance.field_separator
84
84
 
85
85
  assert_nil d.instance.remove_prefix
86
86
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-hoop
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.3
4
+ version: 0.1.4
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2012-06-13 00:00:00.000000000 Z
12
+ date: 2012-07-13 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: rdoc
@@ -28,14 +28,14 @@ dependencies:
28
28
  - !ruby/object:Gem::Version
29
29
  version: '0'
30
30
  - !ruby/object:Gem::Dependency
31
- name: shoulda
31
+ name: fluentd
32
32
  requirement: !ruby/object:Gem::Requirement
33
33
  none: false
34
34
  requirements:
35
35
  - - ! '>='
36
36
  - !ruby/object:Gem::Version
37
37
  version: '0'
38
- type: :development
38
+ type: :runtime
39
39
  prerelease: false
40
40
  version_requirements: !ruby/object:Gem::Requirement
41
41
  none: false
@@ -44,39 +44,39 @@ dependencies:
44
44
  - !ruby/object:Gem::Version
45
45
  version: '0'
46
46
  - !ruby/object:Gem::Dependency
47
- name: bundler
47
+ name: fluent-mixin-plaintextformatter
48
48
  requirement: !ruby/object:Gem::Requirement
49
49
  none: false
50
50
  requirements:
51
- - - ~>
51
+ - - ! '>='
52
52
  - !ruby/object:Gem::Version
53
- version: 1.0.0
54
- type: :development
53
+ version: '0'
54
+ type: :runtime
55
55
  prerelease: false
56
56
  version_requirements: !ruby/object:Gem::Requirement
57
57
  none: false
58
58
  requirements:
59
- - - ~>
59
+ - - ! '>='
60
60
  - !ruby/object:Gem::Version
61
- version: 1.0.0
61
+ version: '0'
62
62
  - !ruby/object:Gem::Dependency
63
- name: jeweler
63
+ name: shoulda
64
64
  requirement: !ruby/object:Gem::Requirement
65
65
  none: false
66
66
  requirements:
67
- - - ~>
67
+ - - ! '>='
68
68
  - !ruby/object:Gem::Version
69
- version: 1.6.4
69
+ version: '0'
70
70
  type: :development
71
71
  prerelease: false
72
72
  version_requirements: !ruby/object:Gem::Requirement
73
73
  none: false
74
74
  requirements:
75
- - - ~>
75
+ - - ! '>='
76
76
  - !ruby/object:Gem::Version
77
- version: 1.6.4
77
+ version: '0'
78
78
  - !ruby/object:Gem::Dependency
79
- name: rcov
79
+ name: bundler
80
80
  requirement: !ruby/object:Gem::Requirement
81
81
  none: false
82
82
  requirements:
@@ -91,6 +91,22 @@ dependencies:
91
91
  - - ! '>='
92
92
  - !ruby/object:Gem::Version
93
93
  version: '0'
94
+ - !ruby/object:Gem::Dependency
95
+ name: jeweler
96
+ requirement: !ruby/object:Gem::Requirement
97
+ none: false
98
+ requirements:
99
+ - - ~>
100
+ - !ruby/object:Gem::Version
101
+ version: 1.6.4
102
+ type: :development
103
+ prerelease: false
104
+ version_requirements: !ruby/object:Gem::Requirement
105
+ none: false
106
+ requirements:
107
+ - - ~>
108
+ - !ruby/object:Gem::Version
109
+ version: 1.6.4
94
110
  - !ruby/object:Gem::Dependency
95
111
  name: fluentd
96
112
  requirement: !ruby/object:Gem::Requirement
@@ -176,7 +192,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
176
192
  version: '0'
177
193
  segments:
178
194
  - 0
179
- hash: -355817273147021109
195
+ hash: 3002117588633528194
180
196
  required_rubygems_version: !ruby/object:Gem::Requirement
181
197
  none: false
182
198
  requirements: