fluent-plugin-webhdfs 0.2.1 → 0.2.2

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 56b8e7efe377002f3f092f79152826975098b9f8
4
- data.tar.gz: 77ef93b880901d5136980b63125f0a36b620fd9c
3
+ metadata.gz: 08f6f306b0cc2e49582bc066e03d3460c69fd245
4
+ data.tar.gz: deb9774da6e4d8da83a844603254e4e3a069d78d
5
5
  SHA512:
6
- metadata.gz: d8f2660f0c17f7956fba570deea3ed4a3301e8ea54f9b3f0e67232c5f24ba508949cfd259398eb22d4a20be1d4a83e1637ddf68c730919e619872b2382d5eac2
7
- data.tar.gz: 3d834d669fab4f4986e46a9e439afe983b53264dd7555db1c48c3565ac4a3cbf82b3d5ebb5c90d0259edde0463b0cfadffde2b93f8a9020e877fe9160e3cc52b
6
+ metadata.gz: 6dc0afebf5e982bcdfa12ae746f17925e4ddc4889a8d0fa4087d3d4d866889f9b3879b75b02a03ebcd833ce39d3815e614da2fc19305e14163101e7d3236c9e6
7
+ data.tar.gz: ddd01525b9bc7536eed1a36055298cddee927b7a8be8b1feb5ac913e07766faad050d2ff8c2d2203a12ad4a1ec79e587893f7b03faf989b4cbf7c9a07ab7576c
@@ -4,7 +4,6 @@ rvm:
4
4
  - 1.9.2
5
5
  - 1.9.3
6
6
  - 2.0.0
7
- - rbx-19mode
8
7
 
9
8
  branches:
10
9
  only:
data/README.md CHANGED
@@ -1,6 +1,6 @@
1
1
  # fluent-plugin-webhdfs
2
2
 
3
- Fluentd output plugin to write data into Hadoop HDFS over WebHDFS/HttpFs.
3
+ [Fluentd](http://fluentd.org/) output plugin to write data into Hadoop HDFS over WebHDFS/HttpFs.
4
4
 
5
5
  WebHDFSOutput slices data by time (specified unit), and store these data as hdfs file of plain text. You can specify to:
6
6
 
@@ -26,6 +26,17 @@ To store data by time,tag,json (same with 'type file') over WebHDFS:
26
26
  path /path/on/hdfs/access.log.%Y%m%d_%H.log
27
27
  </match>
28
28
 
29
+ If you want JSON object only (without time or tag or both on header of lines), specify it by `output_include_time` or `output_include_tag` (default true):
30
+
31
+ <match access.**>
32
+ type webhdfs
33
+ host namenode.your.cluster.local
34
+ port 50070
35
+ path /path/on/hdfs/access.log.%Y%m%d_%H.log
36
+ output_include_time false
37
+ output_include_tag false
38
+ </match>
39
+
29
40
  To specify namenode, `namenode` is also available:
30
41
 
31
42
  <match access.**>
@@ -2,12 +2,13 @@
2
2
 
3
3
  Gem::Specification.new do |gem|
4
4
  gem.name = "fluent-plugin-webhdfs"
5
- gem.version = "0.2.1"
5
+ gem.version = "0.2.2"
6
6
  gem.authors = ["TAGOMORI Satoshi"]
7
7
  gem.email = ["tagomoris@gmail.com"]
8
8
  gem.summary = %q{Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting}
9
9
  gem.description = %q{For WebHDFS and HttpFs of Hadoop HDFS}
10
10
  gem.homepage = "https://github.com/fluent/fluent-plugin-webhdfs"
11
+ gem.license = "APLv2"
11
12
 
12
13
  gem.files = `git ls-files`.split($\)
13
14
  gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
@@ -50,6 +50,11 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
50
50
  require 'webhdfs'
51
51
  end
52
52
 
53
+ # Define `log` method for v0.10.42 or earlier
54
+ unless method_defined?(:log)
55
+ define_method("log") { $log }
56
+ end
57
+
53
58
  def configure(conf)
54
59
  if conf['path']
55
60
  if conf['path'].index('%S')
@@ -125,7 +130,7 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
125
130
  begin
126
131
  client.list('/')
127
132
  rescue => e
128
- $log.warn "webhdfs check request failed. (namenode: #{client.host}:#{client.port}, error: #{e.message})"
133
+ log.warn "webhdfs check request failed. (namenode: #{client.host}:#{client.port}, error: #{e.message})"
129
134
  available = false
130
135
  end
131
136
  available
@@ -138,11 +143,11 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
138
143
  super
139
144
 
140
145
  if namenode_available(@client)
141
- $log.info "webhdfs connection confirmed: #{@namenode_host}:#{@namenode_port}"
146
+ log.info "webhdfs connection confirmed: #{@namenode_host}:#{@namenode_port}"
142
147
  return
143
148
  end
144
149
  if @client_standby && namenode_available(@client_standby)
145
- $log.info "webhdfs connection confirmed: #{@standby_namenode_host}:#{@standby_namenode_port}"
150
+ log.info "webhdfs connection confirmed: #{@standby_namenode_host}:#{@standby_namenode_port}"
146
151
  return
147
152
  end
148
153
 
@@ -166,7 +171,7 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
166
171
  def namenode_failover
167
172
  if @standby_namenode
168
173
  @client, @client_standby = @client_standby, @client
169
- $log.warn "Namenode failovered, now using #{@client.host}:#{@client.port}."
174
+ log.warn "Namenode failovered, now using #{@client.host}:#{@client.port}."
170
175
  end
171
176
  end
172
177
 
@@ -199,18 +204,18 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
199
204
  begin
200
205
  send_data(hdfs_path, chunk.read)
201
206
  rescue => e
202
- $log.warn "failed to communicate hdfs cluster, path: #{hdfs_path}"
207
+ log.warn "failed to communicate hdfs cluster, path: #{hdfs_path}"
203
208
 
204
209
  raise e if !@client_standby || failovered
205
210
 
206
211
  if is_standby_exception(e) && namenode_available(@client_standby)
207
- $log.warn "Seems the connected host status is not active (maybe due to failovers). Gonna try another namenode immediately."
212
+ log.warn "Seems the connected host status is not active (maybe due to failovers). Gonna try another namenode immediately."
208
213
  namenode_failover
209
214
  failovered = true
210
215
  retry
211
216
  end
212
217
  if ((@error_history.size + 1) >= @failures_before_use_standby) && namenode_available(@client_standby)
213
- $log.warn "Too many failures. Try to use the standby namenode instead."
218
+ log.warn "Too many failures. Try to use the standby namenode instead."
214
219
  namenode_failover
215
220
  failovered = true
216
221
  retry
metadata CHANGED
@@ -1,83 +1,83 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-webhdfs
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.1
4
+ version: 0.2.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - TAGOMORI Satoshi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2013-07-23 00:00:00.000000000 Z
11
+ date: 2014-03-07 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - '>='
17
+ - - ">="
18
18
  - !ruby/object:Gem::Version
19
19
  version: '0'
20
20
  type: :development
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - '>='
24
+ - - ">="
25
25
  - !ruby/object:Gem::Version
26
26
  version: '0'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: fluentd
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
- - - '>='
31
+ - - ">="
32
32
  - !ruby/object:Gem::Version
33
33
  version: '0'
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - '>='
38
+ - - ">="
39
39
  - !ruby/object:Gem::Version
40
40
  version: '0'
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: fluent-mixin-plaintextformatter
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
- - - '>='
45
+ - - ">="
46
46
  - !ruby/object:Gem::Version
47
47
  version: 0.2.1
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
- - - '>='
52
+ - - ">="
53
53
  - !ruby/object:Gem::Version
54
54
  version: 0.2.1
55
55
  - !ruby/object:Gem::Dependency
56
56
  name: fluent-mixin-config-placeholders
57
57
  requirement: !ruby/object:Gem::Requirement
58
58
  requirements:
59
- - - ~>
59
+ - - "~>"
60
60
  - !ruby/object:Gem::Version
61
61
  version: 0.2.0
62
62
  type: :runtime
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
- - - ~>
66
+ - - "~>"
67
67
  - !ruby/object:Gem::Version
68
68
  version: 0.2.0
69
69
  - !ruby/object:Gem::Dependency
70
70
  name: webhdfs
71
71
  requirement: !ruby/object:Gem::Requirement
72
72
  requirements:
73
- - - '>='
73
+ - - ">="
74
74
  - !ruby/object:Gem::Version
75
75
  version: 0.5.3
76
76
  type: :runtime
77
77
  prerelease: false
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
80
- - - '>='
80
+ - - ">="
81
81
  - !ruby/object:Gem::Version
82
82
  version: 0.5.3
83
83
  description: For WebHDFS and HttpFs of Hadoop HDFS
@@ -87,8 +87,8 @@ executables: []
87
87
  extensions: []
88
88
  extra_rdoc_files: []
89
89
  files:
90
- - .gitignore
91
- - .travis.yml
90
+ - ".gitignore"
91
+ - ".travis.yml"
92
92
  - Gemfile
93
93
  - LICENSE.txt
94
94
  - README.md
@@ -98,7 +98,8 @@ files:
98
98
  - test/helper.rb
99
99
  - test/plugin/test_out_webhdfs.rb
100
100
  homepage: https://github.com/fluent/fluent-plugin-webhdfs
101
- licenses: []
101
+ licenses:
102
+ - APLv2
102
103
  metadata: {}
103
104
  post_install_message:
104
105
  rdoc_options: []
@@ -106,21 +107,20 @@ require_paths:
106
107
  - lib
107
108
  required_ruby_version: !ruby/object:Gem::Requirement
108
109
  requirements:
109
- - - '>='
110
+ - - ">="
110
111
  - !ruby/object:Gem::Version
111
112
  version: '0'
112
113
  required_rubygems_version: !ruby/object:Gem::Requirement
113
114
  requirements:
114
- - - '>='
115
+ - - ">="
115
116
  - !ruby/object:Gem::Version
116
117
  version: '0'
117
118
  requirements: []
118
119
  rubyforge_project:
119
- rubygems_version: 2.0.2
120
+ rubygems_version: 2.2.2
120
121
  signing_key:
121
122
  specification_version: 4
122
123
  summary: Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting
123
124
  test_files:
124
125
  - test/helper.rb
125
126
  - test/plugin/test_out_webhdfs.rb
126
- has_rdoc: