fluent-plugin-scribe 0.10.12 → 0.10.13

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 4742edb4137955d08718e1cc275f2b163a0f7d98
4
- data.tar.gz: 454ccd6f4ebb77898c159c25bb17e17c0f94526c
3
+ metadata.gz: e82d716df29ee314a3d51ee911e32f9b0de07a0f
4
+ data.tar.gz: 35be26bb3acbe72db1b38bad94b2801875d90e22
5
5
  SHA512:
6
- metadata.gz: 5d607d8c5e8f579efe419e4575340caa1fca4b3103c48895e1208394eae5a30c609fe136875f8ad768b79484add797d12493e071747c6eb9e0d35f86598ac215
7
- data.tar.gz: 5419ee2a3a620720172a909baa0087cf8e15da93ffd8a253a1aee2f891b0d40405272ec6e8e6600391877ac46eb7ca366fc3d670500eabf800b2bf13029f07ee
6
+ metadata.gz: f98b739988b9bdb0e69d34c56c9139e80cf7c9c1834c3d59f6ee22a449552230e471ee711fa3284487f6fb2735d296c6d42b182168653595526dc5a34fce1031
7
+ data.tar.gz: 9dc6b1104e4b3aace045c925961b3224e4cc215de001303191f3cd560a94828b7f6929bccc66a1168de4958383b99d140c6ec923c30c861280af3adae46bdedd
data/ChangeLog CHANGED
@@ -1,3 +1,18 @@
1
+ Release 0.10.13 - 2013/10/29
2
+ * add validation for msg_format
3
+ * use Yajl instead of JSON for parsing record
4
+ * add ignore_invalid_record option to ignore invalid record and return OK to scribed
5
+
6
+ Release 0.10.12 - 2013/09/04
7
+ * support 'log_level' option
8
+ * fix to do shutdown plugin instance not to duplicate listen
9
+
10
+ Release 0.10.11 - 2013/09/04
11
+ * add format_to_json bool option with default value false
12
+
13
+ Release 0.10.10 - 2013/02/06
14
+ * change 'written entries' log level from 'info' to 'debug'
15
+
1
16
  Release 0.10.9 - 2011/05/30
2
17
  * overwrite default value of buffer_chunk_limit to 1MB
3
18
  * improve memory usage efficiency
@@ -1,8 +1,8 @@
1
- = Scribe input plugin for Fluent
1
+ = Scribe input plugin for Fluentd[http://fluentd.org/]
2
2
 
3
3
  == Overview
4
4
 
5
- This is a plugin for fluentd[https://github.com/fluentd] event collector. This plugin adds the Scribe[https://github.com/facebook/scribe] compatible interface to fluentd.
5
+ This is a plugin for fluentd[http://fluentd.org] data collector. This plugin adds the Scribe[https://github.com/facebook/scribe] compatible interface to fluentd.
6
6
 
7
7
  == What's Scribe?
8
8
 
@@ -2,7 +2,7 @@
2
2
 
3
3
  Gem::Specification.new do |gem|
4
4
  gem.name = "fluent-plugin-scribe"
5
- gem.version = "0.10.12"
5
+ gem.version = "0.10.13"
6
6
 
7
7
  gem.authors = ["Kazuki Ohta", "TAGOMORI Satoshi"]
8
8
  gem.email = ["kazuki.ohta@gmail.com", "tagomoris@gmail.com"]
@@ -17,6 +17,7 @@ Gem::Specification.new do |gem|
17
17
  gem.require_paths = ["lib"]
18
18
 
19
19
  gem.add_development_dependency "rake"
20
+ gem.add_development_dependency "test-unit", '~> 3.0.2'
20
21
  gem.add_runtime_dependency "fluentd"
21
22
  gem.add_runtime_dependency "thrift", "~> 0.8.0"
22
23
  end
@@ -19,6 +19,12 @@ module Fluent
19
19
  class ScribeInput < Input
20
20
  Plugin.register_input('scribe', self)
21
21
 
22
+ SUPPORTED_FORMAT = {
23
+ 'text' => :text,
24
+ 'json' => :json,
25
+ 'url_param' => :url_param,
26
+ }
27
+
22
28
  config_param :port, :integer, :default => 1463
23
29
  config_param :bind, :string, :default => '0.0.0.0'
24
30
  config_param :server_type, :string, :default => 'nonblocking'
@@ -26,7 +32,12 @@ module Fluent
26
32
  config_param :body_size_limit, :size, :default => 32*1024*1024 # TODO default
27
33
  config_param :add_prefix, :string, :default => nil
28
34
  config_param :remove_newline, :bool, :default => false
29
- config_param :msg_format, :string, :default => 'text'
35
+ config_param :ignore_invalid_record, :bool, :default => false
36
+ config_param :msg_format, :default => :text do |val|
37
+ f = SUPPORTED_FORMAT[val]
38
+ raise ConfigError, "unsupported msg_format: #{val}" unless f
39
+ f
40
+ end
30
41
 
31
42
  unless method_defined?(:log)
32
43
  define_method(:log) { $log }
@@ -34,6 +45,7 @@ module Fluent
34
45
 
35
46
  def initialize
36
47
  require 'cgi'
48
+ require 'yajl'
37
49
  require 'thrift'
38
50
  $:.unshift File.join(File.dirname(__FILE__), 'thrift')
39
51
  require 'fb303_types'
@@ -56,6 +68,7 @@ module Fluent
56
68
  handler.add_prefix = @add_prefix
57
69
  handler.remove_newline = @remove_newline
58
70
  handler.msg_format = @msg_format
71
+ handler.ignore_invalid_record = @ignore_invalid_record
59
72
  handler.logger = log
60
73
  processor = Scribe::Processor.new handler
61
74
 
@@ -66,10 +79,6 @@ module Fluent
66
79
  transport_factory = Thrift::BufferedTransportFactory.new
67
80
  end
68
81
 
69
- unless ['text', 'json', 'url_param'].include? @msg_format
70
- raise 'Unknown format: msg_format=#{@msg_format}'
71
- end
72
-
73
82
  # 2011/09/29 Kazuki Ohta <kazuki.ohta@gmail.com>
74
83
  # This section is a workaround to set strict_read and strict_write option.
75
84
  # Ruby-Thrift 0.7 set them both 'true' in default, but Scribe protocol set
@@ -105,8 +114,8 @@ module Fluent
105
114
 
106
115
  def run
107
116
  @server.serve
108
- rescue
109
- log.error "unexpected error", :error=>$!.to_s
117
+ rescue => e
118
+ log.error "unexpected error", :error => e.inspect
110
119
  log.error_backtrace
111
120
  end
112
121
 
@@ -114,12 +123,25 @@ module Fluent
114
123
  attr_accessor :add_prefix
115
124
  attr_accessor :remove_newline
116
125
  attr_accessor :msg_format
126
+ attr_accessor :ignore_invalid_record
117
127
  attr_accessor :logger # Use logger instead of log to avoid confusion with Log method
118
128
 
119
129
  def Log(msgs)
120
130
  begin
121
131
  msgs.each { |msg|
122
- record = create_record(msg)
132
+ begin
133
+ record = create_record(msg)
134
+ rescue => e
135
+ if @ignore_invalid_record
136
+ # This warning can be disabled by 'log_level error'
137
+ logger.warn "got invalid record: #{msg}"
138
+ next
139
+ else
140
+ # Keep existence behaviour
141
+ raise e
142
+ end
143
+ end
144
+
123
145
  if @add_prefix
124
146
  Engine.emit(@add_prefix + '.' + msg.category, Engine.now, record)
125
147
  else
@@ -128,7 +150,7 @@ module Fluent
128
150
  }
129
151
  return ResultCode::OK
130
152
  rescue => e
131
- logger.error "unexpected error", :error=>$!.to_s
153
+ logger.error "unexpected error", :error => e.inspect
132
154
  logger.error_backtrace
133
155
  return ResultCode::TRY_LATER
134
156
  end
@@ -137,17 +159,17 @@ module Fluent
137
159
  private
138
160
  def create_record(msg)
139
161
  case @msg_format
140
- when 'text'
162
+ when :text
141
163
  if @remove_newline
142
164
  return { 'message' => msg.message.force_encoding('UTF-8').chomp }
143
165
  else
144
166
  return { 'message' => msg.message.force_encoding('UTF-8') }
145
167
  end
146
- when 'json'
147
- js = JSON.parse(msg.message.force_encoding('UTF-8'))
168
+ when :json
169
+ js = Yajl.load(msg.message.force_encoding('UTF-8'))
148
170
  raise 'body must be a Hash, if json_body=true' unless js.is_a?(Hash)
149
171
  return js
150
- when 'url_param'
172
+ when :url_param
151
173
  s = msg.message.force_encoding('UTF-8')
152
174
  return Hash[ s.split('&').map { |kv|
153
175
  k,v = kv.split('=', 2);
@@ -134,7 +134,7 @@ class ScribeInputTest < Test::Unit::TestCase
134
134
  d = create_driver(CONFIG + %[
135
135
  msg_format json
136
136
  ])
137
- assert_equal 'json', d.instance.msg_format
137
+ assert_equal :json, d.instance.msg_format
138
138
 
139
139
  time = Time.parse("2011-01-02 13:14:15 UTC").to_i
140
140
  Fluent::Engine.now = time
@@ -158,11 +158,41 @@ class ScribeInputTest < Test::Unit::TestCase
158
158
  shutdown_driver(d)
159
159
  end
160
160
 
161
+ data do
162
+ Fluent::ScribeInput.new
163
+ {
164
+ 'true' => [ResultCode::OK, true],
165
+ 'false' => [ResultCode::TRY_LATER, false]
166
+ }
167
+ end
168
+ def test_msg_format_json_with_ignore_invalid_record(data)
169
+ result, opt = data
170
+ d = create_driver(CONFIG + %[
171
+ msg_format json
172
+ ignore_invalid_record #{opt}
173
+ ])
174
+ assert_equal :json, d.instance.msg_format
175
+ assert_equal opt, d.instance.ignore_invalid_record
176
+
177
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
178
+ Fluent::Engine.now = time
179
+
180
+ emits = [['tag1', time, '{"a":']]
181
+ d.run do
182
+ emits.each { |tag, time, message|
183
+ res = message_send(tag, message)
184
+ assert_equal result, res
185
+ }
186
+ end
187
+
188
+ shutdown_driver(d)
189
+ end
190
+
161
191
  def test_msg_format_url_param
162
192
  d = create_driver(CONFIG + %[
163
193
  msg_format url_param
164
194
  ])
165
- assert_equal 'url_param', d.instance.msg_format
195
+ assert_equal :url_param, d.instance.msg_format
166
196
 
167
197
  time = Time.parse("2011-01-02 13:14:15 UTC").to_i
168
198
  Fluent::Engine.now = time
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-scribe
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.10.12
4
+ version: 0.10.13
5
5
  platform: ruby
6
6
  authors:
7
7
  - Kazuki Ohta
@@ -9,48 +9,62 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2014-02-05 00:00:00.000000000 Z
12
+ date: 2014-10-28 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: rake
16
16
  requirement: !ruby/object:Gem::Requirement
17
17
  requirements:
18
- - - '>='
18
+ - - ">="
19
19
  - !ruby/object:Gem::Version
20
20
  version: '0'
21
21
  type: :development
22
22
  prerelease: false
23
23
  version_requirements: !ruby/object:Gem::Requirement
24
24
  requirements:
25
- - - '>='
25
+ - - ">="
26
26
  - !ruby/object:Gem::Version
27
27
  version: '0'
28
+ - !ruby/object:Gem::Dependency
29
+ name: test-unit
30
+ requirement: !ruby/object:Gem::Requirement
31
+ requirements:
32
+ - - "~>"
33
+ - !ruby/object:Gem::Version
34
+ version: 3.0.2
35
+ type: :development
36
+ prerelease: false
37
+ version_requirements: !ruby/object:Gem::Requirement
38
+ requirements:
39
+ - - "~>"
40
+ - !ruby/object:Gem::Version
41
+ version: 3.0.2
28
42
  - !ruby/object:Gem::Dependency
29
43
  name: fluentd
30
44
  requirement: !ruby/object:Gem::Requirement
31
45
  requirements:
32
- - - '>='
46
+ - - ">="
33
47
  - !ruby/object:Gem::Version
34
48
  version: '0'
35
49
  type: :runtime
36
50
  prerelease: false
37
51
  version_requirements: !ruby/object:Gem::Requirement
38
52
  requirements:
39
- - - '>='
53
+ - - ">="
40
54
  - !ruby/object:Gem::Version
41
55
  version: '0'
42
56
  - !ruby/object:Gem::Dependency
43
57
  name: thrift
44
58
  requirement: !ruby/object:Gem::Requirement
45
59
  requirements:
46
- - - ~>
60
+ - - "~>"
47
61
  - !ruby/object:Gem::Version
48
62
  version: 0.8.0
49
63
  type: :runtime
50
64
  prerelease: false
51
65
  version_requirements: !ruby/object:Gem::Requirement
52
66
  requirements:
53
- - - ~>
67
+ - - "~>"
54
68
  - !ruby/object:Gem::Version
55
69
  version: 0.8.0
56
70
  description: Fluentd input/output plugin to handle Facebook scribed thrift protocol
@@ -62,8 +76,8 @@ executables:
62
76
  extensions: []
63
77
  extra_rdoc_files: []
64
78
  files:
65
- - .gitignore
66
- - .travis.yml
79
+ - ".gitignore"
80
+ - ".travis.yml"
67
81
  - AUTHORS
68
82
  - ChangeLog
69
83
  - Gemfile
@@ -95,21 +109,20 @@ require_paths:
95
109
  - lib
96
110
  required_ruby_version: !ruby/object:Gem::Requirement
97
111
  requirements:
98
- - - '>='
112
+ - - ">="
99
113
  - !ruby/object:Gem::Version
100
114
  version: '0'
101
115
  required_rubygems_version: !ruby/object:Gem::Requirement
102
116
  requirements:
103
- - - '>='
117
+ - - ">="
104
118
  - !ruby/object:Gem::Version
105
119
  version: '0'
106
120
  requirements: []
107
121
  rubyforge_project:
108
- rubygems_version: 2.0.3
122
+ rubygems_version: 2.2.2
109
123
  signing_key:
110
124
  specification_version: 4
111
125
  summary: Scribe Input/Output plugin for Fluentd event collector
112
126
  test_files:
113
127
  - test/plugin/test_in_scribe.rb
114
128
  - test/plugin/test_out_scribe.rb
115
- has_rdoc: