fluent-plugin-mongo 0.7.12 → 0.7.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 306a5a645b491a0c8edb197064336662588aab89
4
- data.tar.gz: a80cc84c0e68319cd68a891678a08d4def516d22
3
+ metadata.gz: c00835bf3c3c44d2055e59c9d38d8654c438f83e
4
+ data.tar.gz: 21f1d514533a48beddf1391b68a2467e6131c73c
5
5
  SHA512:
6
- metadata.gz: 15dc38b80bca19d1c7d52914d46f4d7688bdcfe2e5355ed0fdbe2b835d802bba2ed8436b9ed0b4565cdb51cc4cbbf9295daedff78a9fb115fe650da37d58e5e4
7
- data.tar.gz: 23047da16921b3f30a3bc16e6f1881255010952f2542b921b92608aef6ecc7f73ff85c15ff2a27ab3bf05ad04b41f6cadec006eccb3436ee58c4e4c0057c0c4f
6
+ metadata.gz: 83fcb6008050f5eb175764dfaf1ab96836c41168947cd24ced230257fe60d17d04d7de60acfc9c18fe08eda59b103ad39b15b5b1ccbb76ac6f54d2e5c02f04ac
7
+ data.tar.gz: ca4144e0884ee794f8cdd1ef68f65a5e83748bd37b45a1c5a037f9a03cbad9f828292ab8200ab02e70dc3a41e66471e57a646c7855fd8cf7572f25dd97596eb9
data/ChangeLog CHANGED
@@ -1,3 +1,8 @@
1
+ Release 0.7.13 - 2016/06/03
2
+
3
+ * Add mongodb_smaller_bson_limit parameter to disable MongoDB v1.7 or earlier versions by default.
4
+
5
+
1
6
  Release 0.7.12 - 2016/02/09
2
7
 
3
8
  * Support saving last_id to mongod instead of local file
data/Rakefile CHANGED
@@ -4,7 +4,7 @@ Bundler::GemHelper.install_tasks
4
4
  require 'rake/testtask'
5
5
 
6
6
  Rake::TestTask.new(:test) do |test|
7
- test.libs << 'lib' << 'test'
7
+ test.libs << 'test'
8
8
  test.test_files = FileList['test/plugin/*.rb']
9
9
  test.verbose = true
10
10
  end
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.7.12
1
+ 0.7.13
@@ -1,3 +1,5 @@
1
+ require 'fluent/input'
2
+
1
3
  module Fluent
2
4
  class MongoTailInput < Input
3
5
  Plugin.register_input('mongo_tail', self)
@@ -1,3 +1,5 @@
1
+ require 'fluent/output'
2
+
1
3
  module Fluent
2
4
  class MongoOutput < BufferedOutput
3
5
  Plugin.register_output('mongo', self)
@@ -36,6 +38,9 @@ module Fluent
36
38
  config_param :ssl_verify, :bool, :default => false
37
39
  config_param :ssl_ca_cert, :string, :default => nil
38
40
 
41
+ # For older (1.7 or earlier) MongoDB versions
42
+ config_param :mongodb_smaller_bson_limit, :bool, :default => false
43
+
39
44
  attr_reader :collection_options, :connection_options
40
45
 
41
46
  unless method_defined?(:log)
@@ -52,7 +57,31 @@ module Fluent
52
57
  @collection_options = {:capped => false}
53
58
  end
54
59
 
60
+ # Following limits are heuristic. BSON is sometimes bigger than MessagePack and JSON.
61
+ LIMIT_BEFORE_v1_8 = 2 * 1024 * 1024 # 2MB = 4MB / 2
62
+ LIMIT_AFTER_v1_8 = 8 * 1024 * 1024 # 8MB = 16MB / 2
63
+
55
64
  def configure(conf)
65
+ if conf.has_key?('buffer_chunk_limit')
66
+ configured_chunk_limit_size = Config.size_value(conf['buffer_chunk_limit'])
67
+ estimated_limit_size = LIMIT_AFTER_v1_8
68
+ estimated_limit_size_conf = '8m'
69
+ if conf.has_key?('mongodb_smaller_bson_limit') && Config.bool_value(conf['mongodb_smaller_bson_limit'])
70
+ estimated_limit_size = LIMIT_BEFORE_v1_8
71
+ estimated_limit_size_conf = '2m'
72
+ end
73
+ if configured_chunk_limit_size > estimated_limit_size
74
+ log.warn ":buffer_chunk_limit(#{conf['buffer_chunk_limit']}) is large. Reset :buffer_chunk_limit with #{estimated_limit_size_conf}"
75
+ conf['buffer_chunk_limit'] = estimated_limit_size_conf
76
+ end
77
+ else
78
+ if conf.has_key?('mongodb_smaller_bson_limit') && Config.bool_value(conf['mongodb_smaller_bson_limit'])
79
+ conf['buffer_chunk_limit'] = '2m'
80
+ else
81
+ conf['buffer_chunk_limit'] = '8m'
82
+ end
83
+ end
84
+
56
85
  super
57
86
 
58
87
  if conf.has_key?('tag_mapped')
@@ -102,9 +131,6 @@ module Fluent
102
131
  # Non tag mapped mode, we can check collection configuration before server start.
103
132
  get_or_create_collection(@collection) unless @tag_mapped
104
133
 
105
- # From configure for avoding complex method dependency...
106
- @buffer.buffer_chunk_limit = available_buffer_chunk_limit
107
-
108
134
  super
109
135
  end
110
136
 
@@ -232,45 +258,6 @@ module Fluent
232
258
  authenticate(db)
233
259
  end
234
260
 
235
- # Following limits are heuristic. BSON is sometimes bigger than MessagePack and JSON.
236
- LIMIT_BEFORE_v1_8 = 2 * 1024 * 1024 # 2MB = 4MB / 2
237
- LIMIT_AFTER_v1_8 = 8 * 1024 * 1024 # 8MB = 16MB / 2
238
-
239
- def available_buffer_chunk_limit
240
- begin
241
- limit = mongod_version >= "1.8.0" ? LIMIT_AFTER_v1_8 : LIMIT_BEFORE_v1_8
242
- rescue Mongo::ConnectionFailure => e
243
- log.fatal "Failed to connect to 'mongod'. Please restart 'fluentd' after 'mongod' started: #{e}"
244
- exit!
245
- rescue Mongo::OperationFailure => e
246
- log.fatal "Operation failed. Probably, 'mongod' needs an authentication: #{e}"
247
- exit!
248
- rescue Exception => e
249
- log.warn "mongo unknown error #{e}, set #{LIMIT_BEFORE_v1_8} to chunk limit"
250
- limit = LIMIT_BEFORE_v1_8
251
- end
252
-
253
- if @buffer.buffer_chunk_limit > limit
254
- log.warn ":buffer_chunk_limit(#{@buffer.buffer_chunk_limit}) is large. Reset :buffer_chunk_limit with #{limit}"
255
- limit
256
- else
257
- @buffer.buffer_chunk_limit
258
- end
259
- end
260
-
261
- def mongod_version
262
- version = nil
263
-
264
- begin
265
- version = get_connection.command('buildInfo' => 1)['version']
266
- rescue Mongo::OperationFailure
267
- # fallback for buggy mongod version support
268
- version = authenticate(Mongo::MongoClient.new(@host, @port, @connection_options).db('admin')).command('buildInfo' => 1)['version']
269
- end
270
-
271
- version
272
- end
273
-
274
261
  def replace_key_of_hash(hash_or_array, pattern, replacement)
275
262
  case hash_or_array
276
263
  when Array
@@ -1,9 +1,9 @@
1
1
  require 'test_helper'
2
+ require 'fluent/plugin/in_mongo_tail'
2
3
 
3
4
  class MongoTailInputTest < Test::Unit::TestCase
4
5
  def setup
5
6
  Fluent::Test.setup
6
- require 'fluent/plugin/in_mongo_tail'
7
7
  end
8
8
 
9
9
  CONFIG = %[
@@ -1,63 +1,12 @@
1
1
  # -*- coding: utf-8 -*-
2
2
  require 'tools/rs_test_helper'
3
+ require 'fluent/plugin/out_mongo'
3
4
 
4
- class MongoOutputTest < Test::Unit::TestCase
5
- include MongoTestHelper
6
-
7
- def setup
8
- Fluent::Test.setup
9
- require 'fluent/plugin/out_mongo'
10
-
11
- setup_mongod
12
- end
13
-
14
- def teardown
15
- @db.collection(collection_name).drop
16
- teardown_mongod
17
- end
18
-
5
+ module MongoOutputTestCases
19
6
  def collection_name
20
7
  'test'
21
8
  end
22
9
 
23
- def default_config
24
- %[
25
- type mongo
26
- database #{MONGO_DB_DB}
27
- collection #{collection_name}
28
- include_time_key true # TestDriver ignore config_set_default?
29
- ]
30
- end
31
-
32
- def create_driver(conf = default_config)
33
- conf = conf + %[
34
- port #{@@mongod_port}
35
- ]
36
- @db = Mongo::MongoClient.new('localhost', @@mongod_port).db(MONGO_DB_DB)
37
- Fluent::Test::BufferedOutputTestDriver.new(Fluent::MongoOutput).configure(conf)
38
- end
39
-
40
- def test_configure
41
- d = create_driver(%[
42
- type mongo
43
- database fluent_test
44
- collection test_collection
45
-
46
- capped
47
- capped_size 100
48
- ])
49
-
50
- assert_equal('fluent_test', d.instance.database)
51
- assert_equal('test_collection', d.instance.collection)
52
- assert_equal('localhost', d.instance.host)
53
- assert_equal(@@mongod_port, d.instance.port)
54
- assert_equal({:capped => true, :size => 100}, d.instance.collection_options)
55
- assert_equal({:ssl => false, :pool_size => 1, :j => false}, d.instance.connection_options)
56
- # buffer_chunk_limit moved from configure to start
57
- # I will move this test to correct space after BufferedOutputTestDriver supports start method invoking
58
- # assert_equal(Fluent::MongoOutput::LIMIT_BEFORE_v1_8, d.instance.instance_variable_get(:@buffer).buffer_chunk_limit)
59
- end
60
-
61
10
  def test_configure_with_write_concern
62
11
  d = create_driver(default_config + %[
63
12
  write_concern 2
@@ -223,16 +172,86 @@ class MongoOutputTest < Test::Unit::TestCase
223
172
  end
224
173
  end
225
174
 
226
- class MongoReplOutputTest < MongoOutputTest
175
+ class MongoOutputTest < Test::Unit::TestCase
176
+ include MongoOutputTestCases
177
+
178
+ class << self
179
+ def startup
180
+ MongoTestHelper.setup_mongod
181
+ end
182
+
183
+ def shutdown
184
+ MongoTestHelper.teardown_mongod
185
+ end
186
+ end
187
+
188
+ def setup
189
+ Fluent::Test.setup
190
+ end
191
+
192
+ def teardown
193
+ @db.collection(collection_name).drop
194
+ end
195
+
196
+ def default_config
197
+ %[
198
+ type mongo
199
+ database #{MONGO_DB_DB}
200
+ collection #{collection_name}
201
+ include_time_key true # TestDriver ignore config_set_default?
202
+ ]
203
+ end
204
+
205
+ def create_driver(conf = default_config)
206
+ conf = conf + %[
207
+ port #{MongoTestHelper.mongod_port}
208
+ ]
209
+ @db = Mongo::MongoClient.new('localhost', MongoTestHelper.mongod_port).db(MONGO_DB_DB)
210
+ Fluent::Test::BufferedOutputTestDriver.new(Fluent::MongoOutput).configure(conf)
211
+ end
212
+
213
+ def test_configure
214
+ d = create_driver(%[
215
+ type mongo
216
+ database fluent_test
217
+ collection test_collection
218
+
219
+ capped
220
+ capped_size 100
221
+ ])
222
+
223
+ assert_equal('fluent_test', d.instance.database)
224
+ assert_equal('test_collection', d.instance.collection)
225
+ assert_equal('localhost', d.instance.host)
226
+ assert_equal(MongoTestHelper.mongod_port, d.instance.port)
227
+ assert_equal({:capped => true, :size => 100}, d.instance.collection_options)
228
+ assert_equal({:ssl => false, :pool_size => 1, :j => false}, d.instance.connection_options)
229
+ # buffer_chunk_limit moved from configure to start
230
+ # I will move this test to correct space after BufferedOutputTestDriver supports start method invoking
231
+ # assert_equal(Fluent::MongoOutput::LIMIT_BEFORE_v1_8, d.instance.instance_variable_get(:@buffer).buffer_chunk_limit)
232
+ end
233
+ end
234
+
235
+ class MongoReplOutputTest < Test::Unit::TestCase
236
+ include MongoOutputTestCases
237
+
238
+ class << self
239
+ def startup
240
+ setup_rs
241
+ end
242
+
243
+ def shutdown
244
+ teardown_rs
245
+ end
246
+ end
247
+
227
248
  def setup
228
249
  Fluent::Test.setup
229
250
  require 'fluent/plugin/out_mongo_replset'
230
-
231
- ensure_rs
232
251
  end
233
252
 
234
253
  def teardown
235
- @rs.restart_killed_nodes
254
+ @@rs.restart_killed_nodes
236
255
  if defined?(@db) && @db
237
256
  @db.collection(collection_name).drop
238
257
  @db.connection.close
@@ -251,7 +270,7 @@ class MongoReplOutputTest < MongoOutputTest
251
270
  end
252
271
 
253
272
  def create_driver(conf = default_config)
254
- @db = Mongo::MongoReplicaSetClient.new(build_seeds(3), :name => @rs.name).db(MONGO_DB_DB)
273
+ @db = Mongo::MongoReplicaSetClient.new(build_seeds(3), :name => @@rs.name).db(MONGO_DB_DB)
255
274
  Fluent::Test::BufferedOutputTestDriver.new(Fluent::MongoOutputReplset).configure(conf)
256
275
  end
257
276
 
@@ -1,9 +1,9 @@
1
1
  require 'test_helper'
2
+ require 'fluent/plugin/out_mongo'
2
3
 
3
4
  class MongoTagCollectionTest < Test::Unit::TestCase
4
5
  def setup
5
6
  Fluent::Test.setup
6
- require 'fluent/plugin/out_mongo'
7
7
  end
8
8
 
9
9
  CONFIG = %[
@@ -33,33 +33,27 @@ MONGO_DB_DB = 'fluent_test'
33
33
  MONGO_DB_PATH = File.join(File.dirname(__FILE__), 'plugin', 'data')
34
34
 
35
35
  module MongoTestHelper
36
- @@setup_count = 0
37
-
38
- def cleanup_mongod_env
39
- system("killall mongod")
36
+ def self.cleanup_mongod_env
37
+ Process.kill "TERM", @@pid
38
+ Process.waitpid @@pid
40
39
  system("rm -rf #{MONGO_DB_PATH}")
41
- system("mkdir -p #{MONGO_DB_PATH}")
42
40
  end
43
41
 
44
- def setup_mongod
45
- unless defined?(@@current_mongo_test_class) and @@current_mongo_test_class == self.class
46
- cleanup_mongod_env
42
+ def self.setup_mongod
43
+ system("rm -rf #{MONGO_DB_PATH}")
44
+ system("mkdir -p #{MONGO_DB_PATH}")
47
45
 
48
- @@current_mongo_test_class = self.class
49
- @@mongod_port = unused_port
50
- @@pid = spawn(ENV['mongod'], "--port=#{@@mongod_port}", "--dbpath=#{MONGO_DB_PATH}")
51
- sleep 3
52
- end
46
+ @@mongod_port = unused_port
47
+ @@pid = spawn(ENV['mongod'], "--port=#{@@mongod_port}", "--dbpath=#{MONGO_DB_PATH}")
48
+ sleep 3
49
+ end
53
50
 
54
- @@setup_count += 1;
51
+ def self.teardown_mongod
52
+ Mongo::Connection.new('localhost', @@mongod_port).drop_database(MONGO_DB_DB)
53
+ cleanup_mongod_env
55
54
  end
56
55
 
57
- def teardown_mongod
58
- if defined?(@@current_mongo_test_class)
59
- Mongo::Connection.new('localhost', @@mongod_port).drop_database(MONGO_DB_DB)
60
- end
61
- if @@setup_count == self.class.methods.size
62
- cleanup_mongod_env
63
- end
56
+ def self.mongod_port
57
+ @@mongod_port
64
58
  end
65
59
  end
@@ -45,8 +45,9 @@ class ReplSetManager
45
45
  end
46
46
 
47
47
  def start_set
48
- system("killall mongod")
49
- sleep(1)
48
+ @mongods.keys.each do |node|
49
+ kill(node)
50
+ end
50
51
  should_start = true
51
52
  puts "** Starting a replica set with #{@count} nodes"
52
53
 
@@ -79,7 +80,9 @@ class ReplSetManager
79
80
  end
80
81
 
81
82
  def cleanup_set
82
- system("killall mongod")
83
+ @mongods.keys.each do |node|
84
+ kill(node)
85
+ end
83
86
  @count.times do |n|
84
87
  system("rm -rf #{@mongods[n]['db_path']}")
85
88
  end
@@ -193,9 +196,11 @@ class ReplSetManager
193
196
  end
194
197
 
195
198
  def kill(node, signal=2)
199
+ return unless @mongods[node]['up']
200
+
196
201
  pid = @mongods[node]['pid']
197
202
  puts "** Killing node with pid #{pid} at port #{@mongods[node]['port']}"
198
- system("kill #{pid}")
203
+ Process.kill signal, pid
199
204
  dead = wait_for_death(pid)
200
205
  @mongods[node]['up'] = false if dead
201
206
  end
@@ -254,7 +259,7 @@ class ReplSetManager
254
259
  system(@mongods[node]['start'])
255
260
  @mongods[node]['up'] = true
256
261
  sleep(0.5)
257
- @mongods[node]['pid'] = File.open(File.join(@mongods[node]['db_path'], 'mongod.lock')).read.strip
262
+ @mongods[node]['pid'] = File.open(File.join(@mongods[node]['db_path'], 'mongod.lock')).read.strip.to_i
258
263
  end
259
264
  alias :restart :start
260
265
 
@@ -5,13 +5,13 @@ require 'tools/repl_set_manager'
5
5
  class Test::Unit::TestCase
6
6
  # Ensure replica set is available as an instance variable and that
7
7
  # a new set is spun up for each TestCase class
8
- def ensure_rs
9
- unless defined?(@@current_class) and @@current_class == self.class
10
- @@current_class = self.class
11
- @@rs = ReplSetManager.new
12
- @@rs.start_set
13
- end
14
- @rs = @@rs
8
+ def self.setup_rs
9
+ @@rs = ReplSetManager.new
10
+ @@rs.start_set
11
+ end
12
+
13
+ def self.teardown_rs
14
+ @@rs.cleanup_set
15
15
  end
16
16
 
17
17
  # Generic code for rescuing connection failures and retrying operations.
@@ -32,7 +32,7 @@ class Test::Unit::TestCase
32
32
  def build_seeds(num_hosts)
33
33
  seeds = []
34
34
  num_hosts.times do |n|
35
- seeds << "#{@rs.host}:#{@rs.ports[n]}"
35
+ seeds << "#{@@rs.host}:#{@@rs.ports[n]}"
36
36
  end
37
37
  seeds
38
38
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-mongo
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.7.12
4
+ version: 0.7.13
5
5
  platform: ruby
6
6
  authors:
7
7
  - Masahiro Nakagawa
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-02-09 00:00:00.000000000 Z
11
+ date: 2016-06-03 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -149,7 +149,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
149
149
  version: '0'
150
150
  requirements: []
151
151
  rubyforge_project:
152
- rubygems_version: 2.2.2
152
+ rubygems_version: 2.5.1
153
153
  signing_key:
154
154
  specification_version: 4
155
155
  summary: MongoDB plugin for Fluentd