fluent-plugin-mongo 0.7.16 → 0.8.0.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,25 @@
1
+ module Fluent
2
+ module LoggerSupport
3
+ def self.included(klass)
4
+ klass.instance_eval {
5
+ desc "MongoDB log level"
6
+ config_param :mongo_log_level, :string, default: 'info'
7
+ }
8
+ end
9
+
10
+ def configure_logger(mongo_log_level)
11
+ Mongo::Logger.level = case @mongo_log_level.downcase
12
+ when 'fatal'
13
+ Logger::FATAL
14
+ when 'error'
15
+ Logger::ERROR
16
+ when 'warn'
17
+ Logger::WARN
18
+ when 'info'
19
+ Logger::INFO
20
+ when 'debug'
21
+ Logger::DEBUG
22
+ end
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,32 @@
1
+ module Fluent
2
+ module MongoAuthParams
3
+ def self.included(klass)
4
+ klass.instance_eval {
5
+ desc "MongoDB user"
6
+ config_param :user, :string, default: nil
7
+ desc "MongoDB password"
8
+ config_param :password, :string, default: nil, secret: true
9
+ desc "MongoDB authentication database"
10
+ config_param :auth_source, :string, default: nil
11
+ }
12
+ end
13
+ end
14
+
15
+ module MongoAuth
16
+ def authenticate(client)
17
+ unless @user.nil? || @password.nil?
18
+ begin
19
+ if @auth_source.nil?
20
+ client = client.with(user: @user, password: @password)
21
+ else
22
+ client = client.with(user: @user, password: @password, auth_source: @auth_source)
23
+ end
24
+ rescue Mongo::Auth::Unauthorized => e
25
+ log.fatal e
26
+ exit!
27
+ end
28
+ end
29
+ client
30
+ end
31
+ end
32
+ end
@@ -4,8 +4,15 @@ module Fluent
4
4
  class MongoOutput < BufferedOutput
5
5
  Plugin.register_output('mongo', self)
6
6
 
7
- require 'fluent/plugin/mongo_util'
8
- include MongoUtil
7
+ unless method_defined?(:log)
8
+ define_method(:log) { $log }
9
+ end
10
+
11
+ require 'fluent/plugin/mongo_auth'
12
+ include MongoAuthParams
13
+ include MongoAuth
14
+ require 'fluent/plugin/logger_support'
15
+ include LoggerSupport
9
16
 
10
17
  include SetTagKeyMixin
11
18
  config_set_default :include_tag_key, false
@@ -13,49 +20,47 @@ module Fluent
13
20
  include SetTimeKeyMixin
14
21
  config_set_default :include_time_key, true
15
22
 
23
+ desc "MongoDB database"
16
24
  config_param :database, :string
17
- config_param :collection, :string, :default => 'untagged'
18
- config_param :host, :string, :default => 'localhost'
19
- config_param :port, :integer, :default => 27017
20
- config_param :ignore_invalid_record, :bool, :default => false,
21
- :deprecated => "This parameter will be ignored since v0.8 because mongo driver 2.x doesn't support base functionality for this parameter"
22
- config_param :disable_collection_check, :bool, :default => nil
23
- config_param :exclude_broken_fields, :string, :default => nil
24
- config_param :write_concern, :integer, :default => nil
25
- config_param :journaled, :bool, :default => false
26
- config_param :socket_pool_size, :integer, :default => 1
27
- config_param :replace_dot_in_key_with, :string, :default => nil
28
- config_param :replace_dollar_in_key_with, :string, :default => nil
25
+ desc "MongoDB collection"
26
+ config_param :collection, :string, default: 'untagged'
27
+ desc "MongoDB host"
28
+ config_param :host, :string, default: 'localhost'
29
+ desc "MongoDB port"
30
+ config_param :port, :integer, default: 27017
31
+ desc "MongoDB write_concern"
32
+ config_param :write_concern, :integer, default: nil
33
+ desc "MongoDB journaled"
34
+ config_param :journaled, :bool, default: false
35
+ desc "Replace dot with specified string"
36
+ config_param :replace_dot_in_key_with, :string, default: nil
37
+ desc "Replace dollar with specified string"
38
+ config_param :replace_dollar_in_key_with, :string, default: nil
29
39
 
30
40
  # tag mapping mode
31
- config_param :tag_mapped, :bool, :default => false
32
- config_param :remove_tag_prefix, :string, :default => nil
41
+ desc "Use tag_mapped mode"
42
+ config_param :tag_mapped, :bool, default: false
43
+ desc "Remove tag prefix"
44
+ config_param :remove_tag_prefix, :string, default: nil
33
45
 
34
46
  # SSL connection
35
- config_param :ssl, :bool, :default => false
36
- config_param :ssl_cert, :string, :default => nil
37
- config_param :ssl_key, :string, :default => nil
38
- config_param :ssl_key_pass_phrase, :string, :default => nil, :secret => true
39
- config_param :ssl_verify, :bool, :default => false
40
- config_param :ssl_ca_cert, :string, :default => nil
41
-
42
- # For older (1.7 or earlier) MongoDB versions
43
- config_param :mongodb_smaller_bson_limit, :bool, :default => false
47
+ config_param :ssl, :bool, default: false
48
+ config_param :ssl_cert, :string, default: nil
49
+ config_param :ssl_key, :string, default: nil
50
+ config_param :ssl_key_pass_phrase, :string, default: nil, secret: true
51
+ config_param :ssl_verify, :bool, default: false
52
+ config_param :ssl_ca_cert, :string, default: nil
44
53
 
45
- attr_reader :collection_options, :connection_options
46
-
47
- unless method_defined?(:log)
48
- define_method(:log) { $log }
49
- end
54
+ attr_reader :client_options, :collection_options
50
55
 
51
56
  def initialize
52
57
  super
58
+
53
59
  require 'mongo'
54
60
  require 'msgpack'
55
61
 
56
- @clients = {}
57
- @connection_options = {}
58
- @collection_options = {:capped => false}
62
+ @client_options = {}
63
+ @collection_options = {capped: false}
59
64
  end
60
65
 
61
66
  # Following limits are heuristic. BSON is sometimes bigger than MessagePack and JSON.
@@ -91,18 +96,9 @@ module Fluent
91
96
 
92
97
  if conf.has_key?('tag_mapped')
93
98
  @tag_mapped = true
94
- @disable_collection_check = true if @disable_collection_check.nil?
95
- else
96
- @disable_collection_check = false if @disable_collection_check.nil?
97
99
  end
98
100
  raise ConfigError, "normal mode requires collection parameter" if !@tag_mapped and !conf.has_key?('collection')
99
101
 
100
- if remove_tag_prefix = conf['remove_tag_prefix']
101
- @remove_tag_prefix = Regexp.new('^' + Regexp.escape(remove_tag_prefix))
102
- end
103
-
104
- @exclude_broken_fields = @exclude_broken_fields.split(',') if @exclude_broken_fields
105
-
106
102
  if conf.has_key?('capped')
107
103
  raise ConfigError, "'capped_size' parameter is required on <store> of Mongo output" unless conf.has_key?('capped_size')
108
104
  @collection_options[:capped] = true
@@ -110,18 +106,20 @@ module Fluent
110
106
  @collection_options[:max] = Config.size_value(conf['capped_max']) if conf.has_key?('capped_max')
111
107
  end
112
108
 
113
- @connection_options[:w] = @write_concern unless @write_concern.nil?
114
- @connection_options[:j] = @journaled
115
- @connection_options[:pool_size] = @socket_pool_size
109
+ if remove_tag_prefix = conf['remove_tag_prefix']
110
+ @remove_tag_prefix = Regexp.new('^' + Regexp.escape(remove_tag_prefix))
111
+ end
116
112
 
117
- @connection_options[:ssl] = @ssl
113
+ @client_options[:write] = {j: @journaled}
114
+ @client_options[:write].merge!({w: @write_concern}) unless @write_concern.nil?
115
+ @client_options[:ssl] = @ssl
118
116
 
119
117
  if @ssl
120
- @connection_options[:ssl_cert] = @ssl_cert
121
- @connection_options[:ssl_key] = @ssl_key
122
- @connection_options[:ssl_key_pass_phrase] = @ssl_key_pass_phrase
123
- @connection_options[:ssl_verify] = @ssl_verify
124
- @connection_options[:ssl_ca_cert] = @ssl_ca_cert
118
+ @client_options[:ssl_cert] = @ssl_cert
119
+ @client_options[:ssl_key] = @ssl_key
120
+ @client_options[:ssl_key_pass_phrase] = @ssl_key_pass_phrase
121
+ @client_options[:ssl_verify] = @ssl_verify
122
+ @client_options[:ssl_ca_cert] = @ssl_ca_cert
125
123
  end
126
124
 
127
125
  # MongoDB uses BSON's Date for time.
@@ -129,28 +127,23 @@ module Fluent
129
127
  time
130
128
  end
131
129
 
132
- $log.debug "Setup mongo configuration: mode = #{@tag_mapped ? 'tag mapped' : 'normal'}"
130
+ configure_logger(@mongo_log_level)
131
+
132
+ log.debug "Setup mongo configuration: mode = #{@tag_mapped ? 'tag mapped' : 'normal'}"
133
133
  end
134
134
 
135
135
  def start
136
- # Non tag mapped mode, we can check collection configuration before server start.
137
- get_or_create_collection(@collection) unless @tag_mapped
138
-
136
+ @client = client
137
+ @client = authenticate(@client)
139
138
  super
140
139
  end
141
140
 
142
141
  def shutdown
143
- # Mongo::Connection checks alive or closed myself
144
- @clients.values.each { |client| client.db.connection.close }
142
+ @client.close
145
143
  super
146
144
  end
147
145
 
148
- def format(tag, time, record)
149
- [time, record].to_msgpack
150
- end
151
-
152
146
  def emit(tag, es, chain)
153
- # TODO: Should replacement using eval in configure?
154
147
  if @tag_mapped
155
148
  super(tag, es, chain, tag)
156
149
  else
@@ -158,64 +151,27 @@ module Fluent
158
151
  end
159
152
  end
160
153
 
154
+ def format(tag, time, record)
155
+ [time, record].to_msgpack
156
+ end
157
+
161
158
  def write(chunk)
162
- # TODO: See emit comment
163
159
  collection_name = @tag_mapped ? chunk.key : @collection
164
- operate(get_or_create_collection(collection_name), collect_records(chunk))
160
+ operate(format_collection_name(collection_name), collect_records(chunk))
165
161
  end
166
162
 
167
163
  private
168
164
 
169
- INSERT_ARGUMENT = {:collect_on_error => true}
170
- BROKEN_DATA_KEY = '__broken_data'
171
-
172
- def operate(collection, records)
173
- begin
174
- if @replace_dot_in_key_with
175
- records.map! do |r|
176
- replace_key_of_hash(r, ".", @replace_dot_in_key_with)
177
- end
178
- end
179
- if @replace_dollar_in_key_with
180
- records.map! do |r|
181
- replace_key_of_hash(r, /^\$/, @replace_dollar_in_key_with)
182
- end
183
- end
184
-
185
- record_ids, error_records = collection.insert(records, INSERT_ARGUMENT)
186
- if !@ignore_invalid_record and error_records.size > 0
187
- operate_invalid_records(collection, error_records)
188
- end
189
- rescue Mongo::OperationFailure => e
190
- # Probably, all records of _records_ are broken...
191
- if e.error_code == 13066 # 13066 means "Message contains no documents"
192
- operate_invalid_records(collection, records) unless @ignore_invalid_record
193
- else
194
- raise e
195
- end
196
- end
197
- records
198
- end
199
-
200
- def operate_invalid_records(collection, records)
201
- converted_records = records.map { |record|
202
- new_record = {}
203
- new_record[@tag_key] = record.delete(@tag_key) if @include_tag_key
204
- new_record[@time_key] = record.delete(@time_key)
205
- if @exclude_broken_fields
206
- @exclude_broken_fields.each { |key|
207
- new_record[key] = record.delete(key)
208
- }
209
- end
210
- new_record[BROKEN_DATA_KEY] = BSON::Binary.new(Marshal.dump(record))
211
- new_record
212
- }
213
- collection.insert(converted_records)
165
+ def client
166
+ @client_options[:database] = @database
167
+ @client_options[:user] = @user if @user
168
+ @client_options[:password] = @password if @password
169
+ Mongo::Client.new(["#{@host}:#{@port}"], @client_options)
214
170
  end
215
171
 
216
172
  def collect_records(chunk)
217
173
  records = []
218
- chunk.msgpack_each { |time, record|
174
+ chunk.msgpack_each {|time, record|
219
175
  record[@time_key] = Time.at(time || record[@time_key]) if @include_time_key
220
176
  records << record
221
177
  }
@@ -232,35 +188,26 @@ module Fluent
232
188
  formatted
233
189
  end
234
190
 
235
- def get_or_create_collection(collection_name)
236
- collection_name = format_collection_name(collection_name)
237
- return @clients[collection_name] if @clients[collection_name]
238
-
239
- @db ||= get_connection
240
- if @db.collection_names.include?(collection_name)
241
- collection = @db.collection(collection_name)
242
- unless @disable_collection_check
243
- capped = collection.capped?
244
- unless @collection_options[:capped] == capped # TODO: Verify capped configuration
245
- new_mode = format_collection_mode(@collection_options[:capped])
246
- old_mode = format_collection_mode(capped)
247
- raise ConfigError, "New configuration is different from existing collection: new = #{new_mode}, old = #{old_mode}"
191
+ def operate(collection, records)
192
+ begin
193
+ if @replace_dot_in_key_with
194
+ records.map! do |r|
195
+ replace_key_of_hash(r, ".", @replace_dot_in_key_with)
196
+ end
197
+ end
198
+ if @replace_dollar_in_key_with
199
+ records.map! do |r|
200
+ replace_key_of_hash(r, /^\$/, @replace_dollar_in_key_with)
248
201
  end
249
202
  end
250
- else
251
- collection = @db.create_collection(collection_name, @collection_options)
252
- end
253
-
254
- @clients[collection_name] = collection
255
- end
256
-
257
- def format_collection_mode(mode)
258
- mode ? 'capped' : 'normal'
259
- end
260
203
 
261
- def get_connection
262
- db = Mongo::MongoClient.new(@host, @port, @connection_options).db(@database)
263
- authenticate(db)
204
+ @client[collection, @collection_options].insert_many(records)
205
+ rescue Mongo::Error::BulkWriteError => e
206
+ log.warn "#{records.size - e.result["n_inserted"]} documents are not inserted. Maybe these documents are invalid as a BSON."
207
+ rescue ArgumentError => e
208
+ log.warn e
209
+ end
210
+ records
264
211
  end
265
212
 
266
213
  def replace_key_of_hash(hash_or_array, pattern, replacement)
@@ -4,66 +4,55 @@ module Fluent
4
4
  class MongoOutputReplset < MongoOutput
5
5
  Plugin.register_output('mongo_replset', self)
6
6
 
7
+ unless method_defined?(:log)
8
+ define_method(:log) { $log }
9
+ end
10
+
7
11
  config_set_default :include_tag_key, false
8
12
  config_set_default :include_time_key, true
9
13
 
10
- config_param :nodes, :string
11
- config_param :name, :string, :default => nil
14
+ desc "Replica set name"
15
+ config_param :replica_set, :string
16
+ desc "Read from specified role"
12
17
  config_param :read, :string, :default => nil
13
- config_param :refresh_mode, :string, :default => nil
14
- config_param :refresh_interval, :integer, :default => nil
18
+ desc "Retry number"
15
19
  config_param :num_retries, :integer, :default => 60
16
20
 
17
- # disable single node configuration
18
- config_param :host, :string, :default => nil
19
- config_param :port, :integer, :default => nil
21
+ unless method_defined?(:log)
22
+ define_method(:log) { $log }
23
+ end
20
24
 
21
25
  def configure(conf)
22
26
  super
23
27
 
24
- @nodes = parse_nodes(conf['nodes'])
25
- if name = conf['name']
26
- @connection_options[:name] = conf['name']
28
+ if replica_set = conf['replica_set']
29
+ @client_options[:replica_set] = replica_set
27
30
  end
28
31
  if read = conf['read']
29
- @connection_options[:read] = read.to_sym
30
- end
31
- if refresh_mode = conf['refresh_mode']
32
- @connection_options[:refresh_mode] = refresh_mode.to_sym
33
- end
34
- if refresh_interval = conf['refresh_interval']
35
- @connection_options[:refresh_interval] = refresh_interval
32
+ @client_options[:read] = read.to_sym
36
33
  end
37
34
 
38
- $log.debug "Setup replica set configuration: nodes = #{conf['nodes']}"
35
+ log.debug "Setup replica set configuration: #{conf['replica_set']}"
39
36
  end
40
37
 
41
38
  private
42
39
 
43
- def operate(collection, records)
40
+ def operate(client, records)
44
41
  rescue_connection_failure do
45
- super(collection, records)
42
+ super(client, records)
46
43
  end
47
44
  end
48
45
 
49
- def parse_nodes(nodes)
50
- nodes.split(',')
51
- end
52
-
53
- def get_connection
54
- db = Mongo::MongoReplicaSetClient.new(@nodes, @connection_options).db(@database)
55
- authenticate(db)
56
- end
57
-
58
46
  def rescue_connection_failure
59
47
  retries = 0
60
48
  begin
61
49
  yield
62
- rescue Mongo::ConnectionFailure => e
50
+ rescue Mongo::Error::OperationFailure => e
63
51
  retries += 1
64
52
  raise e if retries > @num_retries
65
53
 
66
- log.warn "Failed to connect to Replica Set. Try to retry: retry number = #{retries}"
54
+ log.warn "Failed to operate to Replica Set. Try to retry: retry count = #{retries}"
55
+
67
56
  sleep 0.5
68
57
  retry
69
58
  end