fluent-plugin-mongo 0.7.16 → 0.8.0.rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +4 -1
- data/VERSION +1 -1
- data/bin/mongo-tail +37 -30
- data/fluent-plugin-mongo.gemspec +1 -1
- data/lib/fluent/plugin/in_mongo_tail.rb +122 -138
- data/lib/fluent/plugin/logger_support.rb +25 -0
- data/lib/fluent/plugin/mongo_auth.rb +32 -0
- data/lib/fluent/plugin/out_mongo.rb +83 -136
- data/lib/fluent/plugin/out_mongo_replset.rb +20 -31
- data/test/helper.rb +6 -0
- data/test/plugin/test_in_mongo_tail.rb +96 -0
- data/test/plugin/test_out_mongo.rb +282 -0
- data/test/plugin/test_out_mongo_replset.rb +125 -0
- metadata +16 -22
- data/lib/fluent/plugin/mongo_util.rb +0 -27
- data/lib/fluent/plugin/out_mongo_tag_collection.rb +0 -20
- data/test/plugin/in_mongo_tail.rb +0 -73
- data/test/plugin/out_mongo.rb +0 -298
- data/test/plugin/out_mongo_tag_mapped.rb +0 -69
- data/test/test_helper.rb +0 -59
- data/test/tools/auth_repl_set_manager.rb +0 -14
- data/test/tools/repl_set_manager.rb +0 -420
- data/test/tools/rs_test_helper.rb +0 -39
@@ -0,0 +1,25 @@
|
|
1
|
+
module Fluent
|
2
|
+
module LoggerSupport
|
3
|
+
def self.included(klass)
|
4
|
+
klass.instance_eval {
|
5
|
+
desc "MongoDB log level"
|
6
|
+
config_param :mongo_log_level, :string, default: 'info'
|
7
|
+
}
|
8
|
+
end
|
9
|
+
|
10
|
+
def configure_logger(mongo_log_level)
|
11
|
+
Mongo::Logger.level = case @mongo_log_level.downcase
|
12
|
+
when 'fatal'
|
13
|
+
Logger::FATAL
|
14
|
+
when 'error'
|
15
|
+
Logger::ERROR
|
16
|
+
when 'warn'
|
17
|
+
Logger::WARN
|
18
|
+
when 'info'
|
19
|
+
Logger::INFO
|
20
|
+
when 'debug'
|
21
|
+
Logger::DEBUG
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
module Fluent
|
2
|
+
module MongoAuthParams
|
3
|
+
def self.included(klass)
|
4
|
+
klass.instance_eval {
|
5
|
+
desc "MongoDB user"
|
6
|
+
config_param :user, :string, default: nil
|
7
|
+
desc "MongoDB password"
|
8
|
+
config_param :password, :string, default: nil, secret: true
|
9
|
+
desc "MongoDB authentication database"
|
10
|
+
config_param :auth_source, :string, default: nil
|
11
|
+
}
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
module MongoAuth
|
16
|
+
def authenticate(client)
|
17
|
+
unless @user.nil? || @password.nil?
|
18
|
+
begin
|
19
|
+
if @auth_source.nil?
|
20
|
+
client = client.with(user: @user, password: @password)
|
21
|
+
else
|
22
|
+
client = client.with(user: @user, password: @password, auth_source: @auth_source)
|
23
|
+
end
|
24
|
+
rescue Mongo::Auth::Unauthorized => e
|
25
|
+
log.fatal e
|
26
|
+
exit!
|
27
|
+
end
|
28
|
+
end
|
29
|
+
client
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -4,8 +4,15 @@ module Fluent
|
|
4
4
|
class MongoOutput < BufferedOutput
|
5
5
|
Plugin.register_output('mongo', self)
|
6
6
|
|
7
|
-
|
8
|
-
|
7
|
+
unless method_defined?(:log)
|
8
|
+
define_method(:log) { $log }
|
9
|
+
end
|
10
|
+
|
11
|
+
require 'fluent/plugin/mongo_auth'
|
12
|
+
include MongoAuthParams
|
13
|
+
include MongoAuth
|
14
|
+
require 'fluent/plugin/logger_support'
|
15
|
+
include LoggerSupport
|
9
16
|
|
10
17
|
include SetTagKeyMixin
|
11
18
|
config_set_default :include_tag_key, false
|
@@ -13,49 +20,47 @@ module Fluent
|
|
13
20
|
include SetTimeKeyMixin
|
14
21
|
config_set_default :include_time_key, true
|
15
22
|
|
23
|
+
desc "MongoDB database"
|
16
24
|
config_param :database, :string
|
17
|
-
|
18
|
-
config_param :
|
19
|
-
|
20
|
-
config_param :
|
21
|
-
|
22
|
-
config_param :
|
23
|
-
|
24
|
-
config_param :write_concern, :integer, :
|
25
|
-
|
26
|
-
config_param :
|
27
|
-
|
28
|
-
config_param :
|
25
|
+
desc "MongoDB collection"
|
26
|
+
config_param :collection, :string, default: 'untagged'
|
27
|
+
desc "MongoDB host"
|
28
|
+
config_param :host, :string, default: 'localhost'
|
29
|
+
desc "MongoDB port"
|
30
|
+
config_param :port, :integer, default: 27017
|
31
|
+
desc "MongoDB write_concern"
|
32
|
+
config_param :write_concern, :integer, default: nil
|
33
|
+
desc "MongoDB journaled"
|
34
|
+
config_param :journaled, :bool, default: false
|
35
|
+
desc "Replace dot with specified string"
|
36
|
+
config_param :replace_dot_in_key_with, :string, default: nil
|
37
|
+
desc "Replace dollar with specified string"
|
38
|
+
config_param :replace_dollar_in_key_with, :string, default: nil
|
29
39
|
|
30
40
|
# tag mapping mode
|
31
|
-
|
32
|
-
config_param :
|
41
|
+
desc "Use tag_mapped mode"
|
42
|
+
config_param :tag_mapped, :bool, default: false
|
43
|
+
desc "Remove tag prefix"
|
44
|
+
config_param :remove_tag_prefix, :string, default: nil
|
33
45
|
|
34
46
|
# SSL connection
|
35
|
-
config_param :ssl, :bool, :
|
36
|
-
config_param :ssl_cert, :string, :
|
37
|
-
config_param :ssl_key, :string, :
|
38
|
-
config_param :ssl_key_pass_phrase, :string, :
|
39
|
-
config_param :ssl_verify, :bool, :
|
40
|
-
config_param :ssl_ca_cert, :string, :
|
41
|
-
|
42
|
-
# For older (1.7 or earlier) MongoDB versions
|
43
|
-
config_param :mongodb_smaller_bson_limit, :bool, :default => false
|
47
|
+
config_param :ssl, :bool, default: false
|
48
|
+
config_param :ssl_cert, :string, default: nil
|
49
|
+
config_param :ssl_key, :string, default: nil
|
50
|
+
config_param :ssl_key_pass_phrase, :string, default: nil, secret: true
|
51
|
+
config_param :ssl_verify, :bool, default: false
|
52
|
+
config_param :ssl_ca_cert, :string, default: nil
|
44
53
|
|
45
|
-
attr_reader :
|
46
|
-
|
47
|
-
unless method_defined?(:log)
|
48
|
-
define_method(:log) { $log }
|
49
|
-
end
|
54
|
+
attr_reader :client_options, :collection_options
|
50
55
|
|
51
56
|
def initialize
|
52
57
|
super
|
58
|
+
|
53
59
|
require 'mongo'
|
54
60
|
require 'msgpack'
|
55
61
|
|
56
|
-
@
|
57
|
-
@
|
58
|
-
@collection_options = {:capped => false}
|
62
|
+
@client_options = {}
|
63
|
+
@collection_options = {capped: false}
|
59
64
|
end
|
60
65
|
|
61
66
|
# Following limits are heuristic. BSON is sometimes bigger than MessagePack and JSON.
|
@@ -91,18 +96,9 @@ module Fluent
|
|
91
96
|
|
92
97
|
if conf.has_key?('tag_mapped')
|
93
98
|
@tag_mapped = true
|
94
|
-
@disable_collection_check = true if @disable_collection_check.nil?
|
95
|
-
else
|
96
|
-
@disable_collection_check = false if @disable_collection_check.nil?
|
97
99
|
end
|
98
100
|
raise ConfigError, "normal mode requires collection parameter" if !@tag_mapped and !conf.has_key?('collection')
|
99
101
|
|
100
|
-
if remove_tag_prefix = conf['remove_tag_prefix']
|
101
|
-
@remove_tag_prefix = Regexp.new('^' + Regexp.escape(remove_tag_prefix))
|
102
|
-
end
|
103
|
-
|
104
|
-
@exclude_broken_fields = @exclude_broken_fields.split(',') if @exclude_broken_fields
|
105
|
-
|
106
102
|
if conf.has_key?('capped')
|
107
103
|
raise ConfigError, "'capped_size' parameter is required on <store> of Mongo output" unless conf.has_key?('capped_size')
|
108
104
|
@collection_options[:capped] = true
|
@@ -110,18 +106,20 @@ module Fluent
|
|
110
106
|
@collection_options[:max] = Config.size_value(conf['capped_max']) if conf.has_key?('capped_max')
|
111
107
|
end
|
112
108
|
|
113
|
-
|
114
|
-
|
115
|
-
|
109
|
+
if remove_tag_prefix = conf['remove_tag_prefix']
|
110
|
+
@remove_tag_prefix = Regexp.new('^' + Regexp.escape(remove_tag_prefix))
|
111
|
+
end
|
116
112
|
|
117
|
-
@
|
113
|
+
@client_options[:write] = {j: @journaled}
|
114
|
+
@client_options[:write].merge!({w: @write_concern}) unless @write_concern.nil?
|
115
|
+
@client_options[:ssl] = @ssl
|
118
116
|
|
119
117
|
if @ssl
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
118
|
+
@client_options[:ssl_cert] = @ssl_cert
|
119
|
+
@client_options[:ssl_key] = @ssl_key
|
120
|
+
@client_options[:ssl_key_pass_phrase] = @ssl_key_pass_phrase
|
121
|
+
@client_options[:ssl_verify] = @ssl_verify
|
122
|
+
@client_options[:ssl_ca_cert] = @ssl_ca_cert
|
125
123
|
end
|
126
124
|
|
127
125
|
# MongoDB uses BSON's Date for time.
|
@@ -129,28 +127,23 @@ module Fluent
|
|
129
127
|
time
|
130
128
|
end
|
131
129
|
|
132
|
-
|
130
|
+
configure_logger(@mongo_log_level)
|
131
|
+
|
132
|
+
log.debug "Setup mongo configuration: mode = #{@tag_mapped ? 'tag mapped' : 'normal'}"
|
133
133
|
end
|
134
134
|
|
135
135
|
def start
|
136
|
-
|
137
|
-
|
138
|
-
|
136
|
+
@client = client
|
137
|
+
@client = authenticate(@client)
|
139
138
|
super
|
140
139
|
end
|
141
140
|
|
142
141
|
def shutdown
|
143
|
-
|
144
|
-
@clients.values.each { |client| client.db.connection.close }
|
142
|
+
@client.close
|
145
143
|
super
|
146
144
|
end
|
147
145
|
|
148
|
-
def format(tag, time, record)
|
149
|
-
[time, record].to_msgpack
|
150
|
-
end
|
151
|
-
|
152
146
|
def emit(tag, es, chain)
|
153
|
-
# TODO: Should replacement using eval in configure?
|
154
147
|
if @tag_mapped
|
155
148
|
super(tag, es, chain, tag)
|
156
149
|
else
|
@@ -158,64 +151,27 @@ module Fluent
|
|
158
151
|
end
|
159
152
|
end
|
160
153
|
|
154
|
+
def format(tag, time, record)
|
155
|
+
[time, record].to_msgpack
|
156
|
+
end
|
157
|
+
|
161
158
|
def write(chunk)
|
162
|
-
# TODO: See emit comment
|
163
159
|
collection_name = @tag_mapped ? chunk.key : @collection
|
164
|
-
operate(
|
160
|
+
operate(format_collection_name(collection_name), collect_records(chunk))
|
165
161
|
end
|
166
162
|
|
167
163
|
private
|
168
164
|
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
if @replace_dot_in_key_with
|
175
|
-
records.map! do |r|
|
176
|
-
replace_key_of_hash(r, ".", @replace_dot_in_key_with)
|
177
|
-
end
|
178
|
-
end
|
179
|
-
if @replace_dollar_in_key_with
|
180
|
-
records.map! do |r|
|
181
|
-
replace_key_of_hash(r, /^\$/, @replace_dollar_in_key_with)
|
182
|
-
end
|
183
|
-
end
|
184
|
-
|
185
|
-
record_ids, error_records = collection.insert(records, INSERT_ARGUMENT)
|
186
|
-
if !@ignore_invalid_record and error_records.size > 0
|
187
|
-
operate_invalid_records(collection, error_records)
|
188
|
-
end
|
189
|
-
rescue Mongo::OperationFailure => e
|
190
|
-
# Probably, all records of _records_ are broken...
|
191
|
-
if e.error_code == 13066 # 13066 means "Message contains no documents"
|
192
|
-
operate_invalid_records(collection, records) unless @ignore_invalid_record
|
193
|
-
else
|
194
|
-
raise e
|
195
|
-
end
|
196
|
-
end
|
197
|
-
records
|
198
|
-
end
|
199
|
-
|
200
|
-
def operate_invalid_records(collection, records)
|
201
|
-
converted_records = records.map { |record|
|
202
|
-
new_record = {}
|
203
|
-
new_record[@tag_key] = record.delete(@tag_key) if @include_tag_key
|
204
|
-
new_record[@time_key] = record.delete(@time_key)
|
205
|
-
if @exclude_broken_fields
|
206
|
-
@exclude_broken_fields.each { |key|
|
207
|
-
new_record[key] = record.delete(key)
|
208
|
-
}
|
209
|
-
end
|
210
|
-
new_record[BROKEN_DATA_KEY] = BSON::Binary.new(Marshal.dump(record))
|
211
|
-
new_record
|
212
|
-
}
|
213
|
-
collection.insert(converted_records)
|
165
|
+
def client
|
166
|
+
@client_options[:database] = @database
|
167
|
+
@client_options[:user] = @user if @user
|
168
|
+
@client_options[:password] = @password if @password
|
169
|
+
Mongo::Client.new(["#{@host}:#{@port}"], @client_options)
|
214
170
|
end
|
215
171
|
|
216
172
|
def collect_records(chunk)
|
217
173
|
records = []
|
218
|
-
chunk.msgpack_each {
|
174
|
+
chunk.msgpack_each {|time, record|
|
219
175
|
record[@time_key] = Time.at(time || record[@time_key]) if @include_time_key
|
220
176
|
records << record
|
221
177
|
}
|
@@ -232,35 +188,26 @@ module Fluent
|
|
232
188
|
formatted
|
233
189
|
end
|
234
190
|
|
235
|
-
def
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
new_mode = format_collection_mode(@collection_options[:capped])
|
246
|
-
old_mode = format_collection_mode(capped)
|
247
|
-
raise ConfigError, "New configuration is different from existing collection: new = #{new_mode}, old = #{old_mode}"
|
191
|
+
def operate(collection, records)
|
192
|
+
begin
|
193
|
+
if @replace_dot_in_key_with
|
194
|
+
records.map! do |r|
|
195
|
+
replace_key_of_hash(r, ".", @replace_dot_in_key_with)
|
196
|
+
end
|
197
|
+
end
|
198
|
+
if @replace_dollar_in_key_with
|
199
|
+
records.map! do |r|
|
200
|
+
replace_key_of_hash(r, /^\$/, @replace_dollar_in_key_with)
|
248
201
|
end
|
249
202
|
end
|
250
|
-
else
|
251
|
-
collection = @db.create_collection(collection_name, @collection_options)
|
252
|
-
end
|
253
|
-
|
254
|
-
@clients[collection_name] = collection
|
255
|
-
end
|
256
|
-
|
257
|
-
def format_collection_mode(mode)
|
258
|
-
mode ? 'capped' : 'normal'
|
259
|
-
end
|
260
203
|
|
261
|
-
|
262
|
-
|
263
|
-
|
204
|
+
@client[collection, @collection_options].insert_many(records)
|
205
|
+
rescue Mongo::Error::BulkWriteError => e
|
206
|
+
log.warn "#{records.size - e.result["n_inserted"]} documents are not inserted. Maybe these documents are invalid as a BSON."
|
207
|
+
rescue ArgumentError => e
|
208
|
+
log.warn e
|
209
|
+
end
|
210
|
+
records
|
264
211
|
end
|
265
212
|
|
266
213
|
def replace_key_of_hash(hash_or_array, pattern, replacement)
|
@@ -4,66 +4,55 @@ module Fluent
|
|
4
4
|
class MongoOutputReplset < MongoOutput
|
5
5
|
Plugin.register_output('mongo_replset', self)
|
6
6
|
|
7
|
+
unless method_defined?(:log)
|
8
|
+
define_method(:log) { $log }
|
9
|
+
end
|
10
|
+
|
7
11
|
config_set_default :include_tag_key, false
|
8
12
|
config_set_default :include_time_key, true
|
9
13
|
|
10
|
-
|
11
|
-
config_param :
|
14
|
+
desc "Replica set name"
|
15
|
+
config_param :replica_set, :string
|
16
|
+
desc "Read from specified role"
|
12
17
|
config_param :read, :string, :default => nil
|
13
|
-
|
14
|
-
config_param :refresh_interval, :integer, :default => nil
|
18
|
+
desc "Retry number"
|
15
19
|
config_param :num_retries, :integer, :default => 60
|
16
20
|
|
17
|
-
|
18
|
-
|
19
|
-
|
21
|
+
unless method_defined?(:log)
|
22
|
+
define_method(:log) { $log }
|
23
|
+
end
|
20
24
|
|
21
25
|
def configure(conf)
|
22
26
|
super
|
23
27
|
|
24
|
-
|
25
|
-
|
26
|
-
@connection_options[:name] = conf['name']
|
28
|
+
if replica_set = conf['replica_set']
|
29
|
+
@client_options[:replica_set] = replica_set
|
27
30
|
end
|
28
31
|
if read = conf['read']
|
29
|
-
@
|
30
|
-
end
|
31
|
-
if refresh_mode = conf['refresh_mode']
|
32
|
-
@connection_options[:refresh_mode] = refresh_mode.to_sym
|
33
|
-
end
|
34
|
-
if refresh_interval = conf['refresh_interval']
|
35
|
-
@connection_options[:refresh_interval] = refresh_interval
|
32
|
+
@client_options[:read] = read.to_sym
|
36
33
|
end
|
37
34
|
|
38
|
-
|
35
|
+
log.debug "Setup replica set configuration: #{conf['replica_set']}"
|
39
36
|
end
|
40
37
|
|
41
38
|
private
|
42
39
|
|
43
|
-
def operate(
|
40
|
+
def operate(client, records)
|
44
41
|
rescue_connection_failure do
|
45
|
-
super(
|
42
|
+
super(client, records)
|
46
43
|
end
|
47
44
|
end
|
48
45
|
|
49
|
-
def parse_nodes(nodes)
|
50
|
-
nodes.split(',')
|
51
|
-
end
|
52
|
-
|
53
|
-
def get_connection
|
54
|
-
db = Mongo::MongoReplicaSetClient.new(@nodes, @connection_options).db(@database)
|
55
|
-
authenticate(db)
|
56
|
-
end
|
57
|
-
|
58
46
|
def rescue_connection_failure
|
59
47
|
retries = 0
|
60
48
|
begin
|
61
49
|
yield
|
62
|
-
rescue Mongo::
|
50
|
+
rescue Mongo::Error::OperationFailure => e
|
63
51
|
retries += 1
|
64
52
|
raise e if retries > @num_retries
|
65
53
|
|
66
|
-
log.warn "Failed to
|
54
|
+
log.warn "Failed to operate to Replica Set. Try to retry: retry count = #{retries}"
|
55
|
+
|
67
56
|
sleep 0.5
|
68
57
|
retry
|
69
58
|
end
|