fluent-plugin-mongo 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +3 -2
- data/ChangeLog +6 -0
- data/VERSION +1 -1
- data/lib/fluent/plugin/in_mongo_tail.rb +8 -0
- data/lib/fluent/plugin/out_mongo.rb +34 -2
- data/test/plugin/test_in_mongo_tail.rb +1 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0ad7f90a08c0090846884b95f124df0b6257b420
|
4
|
+
data.tar.gz: '094e060574195842fd49c42d5b0fc433c6d16970'
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 508cf655a0807b681f0e7e0229c510d023dd5f0ffe55c98a80d397e9b92caa97566e31ce7b7250900973fc43c4956e51f84f76b9d0a4d48607d050b0ec3fdb4a
|
7
|
+
data.tar.gz: ab536cea004d73000c19dc6d743c224f955d30ab6cfd14f232f046e6cdaba775a2adb446ad70c00b4b6cd341992e91f06e5aac1aadeb7cd02eedd05ca977f170
|
data/.travis.yml
CHANGED
@@ -2,7 +2,8 @@ rvm:
|
|
2
2
|
- 2.1
|
3
3
|
- 2.2.4
|
4
4
|
- 2.3.1
|
5
|
-
- 2.4.
|
5
|
+
- 2.4.3
|
6
|
+
- 2.5.0
|
6
7
|
- ruby-head
|
7
8
|
|
8
9
|
gemfile:
|
@@ -12,7 +13,7 @@ services:
|
|
12
13
|
- mongodb
|
13
14
|
|
14
15
|
before_install:
|
15
|
-
- gem update
|
16
|
+
- gem update --system
|
16
17
|
before_script:
|
17
18
|
- git submodule update -i
|
18
19
|
|
data/ChangeLog
CHANGED
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
1.
|
1
|
+
1.1.0
|
@@ -44,6 +44,9 @@ module Fluent::Plugin
|
|
44
44
|
desc "SSL connection"
|
45
45
|
config_param :ssl, :bool, default: false
|
46
46
|
|
47
|
+
desc "Batch size for each find"
|
48
|
+
config_param :batch_size, :integer, default: nil
|
49
|
+
|
47
50
|
def initialize
|
48
51
|
super
|
49
52
|
|
@@ -69,6 +72,10 @@ module Fluent::Plugin
|
|
69
72
|
@last_id = @id_store_file ? get_last_id : nil
|
70
73
|
@connection_options[:ssl] = @ssl
|
71
74
|
|
75
|
+
if @batch_size && @batch_size <= 0
|
76
|
+
raise Fluent::ConfigError, "Batch size must be positive."
|
77
|
+
end
|
78
|
+
|
72
79
|
configure_logger(@mongo_log_level)
|
73
80
|
end
|
74
81
|
|
@@ -99,6 +106,7 @@ module Fluent::Plugin
|
|
99
106
|
begin
|
100
107
|
option['_id'] = {'$gt' => BSON::ObjectId(@last_id)} if @last_id
|
101
108
|
documents = @collection.find(option)
|
109
|
+
documents = documents.limit(@batch_size) if @batch_size
|
102
110
|
if documents.count >= 1
|
103
111
|
process_documents(documents)
|
104
112
|
end
|
@@ -154,6 +154,7 @@ module Fluent::Plugin
|
|
154
154
|
def start
|
155
155
|
@client = client
|
156
156
|
@client = authenticate(@client)
|
157
|
+
@collections = {}
|
157
158
|
super
|
158
159
|
end
|
159
160
|
|
@@ -202,7 +203,7 @@ module Fluent::Plugin
|
|
202
203
|
|
203
204
|
def collect_records(chunk)
|
204
205
|
records = []
|
205
|
-
time_key = @inject_config.time_key
|
206
|
+
time_key = @inject_config.time_key if @inject_config
|
206
207
|
tag = chunk.metadata.tag
|
207
208
|
chunk.msgpack_each {|time, record|
|
208
209
|
record = inject_values_to_record(tag, time, record)
|
@@ -223,6 +224,36 @@ module Fluent::Plugin
|
|
223
224
|
formatted
|
224
225
|
end
|
225
226
|
|
227
|
+
def list_collections_enabled?
|
228
|
+
@client.cluster.next_primary(false).features.list_collections_enabled?
|
229
|
+
end
|
230
|
+
|
231
|
+
def collection_exists?(name)
|
232
|
+
if list_collections_enabled?
|
233
|
+
r = @client.database.command(
|
234
|
+
{ :listCollections => 1, :filter => { :name => name } }
|
235
|
+
).first
|
236
|
+
r[:ok] && r[:cursor][:firstBatch].size == 1
|
237
|
+
else
|
238
|
+
@client.database.collection_names.include?(name)
|
239
|
+
end
|
240
|
+
end
|
241
|
+
|
242
|
+
def get_collection(name, options)
|
243
|
+
return @client[name] if @collections[name]
|
244
|
+
|
245
|
+
unless collection_exists?(name)
|
246
|
+
log.trace "Create collection #{name} with options #{options}"
|
247
|
+
@client[name, options].create
|
248
|
+
end
|
249
|
+
@collections[name] = true
|
250
|
+
@client[name]
|
251
|
+
end
|
252
|
+
|
253
|
+
def forget_collection(name)
|
254
|
+
@collections.delete(name)
|
255
|
+
end
|
256
|
+
|
226
257
|
def operate(collection, records)
|
227
258
|
begin
|
228
259
|
if @replace_dot_in_key_with
|
@@ -236,9 +267,10 @@ module Fluent::Plugin
|
|
236
267
|
end
|
237
268
|
end
|
238
269
|
|
239
|
-
|
270
|
+
get_collection(collection, @collection_options).insert_many(records)
|
240
271
|
rescue Mongo::Error::BulkWriteError => e
|
241
272
|
log.warn "#{records.size - e.result["n_inserted"]} documents are not inserted. Maybe these documents are invalid as a BSON."
|
273
|
+
forget_collection(collection)
|
242
274
|
rescue ArgumentError => e
|
243
275
|
log.warn e
|
244
276
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-mongo
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Masahiro Nakagawa
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2018-01-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|