fluent-plugin-mongo 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 6da21defe0a09c2330001d3c0839d4b2f090aa57
4
- data.tar.gz: 301c0fd93b3f10f7a4fca779e67e14663e46d106
3
+ metadata.gz: 0ad7f90a08c0090846884b95f124df0b6257b420
4
+ data.tar.gz: '094e060574195842fd49c42d5b0fc433c6d16970'
5
5
  SHA512:
6
- metadata.gz: cc0dd8f45510a50c0bcf8f9d31c8f1ecb2ed6be94072106192d22d5e77ea452c0ac9085884f8271d08cdbbdd6c1938eed3c5936dc2d216b26c1fb2d53aa31754
7
- data.tar.gz: 71aeffbf5e5ac00dda3fe4eb89a2f42e85f203d12cad4de5cc092be3cf32fd738da727c099e3a5d2866bacf765875b777ba1bb1de47979e66aa8e9914c53a446
6
+ metadata.gz: 508cf655a0807b681f0e7e0229c510d023dd5f0ffe55c98a80d397e9b92caa97566e31ce7b7250900973fc43c4956e51f84f76b9d0a4d48607d050b0ec3fdb4a
7
+ data.tar.gz: ab536cea004d73000c19dc6d743c224f955d30ab6cfd14f232f046e6cdaba775a2adb446ad70c00b4b6cd341992e91f06e5aac1aadeb7cd02eedd05ca977f170
data/.travis.yml CHANGED
@@ -2,7 +2,8 @@ rvm:
2
2
  - 2.1
3
3
  - 2.2.4
4
4
  - 2.3.1
5
- - 2.4.0
5
+ - 2.4.3
6
+ - 2.5.0
6
7
  - ruby-head
7
8
 
8
9
  gemfile:
@@ -12,7 +13,7 @@ services:
12
13
  - mongodb
13
14
 
14
15
  before_install:
15
- - gem update bundler
16
+ - gem update --system
16
17
  before_script:
17
18
  - git submodule update -i
18
19
 
data/ChangeLog CHANGED
@@ -1,3 +1,9 @@
1
+ Release 1.1.0 - 2018/01/18
2
+
3
+ * in_mongo: Add batch_size parameter
4
+ * out_mongo: Handle collection options correctly
5
+
6
+
1
7
  Release 1.0.0 - 2017/11/26
2
8
 
3
9
  * Use new Plugin API
data/VERSION CHANGED
@@ -1 +1 @@
1
- 1.0.0
1
+ 1.1.0
@@ -44,6 +44,9 @@ module Fluent::Plugin
44
44
  desc "SSL connection"
45
45
  config_param :ssl, :bool, default: false
46
46
 
47
+ desc "Batch size for each find"
48
+ config_param :batch_size, :integer, default: nil
49
+
47
50
  def initialize
48
51
  super
49
52
 
@@ -69,6 +72,10 @@ module Fluent::Plugin
69
72
  @last_id = @id_store_file ? get_last_id : nil
70
73
  @connection_options[:ssl] = @ssl
71
74
 
75
+ if @batch_size && @batch_size <= 0
76
+ raise Fluent::ConfigError, "Batch size must be positive."
77
+ end
78
+
72
79
  configure_logger(@mongo_log_level)
73
80
  end
74
81
 
@@ -99,6 +106,7 @@ module Fluent::Plugin
99
106
  begin
100
107
  option['_id'] = {'$gt' => BSON::ObjectId(@last_id)} if @last_id
101
108
  documents = @collection.find(option)
109
+ documents = documents.limit(@batch_size) if @batch_size
102
110
  if documents.count >= 1
103
111
  process_documents(documents)
104
112
  end
@@ -154,6 +154,7 @@ module Fluent::Plugin
154
154
  def start
155
155
  @client = client
156
156
  @client = authenticate(@client)
157
+ @collections = {}
157
158
  super
158
159
  end
159
160
 
@@ -202,7 +203,7 @@ module Fluent::Plugin
202
203
 
203
204
  def collect_records(chunk)
204
205
  records = []
205
- time_key = @inject_config.time_key
206
+ time_key = @inject_config.time_key if @inject_config
206
207
  tag = chunk.metadata.tag
207
208
  chunk.msgpack_each {|time, record|
208
209
  record = inject_values_to_record(tag, time, record)
@@ -223,6 +224,36 @@ module Fluent::Plugin
223
224
  formatted
224
225
  end
225
226
 
227
+ def list_collections_enabled?
228
+ @client.cluster.next_primary(false).features.list_collections_enabled?
229
+ end
230
+
231
+ def collection_exists?(name)
232
+ if list_collections_enabled?
233
+ r = @client.database.command(
234
+ { :listCollections => 1, :filter => { :name => name } }
235
+ ).first
236
+ r[:ok] && r[:cursor][:firstBatch].size == 1
237
+ else
238
+ @client.database.collection_names.include?(name)
239
+ end
240
+ end
241
+
242
+ def get_collection(name, options)
243
+ return @client[name] if @collections[name]
244
+
245
+ unless collection_exists?(name)
246
+ log.trace "Create collection #{name} with options #{options}"
247
+ @client[name, options].create
248
+ end
249
+ @collections[name] = true
250
+ @client[name]
251
+ end
252
+
253
+ def forget_collection(name)
254
+ @collections.delete(name)
255
+ end
256
+
226
257
  def operate(collection, records)
227
258
  begin
228
259
  if @replace_dot_in_key_with
@@ -236,9 +267,10 @@ module Fluent::Plugin
236
267
  end
237
268
  end
238
269
 
239
- @client[collection, @collection_options].insert_many(records)
270
+ get_collection(collection, @collection_options).insert_many(records)
240
271
  rescue Mongo::Error::BulkWriteError => e
241
272
  log.warn "#{records.size - e.result["n_inserted"]} documents are not inserted. Maybe these documents are invalid as a BSON."
273
+ forget_collection(collection)
242
274
  rescue ArgumentError => e
243
275
  log.warn e
244
276
  end
@@ -93,6 +93,7 @@ class MongoTailInputTest < Test::Unit::TestCase
93
93
  collection #{collection_name}
94
94
  tag input.mongo
95
95
  time_key time
96
+ batch_size 10000
96
97
  ])
97
98
  d.run(expect_records: 1, timeout: 5) do
98
99
  @client[collection_name].insert_one({message: "test"})
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-mongo
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.0
4
+ version: 1.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Masahiro Nakagawa
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-11-27 00:00:00.000000000 Z
11
+ date: 2018-01-19 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd