logstash-output-mongodb_upsert_custom 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. checksums.yaml +4 -4
  2. data/lib/logstash/outputs/mongodb_upsert_custom.rb +49 -39
  3. data/logstash-output-mongodb_upsert_custom.gemspec +2 -2
  4. metadata +14 -13
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 7f4e52a56cac12d0e591dacb03e3c9968888da377515300e6f80d14858ba98a6
4
- data.tar.gz: deb4b185920556250d6af0455cfa1a464ab7551a41ebd5e085d44b74edebe012
3
+ metadata.gz: 633639345a3e336681a095b7266350293b072e468532023b948a127731cbcddf
4
+ data.tar.gz: 367b8ff4db20259aecae9641d7b2d814d58d651b209b8ad492dee4e58e17e45d
5
5
  SHA512:
6
- metadata.gz: 4c6d965c49a3e955934be9d9aca63c61bb3b63aad6dd2169ec34c252a080b55b735c3feb6df3b12e3f623361ecf8de24745b9f11d5dc8142e10bd88e18c25593
7
- data.tar.gz: 3975b1adc830f4b4a88200b1f2b69d847e3a5976379df73bbf2f6266ed8d2190ad38f3922be631b17453b67fcc8241e99b8b566f2ce4989665be9c692b6aa45d
6
+ metadata.gz: 9495b3174797fec599bd6e4338c9e78aa6a60c42b8dc776995a2b64fecad53e43328015c094829de327204fe50829f9950fc8f4bd374898dcd62414757b78484
7
+ data.tar.gz: b1b3dd86c9efe8e93b458b0b4977aa1917e0ac412d2e6c23d98efe41b9af645c490bfe29bec1ad4cd316fc3716bc6f3f5c92177fc4c330eba3188d94965cd9d8
data/lib/logstash/outputs/mongodb_upsert_custom.rb CHANGED
@@ -45,7 +45,7 @@ class LogStash::Outputs::MongodbUpsertCustom < LogStash::Outputs::Base
45
45
 
46
46
  config :date_keys, :validate => :string, :default => nil
47
47
 
48
- config :is_insert, :validate => :boolean, :default => nil
48
+ config :is_checkpoint, :validate => :boolean, :default => nil
49
49
 
50
50
  config :update_keys, :validate => :string, :default => nil
51
51
 
@@ -56,14 +56,17 @@ class LogStash::Outputs::MongodbUpsertCustom < LogStash::Outputs::Base
56
56
  # Mutex used to synchronize access to 'documents'
57
57
  @@mutex = Mutex.new
58
58
 
59
+ @@log_type = {"OUTPUT" => "output", "INPUT" => "input"}
60
+
59
61
  def register
60
62
  if @bulk_size > 1000
61
63
  raise LogStash::ConfigurationError, "Bulk size must be lower than '1000', currently '#{@bulk_size}'"
62
64
  end
63
65
 
64
66
  Mongo::Logger.logger = @logger
65
- conn = Mongo::Client.new(@uri)
66
- @db = conn.use(@database)
67
+ # conn = Mongo::Client.new(@uri)
68
+ # @db = conn.use(@database)
69
+ @db = Mongo::Client.new(@uri)
67
70
 
68
71
  @closed = Concurrent::AtomicBoolean.new(false)
69
72
  @documents = {}
@@ -74,23 +77,26 @@ class LogStash::Outputs::MongodbUpsertCustom < LogStash::Outputs::Base
74
77
  @@mutex.synchronize do
75
78
  @documents.each do |collection, values|
76
79
  if values.length > 0
77
- if !@is_insert
80
+ # if !@is_checkpoint
78
81
  values.each do |value|
79
82
  criteria = Hash.new
80
83
  criteria_key1 = @filter_key1
81
84
  criteria[criteria_key1] = value[criteria_key1]
82
85
  criteria_key2 = @filter_key2
83
86
  criteria[criteria_key2] = value[criteria_key2]
84
- documentToUpdate = @db[collection].find(criteria)
85
- if documentToUpdate.count() > 0
86
- documentToUpdate.update_many('$set' => value)
87
- else
88
- @db[collection].insert_many([value])
87
+ if @is_checkpoint
88
+ if value['type'] == @@log_type['OUTPUT'] || value['type'] == @@log_type['INPUT']
89
+ if value['time_stamp']
90
+ criteria['time_stamp'] = { '$lte': value["time_stamp"] }
91
+ end
92
+ @db[collection].find(criteria).update_many({"$set": value}, :upsert => :true)
93
+ end
94
+ else
95
+ if value['type'] == @@log_type['OUTPUT']
96
+ @db[collection].find(criteria).update_one(value, :upsert => :true, :new => :true)
97
+ end
89
98
  end
90
99
  end
91
- else
92
- @db[collection].insert_many(values)
93
- end
94
100
  @documents.delete(collection)
95
101
  end
96
102
  end
@@ -133,49 +139,53 @@ class LogStash::Outputs::MongodbUpsertCustom < LogStash::Outputs::Base
133
139
  document["_id"] = BSON::ObjectId.new
134
140
  end
135
141
 
142
+ collection = event.sprintf(@collection)
136
143
  if @bulk
137
- collection = event.sprintf(@collection)
138
144
  @@mutex.synchronize do
139
145
  if(!@documents[collection])
140
146
  @documents[collection] = []
141
147
  end
142
148
  @documents[collection].push(document)
143
149
  if(@documents[collection].length >= @bulk_size)
144
- if !@is_insert
145
- @documents[collection].each do |docRecord|
146
- criteria = Hash.new
147
- criteria_key1 = @filter_key1
148
- criteria[criteria_key1] = docRecord[criteria_key1]
149
- criteria_key2 = @filter_key2
150
- criteria[criteria_key2] = docRecord[criteria_key2]
151
- documentToUpdate = @db[collection].find(criteria)
152
- if documentToUpdate.count() > 0
153
- documentToUpdate.update_many('$set' => docRecord)
154
- else
155
- @db[collection].insert_many([docRecord])
150
+ @documents[collection].each do |docRecord|
151
+ criteria = Hash.new
152
+ criteria_key1 = @filter_key1
153
+ criteria[criteria_key1] = docRecord[criteria_key1]
154
+ criteria_key2 = @filter_key2
155
+ criteria[criteria_key2] = docRecord[criteria_key2]
156
+ if @is_checkpoint
157
+ if docRecord['type'] == @@log_type['OUTPUT'] || docRecord['type'] == @@log_type['INPUT']
158
+ if docRecord['time_stamp']
159
+ criteria['time_stamp'] = { '$lte': docRecord["time_stamp"] }
160
+ end
161
+ @db[collection].find(criteria).update_many({"$set": docRecord}, :upsert => :true)
162
+ end
163
+ else
164
+ if docRecord['type'] == @@log_type['OUTPUT']
165
+ @db[collection].find(criteria).update_one(docRecord, :upsert => :true, :new => :true)
156
166
  end
157
167
  end
158
- else
159
- @db[collection].insert_many(@documents[collection])
160
168
  end
161
169
  @documents.delete(collection)
162
170
  end
163
171
  end
164
172
  else
165
- if !@is_insert
166
- criteria = Hash.new
167
- criteria_key1 = @filter_key1
168
- criteria[criteria_key1] = document[criteria_key1]
169
- criteria_key2 = @filter_key2
170
- criteria[criteria_key2] = document[criteria_key2]
171
- singleDoc = @db[event.sprintf(@collection)].find(criteria)
172
- if singleDoc.count() > 0
173
- singleDoc.update_one('$set' => document)
174
- else
175
- @db[event.sprintf(@collection)].insert_one(document)
173
+ criteria = Hash.new
174
+ criteria_key1 = @filter_key1
175
+ criteria[criteria_key1] = document[criteria_key1]
176
+ criteria_key2 = @filter_key2
177
+ criteria[criteria_key2] = document[criteria_key2]
178
+ if @is_checkpoint
179
+ if document['type'] == @@log_type['OUTPUT'] || document['type'] == @@log_type['INPUT']
180
+ if document['time_stamp']
181
+ criteria['time_stamp'] = { '$lte': document["time_stamp"] }
182
+ end
183
+ @db[collection].find(criteria).update_many({"$set": document}, :upsert => :true)
176
184
  end
177
185
  else
178
- @db[event.sprintf(@collection)].insert_one(document)
186
+ if document['type'] == @@log_type['OUTPUT']
187
+ @db[collection].find(criteria).update_one(document, :upsert => :true, :new => :true)
188
+ end
179
189
  end
180
190
  end
181
191
  rescue => e
data/logstash-output-mongodb_upsert_custom.gemspec CHANGED
@@ -1,12 +1,12 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-output-mongodb_upsert_custom'
3
- s.version = '0.1.0'
3
+ s.version = '0.2.0'
4
4
  s.licenses = ['Apache-2.0']
5
5
  s.summary = 'Plugin to write to mongo db.'
6
6
  s.description = 'This plugin is to write the logstash input to given mongo db.'
7
7
  s.homepage = 'http://www.elastic.co/guide/en/logstash/current/index.html'
8
8
  s.authors = ['jijk']
9
- s.email = 'jiji.k@softwareag.com'
9
+ s.email = 'ravi.shah@softwareag.com'
10
10
  s.require_paths = ['lib']
11
11
 
12
12
  # Files
metadata CHANGED
@@ -1,59 +1,59 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-mongodb_upsert_custom
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - jijk
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-03-26 00:00:00.000000000 Z
11
+ date: 2020-08-06 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- name: logstash-core-plugin-api
15
14
  requirement: !ruby/object:Gem::Requirement
16
15
  requirements:
17
16
  - - "~>"
18
17
  - !ruby/object:Gem::Version
19
18
  version: '2.0'
20
- type: :runtime
19
+ name: logstash-core-plugin-api
21
20
  prerelease: false
21
+ type: :runtime
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
26
  version: '2.0'
27
27
  - !ruby/object:Gem::Dependency
28
- name: logstash-codec-plain
29
28
  requirement: !ruby/object:Gem::Requirement
30
29
  requirements:
31
30
  - - ">="
32
31
  - !ruby/object:Gem::Version
33
32
  version: '0'
34
- type: :runtime
33
+ name: logstash-codec-plain
35
34
  prerelease: false
35
+ type: :runtime
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - ">="
39
39
  - !ruby/object:Gem::Version
40
40
  version: '0'
41
41
  - !ruby/object:Gem::Dependency
42
- name: logstash-devutils
43
42
  requirement: !ruby/object:Gem::Requirement
44
43
  requirements:
45
44
  - - ">="
46
45
  - !ruby/object:Gem::Version
47
46
  version: '0'
48
- type: :development
47
+ name: logstash-devutils
49
48
  prerelease: false
49
+ type: :development
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
52
  - - ">="
53
53
  - !ruby/object:Gem::Version
54
54
  version: '0'
55
55
  description: This plugin is to write the logstash input to given mongo db.
56
- email: jiji.k@softwareag.com
56
+ email: ravi.shah@softwareag.com
57
57
  executables: []
58
58
  extensions: []
59
59
  extra_rdoc_files: []
@@ -76,7 +76,7 @@ licenses:
76
76
  metadata:
77
77
  logstash_plugin: 'true'
78
78
  logstash_group: output
79
- post_install_message:
79
+ post_install_message:
80
80
  rdoc_options: []
81
81
  require_paths:
82
82
  - lib
@@ -91,8 +91,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
91
91
  - !ruby/object:Gem::Version
92
92
  version: '0'
93
93
  requirements: []
94
- rubygems_version: 3.0.3
95
- signing_key:
94
+ rubyforge_project:
95
+ rubygems_version: 2.7.10
96
+ signing_key:
96
97
  specification_version: 4
97
98
  summary: Plugin to write to mongo db.
98
99
  test_files: