logstash-output-mongodb_upsert_custom 0.1.0 → 0.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (4) hide show
  1. checksums.yaml +4 -4
  2. data/lib/logstash/outputs/mongodb_upsert_custom.rb +49 -39
  3. data/logstash-output-mongodb_upsert_custom.gemspec +2 -2
  4. metadata +14 -13
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 7f4e52a56cac12d0e591dacb03e3c9968888da377515300e6f80d14858ba98a6
4
- data.tar.gz: deb4b185920556250d6af0455cfa1a464ab7551a41ebd5e085d44b74edebe012
3
+ metadata.gz: 633639345a3e336681a095b7266350293b072e468532023b948a127731cbcddf
4
+ data.tar.gz: 367b8ff4db20259aecae9641d7b2d814d58d651b209b8ad492dee4e58e17e45d
5
5
  SHA512:
6
- metadata.gz: 4c6d965c49a3e955934be9d9aca63c61bb3b63aad6dd2169ec34c252a080b55b735c3feb6df3b12e3f623361ecf8de24745b9f11d5dc8142e10bd88e18c25593
7
- data.tar.gz: 3975b1adc830f4b4a88200b1f2b69d847e3a5976379df73bbf2f6266ed8d2190ad38f3922be631b17453b67fcc8241e99b8b566f2ce4989665be9c692b6aa45d
6
+ metadata.gz: 9495b3174797fec599bd6e4338c9e78aa6a60c42b8dc776995a2b64fecad53e43328015c094829de327204fe50829f9950fc8f4bd374898dcd62414757b78484
7
+ data.tar.gz: b1b3dd86c9efe8e93b458b0b4977aa1917e0ac412d2e6c23d98efe41b9af645c490bfe29bec1ad4cd316fc3716bc6f3f5c92177fc4c330eba3188d94965cd9d8
data/lib/logstash/outputs/mongodb_upsert_custom.rb CHANGED
@@ -45,7 +45,7 @@ class LogStash::Outputs::MongodbUpsertCustom < LogStash::Outputs::Base
45
45
 
46
46
  config :date_keys, :validate => :string, :default => nil
47
47
 
48
- config :is_insert, :validate => :boolean, :default => nil
48
+ config :is_checkpoint, :validate => :boolean, :default => nil
49
49
 
50
50
  config :update_keys, :validate => :string, :default => nil
51
51
 
@@ -56,14 +56,17 @@ class LogStash::Outputs::MongodbUpsertCustom < LogStash::Outputs::Base
56
56
  # Mutex used to synchronize access to 'documents'
57
57
  @@mutex = Mutex.new
58
58
 
59
+ @@log_type = {"OUTPUT" => "output", "INPUT" => "input"}
60
+
59
61
  def register
60
62
  if @bulk_size > 1000
61
63
  raise LogStash::ConfigurationError, "Bulk size must be lower than '1000', currently '#{@bulk_size}'"
62
64
  end
63
65
 
64
66
  Mongo::Logger.logger = @logger
65
- conn = Mongo::Client.new(@uri)
66
- @db = conn.use(@database)
67
+ # conn = Mongo::Client.new(@uri)
68
+ # @db = conn.use(@database)
69
+ @db = Mongo::Client.new(@uri)
67
70
 
68
71
  @closed = Concurrent::AtomicBoolean.new(false)
69
72
  @documents = {}
@@ -74,23 +77,26 @@ class LogStash::Outputs::MongodbUpsertCustom < LogStash::Outputs::Base
74
77
  @@mutex.synchronize do
75
78
  @documents.each do |collection, values|
76
79
  if values.length > 0
77
- if !@is_insert
80
+ # if !@is_checkpoint
78
81
  values.each do |value|
79
82
  criteria = Hash.new
80
83
  criteria_key1 = @filter_key1
81
84
  criteria[criteria_key1] = value[criteria_key1]
82
85
  criteria_key2 = @filter_key2
83
86
  criteria[criteria_key2] = value[criteria_key2]
84
- documentToUpdate = @db[collection].find(criteria)
85
- if documentToUpdate.count() > 0
86
- documentToUpdate.update_many('$set' => value)
87
- else
88
- @db[collection].insert_many([value])
87
+ if @is_checkpoint
88
+ if value['type'] == @@log_type['OUTPUT'] || value['type'] == @@log_type['INPUT']
89
+ if value['time_stamp']
90
+ criteria['time_stamp'] = { '$lte': value["time_stamp"] }
91
+ end
92
+ @db[collection].find(criteria).update_many({"$set": value}, :upsert => :true)
93
+ end
94
+ else
95
+ if value['type'] == @@log_type['OUTPUT']
96
+ @db[collection].find(criteria).update_one(value, :upsert => :true, :new => :true)
97
+ end
89
98
  end
90
99
  end
91
- else
92
- @db[collection].insert_many(values)
93
- end
94
100
  @documents.delete(collection)
95
101
  end
96
102
  end
@@ -133,49 +139,53 @@ class LogStash::Outputs::MongodbUpsertCustom < LogStash::Outputs::Base
133
139
  document["_id"] = BSON::ObjectId.new
134
140
  end
135
141
 
142
+ collection = event.sprintf(@collection)
136
143
  if @bulk
137
- collection = event.sprintf(@collection)
138
144
  @@mutex.synchronize do
139
145
  if(!@documents[collection])
140
146
  @documents[collection] = []
141
147
  end
142
148
  @documents[collection].push(document)
143
149
  if(@documents[collection].length >= @bulk_size)
144
- if !@is_insert
145
- @documents[collection].each do |docRecord|
146
- criteria = Hash.new
147
- criteria_key1 = @filter_key1
148
- criteria[criteria_key1] = docRecord[criteria_key1]
149
- criteria_key2 = @filter_key2
150
- criteria[criteria_key2] = docRecord[criteria_key2]
151
- documentToUpdate = @db[collection].find(criteria)
152
- if documentToUpdate.count() > 0
153
- documentToUpdate.update_many('$set' => docRecord)
154
- else
155
- @db[collection].insert_many([docRecord])
150
+ @documents[collection].each do |docRecord|
151
+ criteria = Hash.new
152
+ criteria_key1 = @filter_key1
153
+ criteria[criteria_key1] = docRecord[criteria_key1]
154
+ criteria_key2 = @filter_key2
155
+ criteria[criteria_key2] = docRecord[criteria_key2]
156
+ if @is_checkpoint
157
+ if docRecord['type'] == @@log_type['OUTPUT'] || docRecord['type'] == @@log_type['INPUT']
158
+ if docRecord['time_stamp']
159
+ criteria['time_stamp'] = { '$lte': docRecord["time_stamp"] }
160
+ end
161
+ @db[collection].find(criteria).update_many({"$set": docRecord}, :upsert => :true)
162
+ end
163
+ else
164
+ if docRecord['type'] == @@log_type['OUTPUT']
165
+ @db[collection].find(criteria).update_one(docRecord, :upsert => :true, :new => :true)
156
166
  end
157
167
  end
158
- else
159
- @db[collection].insert_many(@documents[collection])
160
168
  end
161
169
  @documents.delete(collection)
162
170
  end
163
171
  end
164
172
  else
165
- if !@is_insert
166
- criteria = Hash.new
167
- criteria_key1 = @filter_key1
168
- criteria[criteria_key1] = document[criteria_key1]
169
- criteria_key2 = @filter_key2
170
- criteria[criteria_key2] = document[criteria_key2]
171
- singleDoc = @db[event.sprintf(@collection)].find(criteria)
172
- if singleDoc.count() > 0
173
- singleDoc.update_one('$set' => document)
174
- else
175
- @db[event.sprintf(@collection)].insert_one(document)
173
+ criteria = Hash.new
174
+ criteria_key1 = @filter_key1
175
+ criteria[criteria_key1] = document[criteria_key1]
176
+ criteria_key2 = @filter_key2
177
+ criteria[criteria_key2] = document[criteria_key2]
178
+ if @is_checkpoint
179
+ if document['type'] == @@log_type['OUTPUT'] || document['type'] == @@log_type['INPUT']
180
+ if document['time_stamp']
181
+ criteria['time_stamp'] = { '$lte': document["time_stamp"] }
182
+ end
183
+ @db[collection].find(criteria).update_many({"$set": document}, :upsert => :true)
176
184
  end
177
185
  else
178
- @db[event.sprintf(@collection)].insert_one(document)
186
+ if document['type'] == @@log_type['OUTPUT']
187
+ @db[collection].find(criteria).update_one(document, :upsert => :true, :new => :true)
188
+ end
179
189
  end
180
190
  end
181
191
  rescue => e
data/logstash-output-mongodb_upsert_custom.gemspec CHANGED
@@ -1,12 +1,12 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-output-mongodb_upsert_custom'
3
- s.version = '0.1.0'
3
+ s.version = '0.2.0'
4
4
  s.licenses = ['Apache-2.0']
5
5
  s.summary = 'Plugin to write to mongo db.'
6
6
  s.description = 'This plugin is to write the logstash input to given mongo db.'
7
7
  s.homepage = 'http://www.elastic.co/guide/en/logstash/current/index.html'
8
8
  s.authors = ['jijk']
9
- s.email = 'jiji.k@softwareag.com'
9
+ s.email = 'ravi.shah@softwareag.com'
10
10
  s.require_paths = ['lib']
11
11
 
12
12
  # Files
metadata CHANGED
@@ -1,59 +1,59 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-mongodb_upsert_custom
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - jijk
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-03-26 00:00:00.000000000 Z
11
+ date: 2020-08-06 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- name: logstash-core-plugin-api
15
14
  requirement: !ruby/object:Gem::Requirement
16
15
  requirements:
17
16
  - - "~>"
18
17
  - !ruby/object:Gem::Version
19
18
  version: '2.0'
20
- type: :runtime
19
+ name: logstash-core-plugin-api
21
20
  prerelease: false
21
+ type: :runtime
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
26
  version: '2.0'
27
27
  - !ruby/object:Gem::Dependency
28
- name: logstash-codec-plain
29
28
  requirement: !ruby/object:Gem::Requirement
30
29
  requirements:
31
30
  - - ">="
32
31
  - !ruby/object:Gem::Version
33
32
  version: '0'
34
- type: :runtime
33
+ name: logstash-codec-plain
35
34
  prerelease: false
35
+ type: :runtime
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - ">="
39
39
  - !ruby/object:Gem::Version
40
40
  version: '0'
41
41
  - !ruby/object:Gem::Dependency
42
- name: logstash-devutils
43
42
  requirement: !ruby/object:Gem::Requirement
44
43
  requirements:
45
44
  - - ">="
46
45
  - !ruby/object:Gem::Version
47
46
  version: '0'
48
- type: :development
47
+ name: logstash-devutils
49
48
  prerelease: false
49
+ type: :development
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
52
  - - ">="
53
53
  - !ruby/object:Gem::Version
54
54
  version: '0'
55
55
  description: This plugin is to write the logstash input to given mongo db.
56
- email: jiji.k@softwareag.com
56
+ email: ravi.shah@softwareag.com
57
57
  executables: []
58
58
  extensions: []
59
59
  extra_rdoc_files: []
@@ -76,7 +76,7 @@ licenses:
76
76
  metadata:
77
77
  logstash_plugin: 'true'
78
78
  logstash_group: output
79
- post_install_message:
79
+ post_install_message:
80
80
  rdoc_options: []
81
81
  require_paths:
82
82
  - lib
@@ -91,8 +91,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
91
91
  - !ruby/object:Gem::Version
92
92
  version: '0'
93
93
  requirements: []
94
- rubygems_version: 3.0.3
95
- signing_key:
94
+ rubyforge_project:
95
+ rubygems_version: 2.7.10
96
+ signing_key:
96
97
  specification_version: 4
97
98
  summary: Plugin to write to mongo db.
98
99
  test_files: