microsoft-sentinel-log-analytics-logstash-output-plugin 1.1.4 → 2.2.1-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,116 +1,7 @@
1
- # encoding: utf-8
2
- require "logstash/outputs/base"
3
- require "logstash/namespace"
4
- require "logstash/sentinel_la/logstashLoganalyticsConfiguration"
5
- require "logstash/sentinel_la/sampleFileCreator"
6
- require "logstash/sentinel_la/logsSender"
7
-
8
-
9
- class LogStash::Outputs::MicrosoftSentinelOutput < LogStash::Outputs::Base
10
-
11
- config_name "microsoft-sentinel-log-analytics-logstash-output-plugin"
12
-
13
- # Stating that the output plugin will run in concurrent mode
14
- concurrency :shared
15
-
16
- # Your registered app ID
17
- config :client_app_Id, :validate => :string
18
-
19
- # The registered app's secret, required by Azure Loganalytics REST API
20
- config :client_app_secret, :validate => :string
21
-
22
- # Your Operations Management Suite Tenant ID
23
- config :tenant_id, :validate => :string
24
-
25
- # Your data collection rule endpoint
26
- config :data_collection_endpoint, :validate => :string
27
-
28
- # Your data collection rule ID
29
- config :dcr_immutable_id, :validate => :string
30
-
31
- # Your dcr data stream name
32
- config :dcr_stream_name, :validate => :string
33
-
34
- # Subset of keys to send to the Azure Loganalytics workspace
35
- config :key_names, :validate => :array, :default => []
36
-
37
- # Max number of seconds to wait between flushes. Default 5
38
- config :plugin_flush_interval, :validate => :number, :default => 5
39
-
40
- # Factor for adding to the amount of messages sent
41
- config :decrease_factor, :validate => :number, :default => 100
42
-
43
- # This will trigger message amount resizing in a REST request to LA
44
- config :amount_resizing, :validate => :boolean, :default => true
45
-
46
- # Setting the default amount of messages sent
47
- # it this is set with amount_resizing=false --> each message will have max_items
48
- config :max_items, :validate => :number, :default => 2000
49
-
50
- # Setting default proxy to be used for all communication with azure
51
- config :proxy, :validate => :string
52
-
53
- # Setting proxy_aad to be used for communicating with azure active directory service
54
- config :proxy_aad, :validate => :string
55
-
56
- # Setting proxy to be used for the LogAnalytics endpoint REST client
57
- config :proxy_endpoint, :validate => :string
58
-
59
- # This will set the amount of time given for retransmitting messages once sending is failed
60
- config :retransmission_time, :validate => :number, :default => 10
61
-
62
- # Compress the message body before sending to LA
63
- config :compress_data, :validate => :boolean, :default => false
64
-
65
- # Generate sample file from incoming events
66
- config :create_sample_file, :validate => :boolean, :default => false
67
-
68
- # Path where to place the sample file created
69
- config :sample_file_path, :validate => :string
70
-
71
- # Used to specify the name of the Azure cloud that is being used. By default, the value is set to "AzureCloud", which
72
- # is the public Azure cloud. However, you can specify a different Azure cloud if you are
73
- # using a different environment, such as Azure Government or Azure China.
74
- config :azure_cloud, :validate => :string
75
-
76
- public
77
- def register
78
- @logstash_configuration= build_logstash_configuration()
79
-
80
- # Validate configuration correctness
81
- @logstash_configuration.validate_configuration()
82
-
83
- @events_handler = @logstash_configuration.create_sample_file ?
84
- LogStash::Outputs::MicrosoftSentinelOutputInternal::SampleFileCreator::new(@logstash_configuration) :
85
- LogStash::Outputs::MicrosoftSentinelOutputInternal::LogsSender::new(@logstash_configuration)
86
- end # def register
87
-
88
- def multi_receive(events)
89
- @events_handler.handle_events(events)
90
- end # def multi_receive
91
-
92
- def close
93
- @events_handler.close
94
- end
95
-
96
- #private
97
- private
98
-
99
- # Building the logstash object configuration from the output configuration provided by the user
100
- # Return LogstashLoganalyticsOutputConfiguration populated with the configuration values
101
- def build_logstash_configuration()
102
- logstash_configuration= LogStash::Outputs::MicrosoftSentinelOutputInternal::LogstashLoganalyticsOutputConfiguration::new(@client_app_Id, @client_app_secret, @tenant_id, @data_collection_endpoint, @dcr_immutable_id, @dcr_stream_name, @compress_data, @create_sample_file, @sample_file_path, @logger)
103
- logstash_configuration.key_names = @key_names
104
- logstash_configuration.plugin_flush_interval = @plugin_flush_interval
105
- logstash_configuration.decrease_factor = @decrease_factor
106
- logstash_configuration.amount_resizing = @amount_resizing
107
- logstash_configuration.max_items = @max_items
108
- logstash_configuration.proxy_aad = @proxy_aad || @proxy || ENV['http_proxy']
109
- logstash_configuration.proxy_endpoint = @proxy_endpoint || @proxy || ENV['http_proxy']
110
- logstash_configuration.retransmission_time = @retransmission_time
111
- logstash_configuration.azure_cloud = @azure_cloud || "AzureCloud"
112
-
113
- return logstash_configuration
114
- end # def build_logstash_configuration
115
-
116
- end # class LogStash::Outputs::MicrosoftSentinelOutput
1
+ # AUTOGENERATED BY THE GRADLE SCRIPT. EDITS WILL BE OVERWRITTEN.
2
+ # encoding: utf-8
3
+
4
+ # NOTE: Java plugin (no Ruby plugin class). This file exists only to ensure
5
+ # jar-dependencies loads the packaged jar into JRuby's classpath.
6
+ require "logstash-output-microsoft-sentinel-log-analytics-logstash-output-plugin_jars"
7
+ require "java"
@@ -0,0 +1,5 @@
1
+ # AUTOGENERATED BY THE GRADLE SCRIPT. EDITS WILL BE OVERWRITTEN.
2
+ # encoding: utf-8
3
+
4
+ require 'jar_dependencies'
5
+ require_jar('org.logstashplugins', 'logstash-output-microsoft-sentinel-log-analytics-logstash-output-plugin', '2.2.1')
@@ -0,0 +1,4 @@
1
+ # AUTOGENERATED BY THE GRADLE SCRIPT. EDITS WILL BE OVERWRITTEN.
2
+ # encoding: utf-8
3
+ require "logstash-output-microsoft-sentinel-log-analytics-logstash-output-plugin_jars"
4
+ require "java"
@@ -0,0 +1,22 @@
1
+ # AUTOGENERATED BY THE GRADLE SCRIPT. EDITS WILL BE OVERWRITTEN.
2
+ Gem::Specification.new do |s|
3
+ s.name = 'microsoft-sentinel-log-analytics-logstash-output-plugin'
4
+ s.version = '2.2.1'
5
+ s.licenses = ['Apache-2.0']
6
+ s.summary = 'Microsoft Sentinel Log Analytics output plugin'
7
+ s.description = 'Microsoft Sentinel provides a new output plugin for Logstash. Use this output plugin to send any log via Logstash to the Microsoft Sentinel/Log Analytics workspace. This is done with the Log Analytics DCR-based API.'
8
+ s.authors = ['Microsoft']
9
+ s.email = ['msg-ucp@service.microsoft.com']
10
+ s.platform = 'java'
11
+ s.require_paths = ['lib', 'vendor/jar-dependencies']
12
+
13
+ s.files = Dir["lib/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
14
+
15
+ # Special flag to let us know this is actually a logstash plugin
16
+ s.metadata = { 'logstash_plugin' => 'true', 'logstash_group' => 'output', 'java_plugin' => 'true'}
17
+
18
+ # Gem dependencies
19
+ s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
20
+ s.add_runtime_dependency 'jar-dependencies'
21
+ s.add_development_dependency 'logstash-devutils'
22
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: microsoft-sentinel-log-analytics-logstash-output-plugin
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.4
5
- platform: ruby
4
+ version: 2.2.1
5
+ platform: java
6
6
  authors:
7
- - Microsoft Sentinel
7
+ - Microsoft
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2024-12-10 00:00:00.000000000 Z
11
+ date: 2026-05-27 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -36,7 +36,7 @@ dependencies:
36
36
  - - ">="
37
37
  - !ruby/object:Gem::Version
38
38
  version: '0'
39
- name: logstash-codec-plain
39
+ name: jar-dependencies
40
40
  type: :runtime
41
41
  prerelease: false
42
42
  version_requirements: !ruby/object:Gem::Requirement
@@ -44,26 +44,6 @@ dependencies:
44
44
  - - ">="
45
45
  - !ruby/object:Gem::Version
46
46
  version: '0'
47
- - !ruby/object:Gem::Dependency
48
- requirement: !ruby/object:Gem::Requirement
49
- requirements:
50
- - - ">="
51
- - !ruby/object:Gem::Version
52
- version: 0.88.0
53
- - - "<"
54
- - !ruby/object:Gem::Version
55
- version: 1.0.0
56
- name: excon
57
- type: :runtime
58
- prerelease: false
59
- version_requirements: !ruby/object:Gem::Requirement
60
- requirements:
61
- - - ">="
62
- - !ruby/object:Gem::Version
63
- version: 0.88.0
64
- - - "<"
65
- - !ruby/object:Gem::Version
66
- version: 1.0.0
67
47
  - !ruby/object:Gem::Dependency
68
48
  requirement: !ruby/object:Gem::Requirement
69
49
  requirements:
@@ -81,38 +61,33 @@ dependencies:
81
61
  description: Microsoft Sentinel provides a new output plugin for Logstash. Use this
82
62
  output plugin to send any log via Logstash to the Microsoft Sentinel/Log Analytics
83
63
  workspace. This is done with the Log Analytics DCR-based API.
84
- email: AzureSentinel@microsoft.com
64
+ email:
65
+ - msg-ucp@service.microsoft.com
85
66
  executables: []
86
67
  extensions: []
87
68
  extra_rdoc_files: []
88
69
  files:
89
70
  - CHANGELOG.md
90
71
  - Gemfile
91
- - LICENSE
92
72
  - README.md
73
+ - VERSION
74
+ - lib/logstash-output-microsoft-sentinel-log-analytics-logstash-output-plugin_jars.rb
93
75
  - lib/logstash/outputs/microsoft-sentinel-log-analytics-logstash-output-plugin.rb
94
- - lib/logstash/sentinel_la/customSizeBasedBuffer.rb
95
- - lib/logstash/sentinel_la/eventsHandler.rb
96
- - lib/logstash/sentinel_la/logAnalyticsAadTokenProvider.rb
97
- - lib/logstash/sentinel_la/logAnalyticsClient.rb
98
- - lib/logstash/sentinel_la/logStashAutoResizeBuffer.rb
99
- - lib/logstash/sentinel_la/logStashCompressedStream.rb
100
- - lib/logstash/sentinel_la/logStashEventsBatcher.rb
101
- - lib/logstash/sentinel_la/logsSender.rb
102
- - lib/logstash/sentinel_la/logstashLoganalyticsConfiguration.rb
103
- - lib/logstash/sentinel_la/sampleFileCreator.rb
104
- - lib/logstash/sentinel_la/version.rb
105
- - microsoft-sentinel-log-analytics-logstash-output-plugin.gemspec
106
- homepage: https://github.com/Azure/Azure-Sentinel
76
+ - lib/logstash_registry.rb
77
+ - logstash-output-microsoft-sentinel-log-analytics-logstash-output-plugin.gemspec
78
+ - vendor/jar-dependencies/org/logstashplugins/logstash-output-microsoft-sentinel-log-analytics-logstash-output-plugin/2.2.1/logstash-output-microsoft-sentinel-log-analytics-logstash-output-plugin-2.2.1.jar
79
+ homepage:
107
80
  licenses:
108
- - MIT
81
+ - Apache-2.0
109
82
  metadata:
110
83
  logstash_plugin: 'true'
111
84
  logstash_group: output
85
+ java_plugin: 'true'
112
86
  post_install_message:
113
87
  rdoc_options: []
114
88
  require_paths:
115
89
  - lib
90
+ - vendor/jar-dependencies
116
91
  required_ruby_version: !ruby/object:Gem::Requirement
117
92
  requirements:
118
93
  - - ">="
@@ -127,7 +102,5 @@ requirements: []
127
102
  rubygems_version: 3.3.26
128
103
  signing_key:
129
104
  specification_version: 4
130
- summary: Microsoft Sentinel provides a new output plugin for Logstash. Use this output
131
- plugin to send any log via Logstash to the Microsoft Sentinel/Log Analytics workspace.
132
- This is done with the Log Analytics DCR-based API.
105
+ summary: Microsoft Sentinel Log Analytics output plugin
133
106
  test_files: []
data/LICENSE DELETED
@@ -1,21 +0,0 @@
1
- MIT License
2
-
3
- Copyright (c) Microsoft Corporation. All rights reserved.
4
-
5
- Permission is hereby granted, free of charge, to any person obtaining a copy
6
- of this software and associated documentation files (the "Software"), to deal
7
- in the Software without restriction, including without limitation the rights
8
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
- copies of the Software, and to permit persons to whom the Software is
10
- furnished to do so, subject to the following conditions:
11
-
12
- The above copyright notice and this permission notice shall be included in all
13
- copies or substantial portions of the Software.
14
-
15
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
- SOFTWARE
@@ -1,293 +0,0 @@
1
- # This code is from a PR for the official repo of ruby-stud
2
- # with a small change to calculating the event size in the var_size function
3
- # https://github.com/jordansissel/ruby-stud/pull/19
4
- #
5
- # @author {Alex Dean}[http://github.com/alexdean]
6
- #
7
- # Implements a generic framework for accepting events which are later flushed
8
- # in batches. Flushing occurs whenever +:max_items+ or +:max_interval+ (seconds)
9
- # has been reached or if the event size outgrows +:flush_each+ (bytes)
10
- #
11
- # Including class must implement +flush+, which will be called with all
12
- # accumulated items either when the output buffer fills (+:max_items+ or
13
- # +:flush_each+) or when a fixed amount of time (+:max_interval+) passes.
14
- #
15
- # == batch_receive and flush
16
- # General receive/flush can be implemented in one of two ways.
17
- #
18
- # === batch_receive(event) / flush(events)
19
- # +flush+ will receive an array of events which were passed to +buffer_receive+.
20
- #
21
- # batch_receive('one')
22
- # batch_receive('two')
23
- #
24
- # will cause a flush invocation like
25
- #
26
- # flush(['one', 'two'])
27
- #
28
- # === batch_receive(event, group) / flush(events, group)
29
- # flush() will receive an array of events, plus a grouping key.
30
- #
31
- # batch_receive('one', :server => 'a')
32
- # batch_receive('two', :server => 'b')
33
- # batch_receive('three', :server => 'a')
34
- # batch_receive('four', :server => 'b')
35
- #
36
- # will result in the following flush calls
37
- #
38
- # flush(['one', 'three'], {:server => 'a'})
39
- # flush(['two', 'four'], {:server => 'b'})
40
- #
41
- # Grouping keys can be anything which are valid Hash keys. (They don't have to
42
- # be hashes themselves.) Strings or Fixnums work fine. Use anything which you'd
43
- # like to receive in your +flush+ method to help enable different handling for
44
- # various groups of events.
45
- #
46
- # == on_flush_error
47
- # Including class may implement +on_flush_error+, which will be called with an
48
- # Exception instance whenever buffer_flush encounters an error.
49
- #
50
- # * +buffer_flush+ will automatically re-try failed flushes, so +on_flush_error+
51
- # should not try to implement retry behavior.
52
- # * Exceptions occurring within +on_flush_error+ are not handled by
53
- # +buffer_flush+.
54
- #
55
- # == on_full_buffer_receive
56
- # Including class may implement +on_full_buffer_receive+, which will be called
57
- # whenever +buffer_receive+ is called while the buffer is full.
58
- #
59
- # +on_full_buffer_receive+ will receive a Hash like <code>{:pending => 30,
60
- # :outgoing => 20}</code> which describes the internal state of the module at
61
- # the moment.
62
- #
63
- # == final flush
64
- # Including class should call <code>buffer_flush(:final => true)</code>
65
- # during a teardown/shutdown routine (after the last call to buffer_receive)
66
- # to ensure that all accumulated messages are flushed.
67
- module LogStash; module Outputs; class MicrosoftSentinelOutputInternal
68
- module CustomSizeBasedBuffer
69
-
70
- public
71
- # Initialize the buffer.
72
- #
73
- # Call directly from your constructor if you wish to set some non-default
74
- # options. Otherwise buffer_initialize will be called automatically during the
75
- # first buffer_receive call.
76
- #
77
- # Options:
78
- # * :max_items, Max number of items to buffer before flushing. Default 50.
79
- # * :flush_each, Flush each bytes of buffer. Default 0 (no flushing fired by
80
- # a buffer size).
81
- # * :max_interval, Max number of seconds to wait between flushes. Default 5.
82
- # * :logger, A logger to write log messages to. No default. Optional.
83
- #
84
- # @param [Hash] options
85
- def buffer_initialize(options={})
86
- if ! self.class.method_defined?(:flush)
87
- raise ArgumentError, "Any class including Stud::Buffer must define a flush() method."
88
- end
89
-
90
- @buffer_config = {
91
- :max_items => options[:max_items] || 50,
92
- :flush_each => options[:flush_each].to_i || 0,
93
- :max_interval => options[:max_interval] || 5,
94
- :logger => options[:logger] || nil,
95
- :has_on_flush_error => self.class.method_defined?(:on_flush_error),
96
- :has_on_full_buffer_receive => self.class.method_defined?(:on_full_buffer_receive)
97
- }
98
- @buffer_state = {
99
- # items accepted from including class
100
- :pending_items => {},
101
- :pending_count => 0,
102
- :pending_size => 0,
103
-
104
- # guard access to pending_items & pending_count & pending_size
105
- :pending_mutex => Mutex.new,
106
-
107
- # items which are currently being flushed
108
- :outgoing_items => {},
109
- :outgoing_count => 0,
110
- :outgoing_size => 0,
111
-
112
- # ensure only 1 flush is operating at once
113
- :flush_mutex => Mutex.new,
114
-
115
- # data for timed flushes
116
- :last_flush => Time.now.to_i,
117
- :timer => Thread.new do
118
- loop do
119
- sleep(@buffer_config[:max_interval])
120
- buffer_flush(:force => true)
121
- end
122
- end
123
- }
124
-
125
- # events we've accumulated
126
- buffer_clear_pending
127
- end
128
-
129
- # Determine if +:max_items+ or +:flush_each+ has been reached.
130
- #
131
- # buffer_receive calls will block while <code>buffer_full? == true</code>.
132
- #
133
- # @return [bool] Is the buffer full?
134
- def buffer_full?
135
- (@buffer_state[:pending_count] + @buffer_state[:outgoing_count] >= @buffer_config[:max_items]) || \
136
- (@buffer_config[:flush_each] != 0 && @buffer_state[:pending_size] + @buffer_state[:outgoing_size] >= @buffer_config[:flush_each])
137
- end
138
-
139
- # Save an event for later delivery
140
- #
141
- # Events are grouped by the (optional) group parameter you provide.
142
- # Groups of events, plus the group name, are later passed to +flush+.
143
- #
144
- # This call will block if +:max_items+ or +:flush_each+ has been reached.
145
- #
146
- # @see Stud::Buffer The overview has more information on grouping and flushing.
147
- #
148
- # @param event An item to buffer for flushing later.
149
- # @param group Optional grouping key. All events with the same key will be
150
- # passed to +flush+ together, along with the grouping key itself.
151
- def buffer_receive(event, group=nil)
152
- buffer_initialize if ! @buffer_state
153
-
154
- # block if we've accumulated too many events
155
- while buffer_full? do
156
- on_full_buffer_receive(
157
- :pending => @buffer_state[:pending_count],
158
- :outgoing => @buffer_state[:outgoing_count]
159
- ) if @buffer_config[:has_on_full_buffer_receive]
160
- sleep 0.1
161
- end
162
- @buffer_state[:pending_mutex].synchronize do
163
- @buffer_state[:pending_items][group] << event
164
- @buffer_state[:pending_count] += 1
165
- @buffer_state[:pending_size] += var_size(event) if @buffer_config[:flush_each] != 0
166
- end
167
-
168
- buffer_flush
169
- end
170
-
171
- # Try to flush events.
172
- #
173
- # Returns immediately if flushing is not necessary/possible at the moment:
174
- # * :max_items or :flush_each have not been accumulated
175
- # * :max_interval seconds have not elapased since the last flush
176
- # * another flush is in progress
177
- #
178
- # <code>buffer_flush(:force => true)</code> will cause a flush to occur even
179
- # if +:max_items+ or +:flush_each+ or +:max_interval+ have not been reached. A forced flush
180
- # will still return immediately (without flushing) if another flush is
181
- # currently in progress.
182
- #
183
- # <code>buffer_flush(:final => true)</code> is identical to <code>buffer_flush(:force => true)</code>,
184
- # except that if another flush is already in progress, <code>buffer_flush(:final => true)</code>
185
- # will block/wait for the other flush to finish before proceeding.
186
- #
187
- # @param [Hash] options Optional. May be <code>{:force => true}</code> or <code>{:final => true}</code>.
188
- # @return [Fixnum] The number of items successfully passed to +flush+.
189
- def buffer_flush(options={})
190
- force = options[:force] || options[:final]
191
- final = options[:final]
192
-
193
- # final flush will wait for lock, so we are sure to flush out all buffered events
194
- if options[:final]
195
- @buffer_state[:flush_mutex].lock
196
- elsif ! @buffer_state[:flush_mutex].try_lock # failed to get lock, another flush already in progress
197
- return 0
198
- end
199
-
200
- items_flushed = 0
201
-
202
- begin
203
- return 0 if @buffer_state[:pending_count] == 0
204
-
205
- # compute time_since_last_flush only when some item is pending
206
- time_since_last_flush = get_time_since_last_flush
207
-
208
- return 0 if (!force) &&
209
- (@buffer_state[:pending_count] < @buffer_config[:max_items]) &&
210
- (@buffer_config[:flush_each] == 0 || @buffer_state[:pending_size] < @buffer_config[:flush_each]) &&
211
- (time_since_last_flush < @buffer_config[:max_interval])
212
-
213
- @buffer_state[:pending_mutex].synchronize do
214
- @buffer_state[:outgoing_items] = @buffer_state[:pending_items]
215
- @buffer_state[:outgoing_count] = @buffer_state[:pending_count]
216
- @buffer_state[:outgoing_size] = @buffer_state[:pending_size]
217
- buffer_clear_pending
218
- end
219
- @buffer_config[:logger].debug("Flushing output",
220
- :outgoing_count => @buffer_state[:outgoing_count],
221
- :time_since_last_flush => time_since_last_flush,
222
- :outgoing_events => @buffer_state[:outgoing_items],
223
- :batch_timeout => @buffer_config[:max_interval],
224
- :force => force,
225
- :final => final
226
- ) if @buffer_config[:logger]
227
-
228
- @buffer_state[:outgoing_items].each do |group, events|
229
- begin
230
-
231
- if group.nil?
232
- flush(events,final)
233
- else
234
- flush(events, group, final)
235
- end
236
-
237
- @buffer_state[:outgoing_items].delete(group)
238
- events_size = events.size
239
- @buffer_state[:outgoing_count] -= events_size
240
- if @buffer_config[:flush_each] != 0
241
- events_volume = 0
242
- events.each do |event|
243
- events_volume += var_size(event)
244
- end
245
- @buffer_state[:outgoing_size] -= events_volume
246
- end
247
- items_flushed += events_size
248
-
249
- rescue => e
250
- @buffer_config[:logger].warn("Failed to flush outgoing items",
251
- :outgoing_count => @buffer_state[:outgoing_count],
252
- :exception => e,
253
- :backtrace => e.backtrace
254
- ) if @buffer_config[:logger]
255
-
256
- if @buffer_config[:has_on_flush_error]
257
- on_flush_error e
258
- end
259
-
260
- sleep 1
261
- retry
262
- end
263
- @buffer_state[:last_flush] = Time.now.to_i
264
- end
265
-
266
- ensure
267
- @buffer_state[:flush_mutex].unlock
268
- end
269
-
270
- return items_flushed
271
- end
272
-
273
- private
274
- def buffer_clear_pending
275
- @buffer_state[:pending_items] = Hash.new { |h, k| h[k] = [] }
276
- @buffer_state[:pending_count] = 0
277
- @buffer_state[:pending_size] = 0
278
- end
279
-
280
- private
281
- def var_size(var)
282
- # Calculate event size as a json.
283
- # assuming event is a hash
284
- return var.to_json.bytesize + 2
285
- end
286
-
287
- protected
288
- def get_time_since_last_flush
289
- Time.now.to_i - @buffer_state[:last_flush]
290
- end
291
-
292
- end
293
- end ;end ;end
@@ -1,58 +0,0 @@
1
- # encoding: utf-8
2
- require "logstash/sentinel_la/logstashLoganalyticsConfiguration"
3
-
4
- module LogStash
5
- module Outputs
6
- class MicrosoftSentinelOutputInternal
7
- class EventsHandler
8
-
9
- def initialize(logstashLogAnalyticsConfiguration)
10
- @logstashLogAnalyticsConfiguration = logstashLogAnalyticsConfiguration
11
- @logger = logstashLogAnalyticsConfiguration.logger
12
- @key_names = logstashLogAnalyticsConfiguration.key_names
13
- @columns_to_modify = {"@timestamp" => "ls_timestamp", "@version" => "ls_version"}
14
- end
15
-
16
- def handle_events(events)
17
- raise "Method handle_events not implemented"
18
- end
19
-
20
- def close
21
- raise "Method close not implemented"
22
- end
23
-
24
- # In case that the user has defined key_names meaning that he would like to a subset of the data,
25
- # we would like to insert only those keys.
26
- # If no keys were defined we will send all the data
27
- def create_event_document(event)
28
- document = {}
29
- event_hash = event.to_hash
30
-
31
- @columns_to_modify.each {|original_key, new_key|
32
- if event_hash.has_key?(original_key)
33
- event_hash[new_key] = event_hash[original_key]
34
- event_hash.delete(original_key)
35
- end
36
- }
37
-
38
- if @key_names.length > 0
39
- # Get the intersection of key_names and keys of event_hash
40
- keys_intersection = @key_names & event_hash.keys
41
- keys_intersection.each do |key|
42
- document[key] = event_hash[key]
43
- end
44
- if document.keys.length < 1
45
- @logger.warn("No keys found, message is dropped. Plugin keys: #{@key_names}, Event keys: #{event_hash}. The event message do not match event expected structre. Please edit key_names section in output plugin and try again.")
46
- end
47
- else
48
- document = event_hash
49
- end
50
-
51
- return document
52
- end
53
- # def create_event_document
54
-
55
- end
56
- end
57
- end
58
- end