microsoft-sentinel-log-analytics-logstash-output-plugin 1.1.4 → 2.2.1-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +18 -16
- data/Gemfile +12 -2
- data/README.md +251 -258
- data/VERSION +1 -0
- data/lib/logstash/outputs/microsoft-sentinel-log-analytics-logstash-output-plugin.rb +7 -116
- data/lib/logstash-output-microsoft-sentinel-log-analytics-logstash-output-plugin_jars.rb +5 -0
- data/lib/logstash_registry.rb +4 -0
- data/logstash-output-microsoft-sentinel-log-analytics-logstash-output-plugin.gemspec +22 -0
- data/vendor/jar-dependencies/org/logstashplugins/logstash-output-microsoft-sentinel-log-analytics-logstash-output-plugin/2.2.1/logstash-output-microsoft-sentinel-log-analytics-logstash-output-plugin-2.2.1.jar +0 -0
- metadata +17 -44
- data/LICENSE +0 -21
- data/lib/logstash/sentinel_la/customSizeBasedBuffer.rb +0 -293
- data/lib/logstash/sentinel_la/eventsHandler.rb +0 -58
- data/lib/logstash/sentinel_la/logAnalyticsAadTokenProvider.rb +0 -90
- data/lib/logstash/sentinel_la/logAnalyticsClient.rb +0 -135
- data/lib/logstash/sentinel_la/logStashAutoResizeBuffer.rb +0 -157
- data/lib/logstash/sentinel_la/logStashCompressedStream.rb +0 -144
- data/lib/logstash/sentinel_la/logStashEventsBatcher.rb +0 -144
- data/lib/logstash/sentinel_la/logsSender.rb +0 -47
- data/lib/logstash/sentinel_la/logstashLoganalyticsConfiguration.rb +0 -243
- data/lib/logstash/sentinel_la/sampleFileCreator.rb +0 -61
- data/lib/logstash/sentinel_la/version.rb +0 -10
- data/microsoft-sentinel-log-analytics-logstash-output-plugin.gemspec +0 -27
|
@@ -1,116 +1,7 @@
|
|
|
1
|
-
#
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
require "logstash
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
class LogStash::Outputs::MicrosoftSentinelOutput < LogStash::Outputs::Base
|
|
10
|
-
|
|
11
|
-
config_name "microsoft-sentinel-log-analytics-logstash-output-plugin"
|
|
12
|
-
|
|
13
|
-
# Stating that the output plugin will run in concurrent mode
|
|
14
|
-
concurrency :shared
|
|
15
|
-
|
|
16
|
-
# Your registered app ID
|
|
17
|
-
config :client_app_Id, :validate => :string
|
|
18
|
-
|
|
19
|
-
# The registered app's secret, required by Azure Loganalytics REST API
|
|
20
|
-
config :client_app_secret, :validate => :string
|
|
21
|
-
|
|
22
|
-
# Your Operations Management Suite Tenant ID
|
|
23
|
-
config :tenant_id, :validate => :string
|
|
24
|
-
|
|
25
|
-
# Your data collection rule endpoint
|
|
26
|
-
config :data_collection_endpoint, :validate => :string
|
|
27
|
-
|
|
28
|
-
# Your data collection rule ID
|
|
29
|
-
config :dcr_immutable_id, :validate => :string
|
|
30
|
-
|
|
31
|
-
# Your dcr data stream name
|
|
32
|
-
config :dcr_stream_name, :validate => :string
|
|
33
|
-
|
|
34
|
-
# Subset of keys to send to the Azure Loganalytics workspace
|
|
35
|
-
config :key_names, :validate => :array, :default => []
|
|
36
|
-
|
|
37
|
-
# Max number of seconds to wait between flushes. Default 5
|
|
38
|
-
config :plugin_flush_interval, :validate => :number, :default => 5
|
|
39
|
-
|
|
40
|
-
# Factor for adding to the amount of messages sent
|
|
41
|
-
config :decrease_factor, :validate => :number, :default => 100
|
|
42
|
-
|
|
43
|
-
# This will trigger message amount resizing in a REST request to LA
|
|
44
|
-
config :amount_resizing, :validate => :boolean, :default => true
|
|
45
|
-
|
|
46
|
-
# Setting the default amount of messages sent
|
|
47
|
-
# it this is set with amount_resizing=false --> each message will have max_items
|
|
48
|
-
config :max_items, :validate => :number, :default => 2000
|
|
49
|
-
|
|
50
|
-
# Setting default proxy to be used for all communication with azure
|
|
51
|
-
config :proxy, :validate => :string
|
|
52
|
-
|
|
53
|
-
# Setting proxy_aad to be used for communicating with azure active directory service
|
|
54
|
-
config :proxy_aad, :validate => :string
|
|
55
|
-
|
|
56
|
-
# Setting proxy to be used for the LogAnalytics endpoint REST client
|
|
57
|
-
config :proxy_endpoint, :validate => :string
|
|
58
|
-
|
|
59
|
-
# This will set the amount of time given for retransmitting messages once sending is failed
|
|
60
|
-
config :retransmission_time, :validate => :number, :default => 10
|
|
61
|
-
|
|
62
|
-
# Compress the message body before sending to LA
|
|
63
|
-
config :compress_data, :validate => :boolean, :default => false
|
|
64
|
-
|
|
65
|
-
# Generate sample file from incoming events
|
|
66
|
-
config :create_sample_file, :validate => :boolean, :default => false
|
|
67
|
-
|
|
68
|
-
# Path where to place the sample file created
|
|
69
|
-
config :sample_file_path, :validate => :string
|
|
70
|
-
|
|
71
|
-
# Used to specify the name of the Azure cloud that is being used. By default, the value is set to "AzureCloud", which
|
|
72
|
-
# is the public Azure cloud. However, you can specify a different Azure cloud if you are
|
|
73
|
-
# using a different environment, such as Azure Government or Azure China.
|
|
74
|
-
config :azure_cloud, :validate => :string
|
|
75
|
-
|
|
76
|
-
public
|
|
77
|
-
def register
|
|
78
|
-
@logstash_configuration= build_logstash_configuration()
|
|
79
|
-
|
|
80
|
-
# Validate configuration correctness
|
|
81
|
-
@logstash_configuration.validate_configuration()
|
|
82
|
-
|
|
83
|
-
@events_handler = @logstash_configuration.create_sample_file ?
|
|
84
|
-
LogStash::Outputs::MicrosoftSentinelOutputInternal::SampleFileCreator::new(@logstash_configuration) :
|
|
85
|
-
LogStash::Outputs::MicrosoftSentinelOutputInternal::LogsSender::new(@logstash_configuration)
|
|
86
|
-
end # def register
|
|
87
|
-
|
|
88
|
-
def multi_receive(events)
|
|
89
|
-
@events_handler.handle_events(events)
|
|
90
|
-
end # def multi_receive
|
|
91
|
-
|
|
92
|
-
def close
|
|
93
|
-
@events_handler.close
|
|
94
|
-
end
|
|
95
|
-
|
|
96
|
-
#private
|
|
97
|
-
private
|
|
98
|
-
|
|
99
|
-
# Building the logstash object configuration from the output configuration provided by the user
|
|
100
|
-
# Return LogstashLoganalyticsOutputConfiguration populated with the configuration values
|
|
101
|
-
def build_logstash_configuration()
|
|
102
|
-
logstash_configuration= LogStash::Outputs::MicrosoftSentinelOutputInternal::LogstashLoganalyticsOutputConfiguration::new(@client_app_Id, @client_app_secret, @tenant_id, @data_collection_endpoint, @dcr_immutable_id, @dcr_stream_name, @compress_data, @create_sample_file, @sample_file_path, @logger)
|
|
103
|
-
logstash_configuration.key_names = @key_names
|
|
104
|
-
logstash_configuration.plugin_flush_interval = @plugin_flush_interval
|
|
105
|
-
logstash_configuration.decrease_factor = @decrease_factor
|
|
106
|
-
logstash_configuration.amount_resizing = @amount_resizing
|
|
107
|
-
logstash_configuration.max_items = @max_items
|
|
108
|
-
logstash_configuration.proxy_aad = @proxy_aad || @proxy || ENV['http_proxy']
|
|
109
|
-
logstash_configuration.proxy_endpoint = @proxy_endpoint || @proxy || ENV['http_proxy']
|
|
110
|
-
logstash_configuration.retransmission_time = @retransmission_time
|
|
111
|
-
logstash_configuration.azure_cloud = @azure_cloud || "AzureCloud"
|
|
112
|
-
|
|
113
|
-
return logstash_configuration
|
|
114
|
-
end # def build_logstash_configuration
|
|
115
|
-
|
|
116
|
-
end # class LogStash::Outputs::MicrosoftSentinelOutput
|
|
1
|
+
# AUTOGENERATED BY THE GRADLE SCRIPT. EDITS WILL BE OVERWRITTEN.
|
|
2
|
+
# encoding: utf-8
|
|
3
|
+
|
|
4
|
+
# NOTE: Java plugin (no Ruby plugin class). This file exists only to ensure
|
|
5
|
+
# jar-dependencies loads the packaged jar into JRuby's classpath.
|
|
6
|
+
require "logstash-output-microsoft-sentinel-log-analytics-logstash-output-plugin_jars"
|
|
7
|
+
require "java"
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# AUTOGENERATED BY THE GRADLE SCRIPT. EDITS WILL BE OVERWRITTEN.
|
|
2
|
+
Gem::Specification.new do |s|
|
|
3
|
+
s.name = 'microsoft-sentinel-log-analytics-logstash-output-plugin'
|
|
4
|
+
s.version = '2.2.1'
|
|
5
|
+
s.licenses = ['Apache-2.0']
|
|
6
|
+
s.summary = 'Microsoft Sentinel Log Analytics output plugin'
|
|
7
|
+
s.description = 'Microsoft Sentinel provides a new output plugin for Logstash. Use this output plugin to send any log via Logstash to the Microsoft Sentinel/Log Analytics workspace. This is done with the Log Analytics DCR-based API.'
|
|
8
|
+
s.authors = ['Microsoft']
|
|
9
|
+
s.email = ['msg-ucp@service.microsoft.com']
|
|
10
|
+
s.platform = 'java'
|
|
11
|
+
s.require_paths = ['lib', 'vendor/jar-dependencies']
|
|
12
|
+
|
|
13
|
+
s.files = Dir["lib/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
|
|
14
|
+
|
|
15
|
+
# Special flag to let us know this is actually a logstash plugin
|
|
16
|
+
s.metadata = { 'logstash_plugin' => 'true', 'logstash_group' => 'output', 'java_plugin' => 'true'}
|
|
17
|
+
|
|
18
|
+
# Gem dependencies
|
|
19
|
+
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
|
20
|
+
s.add_runtime_dependency 'jar-dependencies'
|
|
21
|
+
s.add_development_dependency 'logstash-devutils'
|
|
22
|
+
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: microsoft-sentinel-log-analytics-logstash-output-plugin
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version:
|
|
5
|
-
platform:
|
|
4
|
+
version: 2.2.1
|
|
5
|
+
platform: java
|
|
6
6
|
authors:
|
|
7
|
-
- Microsoft
|
|
7
|
+
- Microsoft
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2026-05-27 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -36,7 +36,7 @@ dependencies:
|
|
|
36
36
|
- - ">="
|
|
37
37
|
- !ruby/object:Gem::Version
|
|
38
38
|
version: '0'
|
|
39
|
-
name:
|
|
39
|
+
name: jar-dependencies
|
|
40
40
|
type: :runtime
|
|
41
41
|
prerelease: false
|
|
42
42
|
version_requirements: !ruby/object:Gem::Requirement
|
|
@@ -44,26 +44,6 @@ dependencies:
|
|
|
44
44
|
- - ">="
|
|
45
45
|
- !ruby/object:Gem::Version
|
|
46
46
|
version: '0'
|
|
47
|
-
- !ruby/object:Gem::Dependency
|
|
48
|
-
requirement: !ruby/object:Gem::Requirement
|
|
49
|
-
requirements:
|
|
50
|
-
- - ">="
|
|
51
|
-
- !ruby/object:Gem::Version
|
|
52
|
-
version: 0.88.0
|
|
53
|
-
- - "<"
|
|
54
|
-
- !ruby/object:Gem::Version
|
|
55
|
-
version: 1.0.0
|
|
56
|
-
name: excon
|
|
57
|
-
type: :runtime
|
|
58
|
-
prerelease: false
|
|
59
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
60
|
-
requirements:
|
|
61
|
-
- - ">="
|
|
62
|
-
- !ruby/object:Gem::Version
|
|
63
|
-
version: 0.88.0
|
|
64
|
-
- - "<"
|
|
65
|
-
- !ruby/object:Gem::Version
|
|
66
|
-
version: 1.0.0
|
|
67
47
|
- !ruby/object:Gem::Dependency
|
|
68
48
|
requirement: !ruby/object:Gem::Requirement
|
|
69
49
|
requirements:
|
|
@@ -81,38 +61,33 @@ dependencies:
|
|
|
81
61
|
description: Microsoft Sentinel provides a new output plugin for Logstash. Use this
|
|
82
62
|
output plugin to send any log via Logstash to the Microsoft Sentinel/Log Analytics
|
|
83
63
|
workspace. This is done with the Log Analytics DCR-based API.
|
|
84
|
-
email:
|
|
64
|
+
email:
|
|
65
|
+
- msg-ucp@service.microsoft.com
|
|
85
66
|
executables: []
|
|
86
67
|
extensions: []
|
|
87
68
|
extra_rdoc_files: []
|
|
88
69
|
files:
|
|
89
70
|
- CHANGELOG.md
|
|
90
71
|
- Gemfile
|
|
91
|
-
- LICENSE
|
|
92
72
|
- README.md
|
|
73
|
+
- VERSION
|
|
74
|
+
- lib/logstash-output-microsoft-sentinel-log-analytics-logstash-output-plugin_jars.rb
|
|
93
75
|
- lib/logstash/outputs/microsoft-sentinel-log-analytics-logstash-output-plugin.rb
|
|
94
|
-
- lib/
|
|
95
|
-
-
|
|
96
|
-
-
|
|
97
|
-
|
|
98
|
-
- lib/logstash/sentinel_la/logStashAutoResizeBuffer.rb
|
|
99
|
-
- lib/logstash/sentinel_la/logStashCompressedStream.rb
|
|
100
|
-
- lib/logstash/sentinel_la/logStashEventsBatcher.rb
|
|
101
|
-
- lib/logstash/sentinel_la/logsSender.rb
|
|
102
|
-
- lib/logstash/sentinel_la/logstashLoganalyticsConfiguration.rb
|
|
103
|
-
- lib/logstash/sentinel_la/sampleFileCreator.rb
|
|
104
|
-
- lib/logstash/sentinel_la/version.rb
|
|
105
|
-
- microsoft-sentinel-log-analytics-logstash-output-plugin.gemspec
|
|
106
|
-
homepage: https://github.com/Azure/Azure-Sentinel
|
|
76
|
+
- lib/logstash_registry.rb
|
|
77
|
+
- logstash-output-microsoft-sentinel-log-analytics-logstash-output-plugin.gemspec
|
|
78
|
+
- vendor/jar-dependencies/org/logstashplugins/logstash-output-microsoft-sentinel-log-analytics-logstash-output-plugin/2.2.1/logstash-output-microsoft-sentinel-log-analytics-logstash-output-plugin-2.2.1.jar
|
|
79
|
+
homepage:
|
|
107
80
|
licenses:
|
|
108
|
-
-
|
|
81
|
+
- Apache-2.0
|
|
109
82
|
metadata:
|
|
110
83
|
logstash_plugin: 'true'
|
|
111
84
|
logstash_group: output
|
|
85
|
+
java_plugin: 'true'
|
|
112
86
|
post_install_message:
|
|
113
87
|
rdoc_options: []
|
|
114
88
|
require_paths:
|
|
115
89
|
- lib
|
|
90
|
+
- vendor/jar-dependencies
|
|
116
91
|
required_ruby_version: !ruby/object:Gem::Requirement
|
|
117
92
|
requirements:
|
|
118
93
|
- - ">="
|
|
@@ -127,7 +102,5 @@ requirements: []
|
|
|
127
102
|
rubygems_version: 3.3.26
|
|
128
103
|
signing_key:
|
|
129
104
|
specification_version: 4
|
|
130
|
-
summary: Microsoft Sentinel
|
|
131
|
-
plugin to send any log via Logstash to the Microsoft Sentinel/Log Analytics workspace.
|
|
132
|
-
This is done with the Log Analytics DCR-based API.
|
|
105
|
+
summary: Microsoft Sentinel Log Analytics output plugin
|
|
133
106
|
test_files: []
|
data/LICENSE
DELETED
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
MIT License
|
|
2
|
-
|
|
3
|
-
Copyright (c) Microsoft Corporation. All rights reserved.
|
|
4
|
-
|
|
5
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
-
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
-
in the Software without restriction, including without limitation the rights
|
|
8
|
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
-
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
-
furnished to do so, subject to the following conditions:
|
|
11
|
-
|
|
12
|
-
The above copyright notice and this permission notice shall be included in all
|
|
13
|
-
copies or substantial portions of the Software.
|
|
14
|
-
|
|
15
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
-
SOFTWARE
|
|
@@ -1,293 +0,0 @@
|
|
|
1
|
-
# This code is from a PR for the official repo of ruby-stud
|
|
2
|
-
# with a small change to calculating the event size in the var_size function
|
|
3
|
-
# https://github.com/jordansissel/ruby-stud/pull/19
|
|
4
|
-
#
|
|
5
|
-
# @author {Alex Dean}[http://github.com/alexdean]
|
|
6
|
-
#
|
|
7
|
-
# Implements a generic framework for accepting events which are later flushed
|
|
8
|
-
# in batches. Flushing occurs whenever +:max_items+ or +:max_interval+ (seconds)
|
|
9
|
-
# has been reached or if the event size outgrows +:flush_each+ (bytes)
|
|
10
|
-
#
|
|
11
|
-
# Including class must implement +flush+, which will be called with all
|
|
12
|
-
# accumulated items either when the output buffer fills (+:max_items+ or
|
|
13
|
-
# +:flush_each+) or when a fixed amount of time (+:max_interval+) passes.
|
|
14
|
-
#
|
|
15
|
-
# == batch_receive and flush
|
|
16
|
-
# General receive/flush can be implemented in one of two ways.
|
|
17
|
-
#
|
|
18
|
-
# === batch_receive(event) / flush(events)
|
|
19
|
-
# +flush+ will receive an array of events which were passed to +buffer_receive+.
|
|
20
|
-
#
|
|
21
|
-
# batch_receive('one')
|
|
22
|
-
# batch_receive('two')
|
|
23
|
-
#
|
|
24
|
-
# will cause a flush invocation like
|
|
25
|
-
#
|
|
26
|
-
# flush(['one', 'two'])
|
|
27
|
-
#
|
|
28
|
-
# === batch_receive(event, group) / flush(events, group)
|
|
29
|
-
# flush() will receive an array of events, plus a grouping key.
|
|
30
|
-
#
|
|
31
|
-
# batch_receive('one', :server => 'a')
|
|
32
|
-
# batch_receive('two', :server => 'b')
|
|
33
|
-
# batch_receive('three', :server => 'a')
|
|
34
|
-
# batch_receive('four', :server => 'b')
|
|
35
|
-
#
|
|
36
|
-
# will result in the following flush calls
|
|
37
|
-
#
|
|
38
|
-
# flush(['one', 'three'], {:server => 'a'})
|
|
39
|
-
# flush(['two', 'four'], {:server => 'b'})
|
|
40
|
-
#
|
|
41
|
-
# Grouping keys can be anything which are valid Hash keys. (They don't have to
|
|
42
|
-
# be hashes themselves.) Strings or Fixnums work fine. Use anything which you'd
|
|
43
|
-
# like to receive in your +flush+ method to help enable different handling for
|
|
44
|
-
# various groups of events.
|
|
45
|
-
#
|
|
46
|
-
# == on_flush_error
|
|
47
|
-
# Including class may implement +on_flush_error+, which will be called with an
|
|
48
|
-
# Exception instance whenever buffer_flush encounters an error.
|
|
49
|
-
#
|
|
50
|
-
# * +buffer_flush+ will automatically re-try failed flushes, so +on_flush_error+
|
|
51
|
-
# should not try to implement retry behavior.
|
|
52
|
-
# * Exceptions occurring within +on_flush_error+ are not handled by
|
|
53
|
-
# +buffer_flush+.
|
|
54
|
-
#
|
|
55
|
-
# == on_full_buffer_receive
|
|
56
|
-
# Including class may implement +on_full_buffer_receive+, which will be called
|
|
57
|
-
# whenever +buffer_receive+ is called while the buffer is full.
|
|
58
|
-
#
|
|
59
|
-
# +on_full_buffer_receive+ will receive a Hash like <code>{:pending => 30,
|
|
60
|
-
# :outgoing => 20}</code> which describes the internal state of the module at
|
|
61
|
-
# the moment.
|
|
62
|
-
#
|
|
63
|
-
# == final flush
|
|
64
|
-
# Including class should call <code>buffer_flush(:final => true)</code>
|
|
65
|
-
# during a teardown/shutdown routine (after the last call to buffer_receive)
|
|
66
|
-
# to ensure that all accumulated messages are flushed.
|
|
67
|
-
module LogStash; module Outputs; class MicrosoftSentinelOutputInternal
|
|
68
|
-
module CustomSizeBasedBuffer
|
|
69
|
-
|
|
70
|
-
public
|
|
71
|
-
# Initialize the buffer.
|
|
72
|
-
#
|
|
73
|
-
# Call directly from your constructor if you wish to set some non-default
|
|
74
|
-
# options. Otherwise buffer_initialize will be called automatically during the
|
|
75
|
-
# first buffer_receive call.
|
|
76
|
-
#
|
|
77
|
-
# Options:
|
|
78
|
-
# * :max_items, Max number of items to buffer before flushing. Default 50.
|
|
79
|
-
# * :flush_each, Flush each bytes of buffer. Default 0 (no flushing fired by
|
|
80
|
-
# a buffer size).
|
|
81
|
-
# * :max_interval, Max number of seconds to wait between flushes. Default 5.
|
|
82
|
-
# * :logger, A logger to write log messages to. No default. Optional.
|
|
83
|
-
#
|
|
84
|
-
# @param [Hash] options
|
|
85
|
-
def buffer_initialize(options={})
|
|
86
|
-
if ! self.class.method_defined?(:flush)
|
|
87
|
-
raise ArgumentError, "Any class including Stud::Buffer must define a flush() method."
|
|
88
|
-
end
|
|
89
|
-
|
|
90
|
-
@buffer_config = {
|
|
91
|
-
:max_items => options[:max_items] || 50,
|
|
92
|
-
:flush_each => options[:flush_each].to_i || 0,
|
|
93
|
-
:max_interval => options[:max_interval] || 5,
|
|
94
|
-
:logger => options[:logger] || nil,
|
|
95
|
-
:has_on_flush_error => self.class.method_defined?(:on_flush_error),
|
|
96
|
-
:has_on_full_buffer_receive => self.class.method_defined?(:on_full_buffer_receive)
|
|
97
|
-
}
|
|
98
|
-
@buffer_state = {
|
|
99
|
-
# items accepted from including class
|
|
100
|
-
:pending_items => {},
|
|
101
|
-
:pending_count => 0,
|
|
102
|
-
:pending_size => 0,
|
|
103
|
-
|
|
104
|
-
# guard access to pending_items & pending_count & pending_size
|
|
105
|
-
:pending_mutex => Mutex.new,
|
|
106
|
-
|
|
107
|
-
# items which are currently being flushed
|
|
108
|
-
:outgoing_items => {},
|
|
109
|
-
:outgoing_count => 0,
|
|
110
|
-
:outgoing_size => 0,
|
|
111
|
-
|
|
112
|
-
# ensure only 1 flush is operating at once
|
|
113
|
-
:flush_mutex => Mutex.new,
|
|
114
|
-
|
|
115
|
-
# data for timed flushes
|
|
116
|
-
:last_flush => Time.now.to_i,
|
|
117
|
-
:timer => Thread.new do
|
|
118
|
-
loop do
|
|
119
|
-
sleep(@buffer_config[:max_interval])
|
|
120
|
-
buffer_flush(:force => true)
|
|
121
|
-
end
|
|
122
|
-
end
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
# events we've accumulated
|
|
126
|
-
buffer_clear_pending
|
|
127
|
-
end
|
|
128
|
-
|
|
129
|
-
# Determine if +:max_items+ or +:flush_each+ has been reached.
|
|
130
|
-
#
|
|
131
|
-
# buffer_receive calls will block while <code>buffer_full? == true</code>.
|
|
132
|
-
#
|
|
133
|
-
# @return [bool] Is the buffer full?
|
|
134
|
-
def buffer_full?
|
|
135
|
-
(@buffer_state[:pending_count] + @buffer_state[:outgoing_count] >= @buffer_config[:max_items]) || \
|
|
136
|
-
(@buffer_config[:flush_each] != 0 && @buffer_state[:pending_size] + @buffer_state[:outgoing_size] >= @buffer_config[:flush_each])
|
|
137
|
-
end
|
|
138
|
-
|
|
139
|
-
# Save an event for later delivery
|
|
140
|
-
#
|
|
141
|
-
# Events are grouped by the (optional) group parameter you provide.
|
|
142
|
-
# Groups of events, plus the group name, are later passed to +flush+.
|
|
143
|
-
#
|
|
144
|
-
# This call will block if +:max_items+ or +:flush_each+ has been reached.
|
|
145
|
-
#
|
|
146
|
-
# @see Stud::Buffer The overview has more information on grouping and flushing.
|
|
147
|
-
#
|
|
148
|
-
# @param event An item to buffer for flushing later.
|
|
149
|
-
# @param group Optional grouping key. All events with the same key will be
|
|
150
|
-
# passed to +flush+ together, along with the grouping key itself.
|
|
151
|
-
def buffer_receive(event, group=nil)
|
|
152
|
-
buffer_initialize if ! @buffer_state
|
|
153
|
-
|
|
154
|
-
# block if we've accumulated too many events
|
|
155
|
-
while buffer_full? do
|
|
156
|
-
on_full_buffer_receive(
|
|
157
|
-
:pending => @buffer_state[:pending_count],
|
|
158
|
-
:outgoing => @buffer_state[:outgoing_count]
|
|
159
|
-
) if @buffer_config[:has_on_full_buffer_receive]
|
|
160
|
-
sleep 0.1
|
|
161
|
-
end
|
|
162
|
-
@buffer_state[:pending_mutex].synchronize do
|
|
163
|
-
@buffer_state[:pending_items][group] << event
|
|
164
|
-
@buffer_state[:pending_count] += 1
|
|
165
|
-
@buffer_state[:pending_size] += var_size(event) if @buffer_config[:flush_each] != 0
|
|
166
|
-
end
|
|
167
|
-
|
|
168
|
-
buffer_flush
|
|
169
|
-
end
|
|
170
|
-
|
|
171
|
-
# Try to flush events.
|
|
172
|
-
#
|
|
173
|
-
# Returns immediately if flushing is not necessary/possible at the moment:
|
|
174
|
-
# * :max_items or :flush_each have not been accumulated
|
|
175
|
-
# * :max_interval seconds have not elapased since the last flush
|
|
176
|
-
# * another flush is in progress
|
|
177
|
-
#
|
|
178
|
-
# <code>buffer_flush(:force => true)</code> will cause a flush to occur even
|
|
179
|
-
# if +:max_items+ or +:flush_each+ or +:max_interval+ have not been reached. A forced flush
|
|
180
|
-
# will still return immediately (without flushing) if another flush is
|
|
181
|
-
# currently in progress.
|
|
182
|
-
#
|
|
183
|
-
# <code>buffer_flush(:final => true)</code> is identical to <code>buffer_flush(:force => true)</code>,
|
|
184
|
-
# except that if another flush is already in progress, <code>buffer_flush(:final => true)</code>
|
|
185
|
-
# will block/wait for the other flush to finish before proceeding.
|
|
186
|
-
#
|
|
187
|
-
# @param [Hash] options Optional. May be <code>{:force => true}</code> or <code>{:final => true}</code>.
|
|
188
|
-
# @return [Fixnum] The number of items successfully passed to +flush+.
|
|
189
|
-
def buffer_flush(options={})
|
|
190
|
-
force = options[:force] || options[:final]
|
|
191
|
-
final = options[:final]
|
|
192
|
-
|
|
193
|
-
# final flush will wait for lock, so we are sure to flush out all buffered events
|
|
194
|
-
if options[:final]
|
|
195
|
-
@buffer_state[:flush_mutex].lock
|
|
196
|
-
elsif ! @buffer_state[:flush_mutex].try_lock # failed to get lock, another flush already in progress
|
|
197
|
-
return 0
|
|
198
|
-
end
|
|
199
|
-
|
|
200
|
-
items_flushed = 0
|
|
201
|
-
|
|
202
|
-
begin
|
|
203
|
-
return 0 if @buffer_state[:pending_count] == 0
|
|
204
|
-
|
|
205
|
-
# compute time_since_last_flush only when some item is pending
|
|
206
|
-
time_since_last_flush = get_time_since_last_flush
|
|
207
|
-
|
|
208
|
-
return 0 if (!force) &&
|
|
209
|
-
(@buffer_state[:pending_count] < @buffer_config[:max_items]) &&
|
|
210
|
-
(@buffer_config[:flush_each] == 0 || @buffer_state[:pending_size] < @buffer_config[:flush_each]) &&
|
|
211
|
-
(time_since_last_flush < @buffer_config[:max_interval])
|
|
212
|
-
|
|
213
|
-
@buffer_state[:pending_mutex].synchronize do
|
|
214
|
-
@buffer_state[:outgoing_items] = @buffer_state[:pending_items]
|
|
215
|
-
@buffer_state[:outgoing_count] = @buffer_state[:pending_count]
|
|
216
|
-
@buffer_state[:outgoing_size] = @buffer_state[:pending_size]
|
|
217
|
-
buffer_clear_pending
|
|
218
|
-
end
|
|
219
|
-
@buffer_config[:logger].debug("Flushing output",
|
|
220
|
-
:outgoing_count => @buffer_state[:outgoing_count],
|
|
221
|
-
:time_since_last_flush => time_since_last_flush,
|
|
222
|
-
:outgoing_events => @buffer_state[:outgoing_items],
|
|
223
|
-
:batch_timeout => @buffer_config[:max_interval],
|
|
224
|
-
:force => force,
|
|
225
|
-
:final => final
|
|
226
|
-
) if @buffer_config[:logger]
|
|
227
|
-
|
|
228
|
-
@buffer_state[:outgoing_items].each do |group, events|
|
|
229
|
-
begin
|
|
230
|
-
|
|
231
|
-
if group.nil?
|
|
232
|
-
flush(events,final)
|
|
233
|
-
else
|
|
234
|
-
flush(events, group, final)
|
|
235
|
-
end
|
|
236
|
-
|
|
237
|
-
@buffer_state[:outgoing_items].delete(group)
|
|
238
|
-
events_size = events.size
|
|
239
|
-
@buffer_state[:outgoing_count] -= events_size
|
|
240
|
-
if @buffer_config[:flush_each] != 0
|
|
241
|
-
events_volume = 0
|
|
242
|
-
events.each do |event|
|
|
243
|
-
events_volume += var_size(event)
|
|
244
|
-
end
|
|
245
|
-
@buffer_state[:outgoing_size] -= events_volume
|
|
246
|
-
end
|
|
247
|
-
items_flushed += events_size
|
|
248
|
-
|
|
249
|
-
rescue => e
|
|
250
|
-
@buffer_config[:logger].warn("Failed to flush outgoing items",
|
|
251
|
-
:outgoing_count => @buffer_state[:outgoing_count],
|
|
252
|
-
:exception => e,
|
|
253
|
-
:backtrace => e.backtrace
|
|
254
|
-
) if @buffer_config[:logger]
|
|
255
|
-
|
|
256
|
-
if @buffer_config[:has_on_flush_error]
|
|
257
|
-
on_flush_error e
|
|
258
|
-
end
|
|
259
|
-
|
|
260
|
-
sleep 1
|
|
261
|
-
retry
|
|
262
|
-
end
|
|
263
|
-
@buffer_state[:last_flush] = Time.now.to_i
|
|
264
|
-
end
|
|
265
|
-
|
|
266
|
-
ensure
|
|
267
|
-
@buffer_state[:flush_mutex].unlock
|
|
268
|
-
end
|
|
269
|
-
|
|
270
|
-
return items_flushed
|
|
271
|
-
end
|
|
272
|
-
|
|
273
|
-
private
|
|
274
|
-
def buffer_clear_pending
|
|
275
|
-
@buffer_state[:pending_items] = Hash.new { |h, k| h[k] = [] }
|
|
276
|
-
@buffer_state[:pending_count] = 0
|
|
277
|
-
@buffer_state[:pending_size] = 0
|
|
278
|
-
end
|
|
279
|
-
|
|
280
|
-
private
|
|
281
|
-
def var_size(var)
|
|
282
|
-
# Calculate event size as a json.
|
|
283
|
-
# assuming event is a hash
|
|
284
|
-
return var.to_json.bytesize + 2
|
|
285
|
-
end
|
|
286
|
-
|
|
287
|
-
protected
|
|
288
|
-
def get_time_since_last_flush
|
|
289
|
-
Time.now.to_i - @buffer_state[:last_flush]
|
|
290
|
-
end
|
|
291
|
-
|
|
292
|
-
end
|
|
293
|
-
end ;end ;end
|
|
@@ -1,58 +0,0 @@
|
|
|
1
|
-
# encoding: utf-8
|
|
2
|
-
require "logstash/sentinel_la/logstashLoganalyticsConfiguration"
|
|
3
|
-
|
|
4
|
-
module LogStash
|
|
5
|
-
module Outputs
|
|
6
|
-
class MicrosoftSentinelOutputInternal
|
|
7
|
-
class EventsHandler
|
|
8
|
-
|
|
9
|
-
def initialize(logstashLogAnalyticsConfiguration)
|
|
10
|
-
@logstashLogAnalyticsConfiguration = logstashLogAnalyticsConfiguration
|
|
11
|
-
@logger = logstashLogAnalyticsConfiguration.logger
|
|
12
|
-
@key_names = logstashLogAnalyticsConfiguration.key_names
|
|
13
|
-
@columns_to_modify = {"@timestamp" => "ls_timestamp", "@version" => "ls_version"}
|
|
14
|
-
end
|
|
15
|
-
|
|
16
|
-
def handle_events(events)
|
|
17
|
-
raise "Method handle_events not implemented"
|
|
18
|
-
end
|
|
19
|
-
|
|
20
|
-
def close
|
|
21
|
-
raise "Method close not implemented"
|
|
22
|
-
end
|
|
23
|
-
|
|
24
|
-
# In case that the user has defined key_names meaning that he would like to a subset of the data,
|
|
25
|
-
# we would like to insert only those keys.
|
|
26
|
-
# If no keys were defined we will send all the data
|
|
27
|
-
def create_event_document(event)
|
|
28
|
-
document = {}
|
|
29
|
-
event_hash = event.to_hash
|
|
30
|
-
|
|
31
|
-
@columns_to_modify.each {|original_key, new_key|
|
|
32
|
-
if event_hash.has_key?(original_key)
|
|
33
|
-
event_hash[new_key] = event_hash[original_key]
|
|
34
|
-
event_hash.delete(original_key)
|
|
35
|
-
end
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
if @key_names.length > 0
|
|
39
|
-
# Get the intersection of key_names and keys of event_hash
|
|
40
|
-
keys_intersection = @key_names & event_hash.keys
|
|
41
|
-
keys_intersection.each do |key|
|
|
42
|
-
document[key] = event_hash[key]
|
|
43
|
-
end
|
|
44
|
-
if document.keys.length < 1
|
|
45
|
-
@logger.warn("No keys found, message is dropped. Plugin keys: #{@key_names}, Event keys: #{event_hash}. The event message do not match event expected structre. Please edit key_names section in output plugin and try again.")
|
|
46
|
-
end
|
|
47
|
-
else
|
|
48
|
-
document = event_hash
|
|
49
|
-
end
|
|
50
|
-
|
|
51
|
-
return document
|
|
52
|
-
end
|
|
53
|
-
# def create_event_document
|
|
54
|
-
|
|
55
|
-
end
|
|
56
|
-
end
|
|
57
|
-
end
|
|
58
|
-
end
|