microsoft-sentinel-log-analytics-logstash-output-plugin 1.1.0 → 1.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +16 -13
- data/Gemfile +2 -2
- data/LICENSE +21 -21
- data/README.md +258 -234
- data/lib/logstash/outputs/microsoft-sentinel-log-analytics-logstash-output-plugin.rb +116 -110
- data/lib/logstash/sentinel_la/logAnalyticsAadTokenProvider.rb +8 -9
- data/lib/logstash/sentinel_la/logAnalyticsClient.rb +60 -10
- data/lib/logstash/sentinel_la/logStashEventsBatcher.rb +32 -30
- data/lib/logstash/sentinel_la/logstashLoganalyticsConfiguration.rb +25 -4
- data/lib/logstash/sentinel_la/version.rb +2 -2
- data/microsoft-sentinel-log-analytics-logstash-output-plugin.gemspec +27 -27
- metadata +25 -19
|
@@ -1,110 +1,116 @@
|
|
|
1
|
-
# encoding: utf-8
|
|
2
|
-
require "logstash/outputs/base"
|
|
3
|
-
require "logstash/namespace"
|
|
4
|
-
require "logstash/sentinel_la/logstashLoganalyticsConfiguration"
|
|
5
|
-
require "logstash/sentinel_la/sampleFileCreator"
|
|
6
|
-
require "logstash/sentinel_la/logsSender"
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
class LogStash::Outputs::MicrosoftSentinelOutput < LogStash::Outputs::Base
|
|
10
|
-
|
|
11
|
-
config_name "microsoft-sentinel-log-analytics-logstash-output-plugin"
|
|
12
|
-
|
|
13
|
-
# Stating that the output plugin will run in concurrent mode
|
|
14
|
-
concurrency :shared
|
|
15
|
-
|
|
16
|
-
# Your registered app ID
|
|
17
|
-
config :client_app_Id, :validate => :string
|
|
18
|
-
|
|
19
|
-
# The registered app's secret, required by Azure Loganalytics REST API
|
|
20
|
-
config :client_app_secret, :validate => :string
|
|
21
|
-
|
|
22
|
-
# Your Operations Management Suite Tenant ID
|
|
23
|
-
config :tenant_id, :validate => :string
|
|
24
|
-
|
|
25
|
-
# Your data collection rule endpoint
|
|
26
|
-
config :data_collection_endpoint, :validate => :string
|
|
27
|
-
|
|
28
|
-
# Your data collection rule ID
|
|
29
|
-
config :dcr_immutable_id, :validate => :string
|
|
30
|
-
|
|
31
|
-
# Your dcr data stream name
|
|
32
|
-
config :dcr_stream_name, :validate => :string
|
|
33
|
-
|
|
34
|
-
# Subset of keys to send to the Azure Loganalytics workspace
|
|
35
|
-
config :key_names, :validate => :array, :default => []
|
|
36
|
-
|
|
37
|
-
# Max number of seconds to wait between flushes. Default 5
|
|
38
|
-
config :plugin_flush_interval, :validate => :number, :default => 5
|
|
39
|
-
|
|
40
|
-
# Factor for adding to the amount of messages sent
|
|
41
|
-
config :decrease_factor, :validate => :number, :default => 100
|
|
42
|
-
|
|
43
|
-
# This will trigger message amount resizing in a REST request to LA
|
|
44
|
-
config :amount_resizing, :validate => :boolean, :default => true
|
|
45
|
-
|
|
46
|
-
# Setting the default amount of messages sent
|
|
47
|
-
# it this is set with amount_resizing=false --> each message will have max_items
|
|
48
|
-
config :max_items, :validate => :number, :default => 2000
|
|
49
|
-
|
|
50
|
-
# Setting default proxy to be used for all communication with azure
|
|
51
|
-
config :proxy, :validate => :string
|
|
52
|
-
|
|
53
|
-
# Setting proxy_aad to be used for communicating with azure active directory service
|
|
54
|
-
config :proxy_aad, :validate => :string
|
|
55
|
-
|
|
56
|
-
# Setting proxy to be used for the LogAnalytics endpoint REST client
|
|
57
|
-
config :proxy_endpoint, :validate => :string
|
|
58
|
-
|
|
59
|
-
# This will set the amount of time given for retransmitting messages once sending is failed
|
|
60
|
-
config :retransmission_time, :validate => :number, :default => 10
|
|
61
|
-
|
|
62
|
-
# Compress the message body before sending to LA
|
|
63
|
-
config :compress_data, :validate => :boolean, :default => false
|
|
64
|
-
|
|
65
|
-
# Generate sample file from incoming events
|
|
66
|
-
config :create_sample_file, :validate => :boolean, :default => false
|
|
67
|
-
|
|
68
|
-
# Path where to place the sample file created
|
|
69
|
-
config :sample_file_path, :validate => :string
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
@
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
logstash_configuration
|
|
103
|
-
logstash_configuration.
|
|
104
|
-
logstash_configuration.
|
|
105
|
-
logstash_configuration.
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
1
|
+
# encoding: utf-8
|
|
2
|
+
require "logstash/outputs/base"
|
|
3
|
+
require "logstash/namespace"
|
|
4
|
+
require "logstash/sentinel_la/logstashLoganalyticsConfiguration"
|
|
5
|
+
require "logstash/sentinel_la/sampleFileCreator"
|
|
6
|
+
require "logstash/sentinel_la/logsSender"
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class LogStash::Outputs::MicrosoftSentinelOutput < LogStash::Outputs::Base
|
|
10
|
+
|
|
11
|
+
config_name "microsoft-sentinel-log-analytics-logstash-output-plugin"
|
|
12
|
+
|
|
13
|
+
# Stating that the output plugin will run in concurrent mode
|
|
14
|
+
concurrency :shared
|
|
15
|
+
|
|
16
|
+
# Your registered app ID
|
|
17
|
+
config :client_app_Id, :validate => :string
|
|
18
|
+
|
|
19
|
+
# The registered app's secret, required by Azure Loganalytics REST API
|
|
20
|
+
config :client_app_secret, :validate => :string
|
|
21
|
+
|
|
22
|
+
# Your Operations Management Suite Tenant ID
|
|
23
|
+
config :tenant_id, :validate => :string
|
|
24
|
+
|
|
25
|
+
# Your data collection rule endpoint
|
|
26
|
+
config :data_collection_endpoint, :validate => :string
|
|
27
|
+
|
|
28
|
+
# Your data collection rule ID
|
|
29
|
+
config :dcr_immutable_id, :validate => :string
|
|
30
|
+
|
|
31
|
+
# Your dcr data stream name
|
|
32
|
+
config :dcr_stream_name, :validate => :string
|
|
33
|
+
|
|
34
|
+
# Subset of keys to send to the Azure Loganalytics workspace
|
|
35
|
+
config :key_names, :validate => :array, :default => []
|
|
36
|
+
|
|
37
|
+
# Max number of seconds to wait between flushes. Default 5
|
|
38
|
+
config :plugin_flush_interval, :validate => :number, :default => 5
|
|
39
|
+
|
|
40
|
+
# Factor for adding to the amount of messages sent
|
|
41
|
+
config :decrease_factor, :validate => :number, :default => 100
|
|
42
|
+
|
|
43
|
+
# This will trigger message amount resizing in a REST request to LA
|
|
44
|
+
config :amount_resizing, :validate => :boolean, :default => true
|
|
45
|
+
|
|
46
|
+
# Setting the default amount of messages sent
|
|
47
|
+
# it this is set with amount_resizing=false --> each message will have max_items
|
|
48
|
+
config :max_items, :validate => :number, :default => 2000
|
|
49
|
+
|
|
50
|
+
# Setting default proxy to be used for all communication with azure
|
|
51
|
+
config :proxy, :validate => :string
|
|
52
|
+
|
|
53
|
+
# Setting proxy_aad to be used for communicating with azure active directory service
|
|
54
|
+
config :proxy_aad, :validate => :string
|
|
55
|
+
|
|
56
|
+
# Setting proxy to be used for the LogAnalytics endpoint REST client
|
|
57
|
+
config :proxy_endpoint, :validate => :string
|
|
58
|
+
|
|
59
|
+
# This will set the amount of time given for retransmitting messages once sending is failed
|
|
60
|
+
config :retransmission_time, :validate => :number, :default => 10
|
|
61
|
+
|
|
62
|
+
# Compress the message body before sending to LA
|
|
63
|
+
config :compress_data, :validate => :boolean, :default => false
|
|
64
|
+
|
|
65
|
+
# Generate sample file from incoming events
|
|
66
|
+
config :create_sample_file, :validate => :boolean, :default => false
|
|
67
|
+
|
|
68
|
+
# Path where to place the sample file created
|
|
69
|
+
config :sample_file_path, :validate => :string
|
|
70
|
+
|
|
71
|
+
# Used to specify the name of the Azure cloud that is being used. By default, the value is set to "AzureCloud", which
|
|
72
|
+
# is the public Azure cloud. However, you can specify a different Azure cloud if you are
|
|
73
|
+
# using a different environment, such as Azure Government or Azure China.
|
|
74
|
+
config :azure_cloud, :validate => :string
|
|
75
|
+
|
|
76
|
+
public
|
|
77
|
+
def register
|
|
78
|
+
@logstash_configuration= build_logstash_configuration()
|
|
79
|
+
|
|
80
|
+
# Validate configuration correctness
|
|
81
|
+
@logstash_configuration.validate_configuration()
|
|
82
|
+
|
|
83
|
+
@events_handler = @logstash_configuration.create_sample_file ?
|
|
84
|
+
LogStash::Outputs::MicrosoftSentinelOutputInternal::SampleFileCreator::new(@logstash_configuration) :
|
|
85
|
+
LogStash::Outputs::MicrosoftSentinelOutputInternal::LogsSender::new(@logstash_configuration)
|
|
86
|
+
end # def register
|
|
87
|
+
|
|
88
|
+
def multi_receive(events)
|
|
89
|
+
@events_handler.handle_events(events)
|
|
90
|
+
end # def multi_receive
|
|
91
|
+
|
|
92
|
+
def close
|
|
93
|
+
@events_handler.close
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
#private
|
|
97
|
+
private
|
|
98
|
+
|
|
99
|
+
# Building the logstash object configuration from the output configuration provided by the user
|
|
100
|
+
# Return LogstashLoganalyticsOutputConfiguration populated with the configuration values
|
|
101
|
+
def build_logstash_configuration()
|
|
102
|
+
logstash_configuration= LogStash::Outputs::MicrosoftSentinelOutputInternal::LogstashLoganalyticsOutputConfiguration::new(@client_app_Id, @client_app_secret, @tenant_id, @data_collection_endpoint, @dcr_immutable_id, @dcr_stream_name, @compress_data, @create_sample_file, @sample_file_path, @logger)
|
|
103
|
+
logstash_configuration.key_names = @key_names
|
|
104
|
+
logstash_configuration.plugin_flush_interval = @plugin_flush_interval
|
|
105
|
+
logstash_configuration.decrease_factor = @decrease_factor
|
|
106
|
+
logstash_configuration.amount_resizing = @amount_resizing
|
|
107
|
+
logstash_configuration.max_items = @max_items
|
|
108
|
+
logstash_configuration.proxy_aad = @proxy_aad || @proxy || ENV['http_proxy']
|
|
109
|
+
logstash_configuration.proxy_endpoint = @proxy_endpoint || @proxy || ENV['http_proxy']
|
|
110
|
+
logstash_configuration.retransmission_time = @retransmission_time
|
|
111
|
+
logstash_configuration.azure_cloud = @azure_cloud || "AzureCloud"
|
|
112
|
+
|
|
113
|
+
return logstash_configuration
|
|
114
|
+
end # def build_logstash_configuration
|
|
115
|
+
|
|
116
|
+
end # class LogStash::Outputs::MicrosoftSentinelOutput
|
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
# encoding: utf-8
|
|
2
2
|
require "logstash/sentinel_la/logstashLoganalyticsConfiguration"
|
|
3
|
-
require 'rest-client'
|
|
4
3
|
require 'json'
|
|
5
4
|
require 'openssl'
|
|
6
5
|
require 'base64'
|
|
7
6
|
require 'time'
|
|
7
|
+
require 'excon'
|
|
8
8
|
|
|
9
9
|
module LogStash; module Outputs; class MicrosoftSentinelOutputInternal
|
|
10
10
|
class LogAnalyticsAadTokenProvider
|
|
11
11
|
def initialize (logstashLoganalyticsConfiguration)
|
|
12
|
-
scope = CGI.escape("
|
|
13
|
-
@aad_uri =
|
|
12
|
+
scope = CGI.escape("#{logstashLoganalyticsConfiguration.get_monitor_endpoint}//.default")
|
|
13
|
+
@aad_uri = logstashLoganalyticsConfiguration.get_aad_endpoint
|
|
14
14
|
@token_request_body = sprintf("client_id=%s&scope=%s&client_secret=%s&grant_type=client_credentials", logstashLoganalyticsConfiguration.client_app_Id, scope, logstashLoganalyticsConfiguration.client_app_secret)
|
|
15
15
|
@token_request_uri = sprintf("%s/%s/oauth2/v2.0/token",@aad_uri, logstashLoganalyticsConfiguration.tenant_id)
|
|
16
16
|
@token_state = {
|
|
@@ -64,14 +64,13 @@ class LogAnalyticsAadTokenProvider
|
|
|
64
64
|
while true
|
|
65
65
|
begin
|
|
66
66
|
# Post REST request
|
|
67
|
-
response =
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
if (response.code == 200 || response.code == 201)
|
|
67
|
+
response = Excon.post(@token_request_uri, :body => @token_request_body, :headers => headers, :proxy => @logstashLoganalyticsConfiguration.proxy_aad, expects: [200, 201])
|
|
68
|
+
|
|
69
|
+
if (response.status == 200 || response.status == 201)
|
|
71
70
|
return JSON.parse(response.body)
|
|
72
71
|
end
|
|
73
|
-
rescue
|
|
74
|
-
@logger.error("
|
|
72
|
+
rescue Excon::Error::HTTPStatus => ex
|
|
73
|
+
@logger.error("Error while authenticating with AAD [#{ex.class}: '#{ex.response.status}', Response: '#{ex.response.body}']")
|
|
75
74
|
rescue Exception => ex
|
|
76
75
|
@logger.trace("Exception while authenticating with AAD API ['#{ex}']")
|
|
77
76
|
end
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
# encoding: utf-8
|
|
2
2
|
require "logstash/sentinel_la/version"
|
|
3
|
-
require 'rest-client'
|
|
4
3
|
require 'json'
|
|
5
4
|
require 'openssl'
|
|
6
5
|
require 'base64'
|
|
7
6
|
require 'time'
|
|
8
7
|
require 'rbconfig'
|
|
8
|
+
require 'excon'
|
|
9
9
|
|
|
10
10
|
module LogStash; module Outputs; class MicrosoftSentinelOutputInternal
|
|
11
11
|
class LogAnalyticsClient
|
|
@@ -14,7 +14,7 @@ require "logstash/sentinel_la/logstashLoganalyticsConfiguration"
|
|
|
14
14
|
require "logstash/sentinel_la/logAnalyticsAadTokenProvider"
|
|
15
15
|
|
|
16
16
|
|
|
17
|
-
def initialize
|
|
17
|
+
def initialize(logstashLoganalyticsConfiguration)
|
|
18
18
|
@logstashLoganalyticsConfiguration = logstashLoganalyticsConfiguration
|
|
19
19
|
@logger = @logstashLoganalyticsConfiguration.logger
|
|
20
20
|
|
|
@@ -22,28 +22,78 @@ require "logstash/sentinel_la/logAnalyticsAadTokenProvider"
|
|
|
22
22
|
@uri = sprintf("%s/dataCollectionRules/%s/streams/%s?api-version=%s",@logstashLoganalyticsConfiguration.data_collection_endpoint, @logstashLoganalyticsConfiguration.dcr_immutable_id, logstashLoganalyticsConfiguration.dcr_stream_name, la_api_version)
|
|
23
23
|
@aadTokenProvider=LogAnalyticsAadTokenProvider::new(logstashLoganalyticsConfiguration)
|
|
24
24
|
@userAgent = getUserAgent()
|
|
25
|
+
|
|
26
|
+
# Auto close connection after 60 seconds of inactivity
|
|
27
|
+
@connectionAutoClose = {
|
|
28
|
+
:last_use => Time.now,
|
|
29
|
+
:lock => Mutex.new,
|
|
30
|
+
:max_idel_time => 60,
|
|
31
|
+
:is_closed => true
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
@timer = Thread.new do
|
|
35
|
+
loop do
|
|
36
|
+
sleep @connectionAutoClose[:max_idel_time] / 2
|
|
37
|
+
if is_connection_stale?
|
|
38
|
+
@connectionAutoClose[:lock].synchronize do
|
|
39
|
+
if is_connection_stale?
|
|
40
|
+
reset_connection
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
|
|
25
48
|
end # def initialize
|
|
26
49
|
|
|
27
50
|
# Post the given json to Azure Loganalytics
|
|
28
51
|
def post_data(body)
|
|
29
52
|
raise ConfigError, 'no json_records' if body.empty?
|
|
53
|
+
response = nil
|
|
54
|
+
|
|
55
|
+
@connectionAutoClose[:lock].synchronize do
|
|
56
|
+
#close connection if its stale
|
|
57
|
+
if is_connection_stale?
|
|
58
|
+
reset_connection
|
|
59
|
+
end
|
|
60
|
+
if @connectionAutoClose[:is_closed]
|
|
61
|
+
open_connection
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
headers = get_header()
|
|
65
|
+
# Post REST request
|
|
66
|
+
response = @connection.request(method: :post, body: body, headers: headers)
|
|
67
|
+
@connectionAutoClose[:is_closed] = false
|
|
68
|
+
@connectionAutoClose[:last_use] = Time.now
|
|
69
|
+
end
|
|
70
|
+
return response
|
|
30
71
|
|
|
31
|
-
# Create REST request header
|
|
32
|
-
headers = get_header()
|
|
33
|
-
|
|
34
|
-
# Post REST request
|
|
35
|
-
|
|
36
|
-
return RestClient::Request.execute(method: :post, url: @uri, payload: body, headers: headers,
|
|
37
|
-
proxy: @logstashLoganalyticsConfiguration.proxy_endpoint, timeout: 120)
|
|
38
72
|
end # def post_data
|
|
39
73
|
|
|
40
74
|
# Static function to return if the response is OK or else
|
|
41
75
|
def self.is_successfully_posted(response)
|
|
42
|
-
return (response.
|
|
76
|
+
return (response.status >= 200 && response.status < 300 ) ? true : false
|
|
43
77
|
end # def self.is_successfully_posted
|
|
44
78
|
|
|
45
79
|
private
|
|
46
80
|
|
|
81
|
+
def open_connection
|
|
82
|
+
@connection = Excon.new(@uri, :persistent => true, :proxy => @logstashLoganalyticsConfiguration.proxy_endpoint,
|
|
83
|
+
expects: [200, 201, 202, 204, 206, 207, 208, 226, 300, 301, 302, 303, 304, 305, 306, 307, 308],
|
|
84
|
+
read_timeout: 240, write_timeout: 240, connect_timeout: 240)
|
|
85
|
+
@logger.trace("Connection to Azure LogAnalytics was opened.");
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def reset_connection
|
|
89
|
+
@connection.reset
|
|
90
|
+
@connectionAutoClose[:is_closed] = true
|
|
91
|
+
@logger.trace("Connection to Azure LogAnalytics was closed due to inactivity.");
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def is_connection_stale?
|
|
95
|
+
return Time.now - @connectionAutoClose[:last_use] > @connectionAutoClose[:max_idel_time] && !@connectionAutoClose[:is_closed]
|
|
96
|
+
end
|
|
47
97
|
# Create a header for the given length
|
|
48
98
|
def get_header()
|
|
49
99
|
# Getting an authorization token bearer (if the token is expired, the method will post a request to get a new authorization token)
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require "logstash/sentinel_la/logAnalyticsClient"
|
|
4
4
|
require "logstash/sentinel_la/logstashLoganalyticsConfiguration"
|
|
5
|
-
|
|
5
|
+
require "excon"
|
|
6
6
|
# LogStashAutoResizeBuffer class setting a resizable buffer which is flushed periodically
|
|
7
7
|
# The buffer resize itself according to Azure Loganalytics and configuration limitations
|
|
8
8
|
module LogStash; module Outputs; class MicrosoftSentinelOutputInternal
|
|
@@ -59,34 +59,35 @@ class LogStashEventsBatcher
|
|
|
59
59
|
return
|
|
60
60
|
else
|
|
61
61
|
@logger.trace("Rest client response ['#{response}']")
|
|
62
|
-
@logger.error("#{api_name} request failed. Error code: #{response.
|
|
62
|
+
@logger.error("#{api_name} request failed. Error code: #{response.pree} #{try_get_info_from_error_response(response)}")
|
|
63
63
|
end
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
64
|
+
rescue Excon::Error::HTTPStatus => ewr
|
|
65
|
+
response = ewr.response
|
|
66
|
+
@logger.trace("Exception in posting data to #{api_name}. Rest client response ['#{response}']. [amount_of_documents=#{amount_of_documents} request payload=#{call_payload}]")
|
|
67
|
+
@logger.error("Exception when posting data to #{api_name}. [Exception: '#{ewr.class}'] #{try_get_info_from_error_response(ewr.response)} [amount of documents=#{amount_of_documents}]'")
|
|
68
68
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
69
|
+
if ewr.class == Excon::Error::BadRequest
|
|
70
|
+
@logger.info("Not trying to resend since exception http code is 400")
|
|
71
|
+
return
|
|
72
|
+
elsif ewr.class == Excon::Error::RequestTimeout
|
|
73
|
+
force_retry = true
|
|
74
|
+
elsif ewr.class == Excon::Error::TooManyRequests
|
|
75
|
+
# throttling detected, backoff before resending
|
|
76
|
+
parsed_retry_after = response.data[:headers].include?('Retry-After') ? response.data[:headers]['Retry-After'].to_i : 0
|
|
77
|
+
seconds_to_sleep = parsed_retry_after > 0 ? parsed_retry_after : 30
|
|
73
78
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
79
|
+
#force another retry even if the next iteration of the loop will be after the retransmission_timeout
|
|
80
|
+
force_retry = true
|
|
81
|
+
end
|
|
82
|
+
rescue Excon::Error::Socket => ex
|
|
83
|
+
@logger.trace("Exception: '#{ex.class.name}]#{ex} in posting data to #{api_name}. [amount_of_documents=#{amount_of_documents}]'")
|
|
78
84
|
force_retry = true
|
|
79
|
-
|
|
80
|
-
#
|
|
81
|
-
parsed_retry_after = response.headers.include?(:retry_after) ? response.headers[:retry_after].to_i : 0
|
|
82
|
-
seconds_to_sleep = parsed_retry_after > 0 ? parsed_retry_after : 30
|
|
83
|
-
|
|
84
|
-
#force another retry even if the next iteration of the loop will be after the retransmission_timeout
|
|
85
|
+
rescue Excon::Error::Timeout => ex
|
|
86
|
+
@logger.trace("Exception: '#{ex.class.name}]#{ex} in posting data to #{api_name}. [amount_of_documents=#{amount_of_documents}]'")
|
|
85
87
|
force_retry = true
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
@logger.error("Exception in posting data to #{api_name}. [Exception: '#{ex}, amount of documents=#{amount_of_documents}]'")
|
|
88
|
+
rescue Exception => ex
|
|
89
|
+
@logger.trace("Exception in posting data to #{api_name}.[amount_of_documents=#{amount_of_documents} request payload=#{call_payload}]")
|
|
90
|
+
@logger.error("Exception in posting data to #{api_name}. [Exception: '[#{ex.class.name}]#{ex}, amount of documents=#{amount_of_documents}]'")
|
|
90
91
|
end
|
|
91
92
|
is_retry = true
|
|
92
93
|
@logger.info("Retrying transmission to #{api_name} in #{seconds_to_sleep} seconds.")
|
|
@@ -110,8 +111,8 @@ class LogStashEventsBatcher
|
|
|
110
111
|
def get_request_id_from_response(response)
|
|
111
112
|
output =""
|
|
112
113
|
begin
|
|
113
|
-
if !response.nil? && response.headers.include?(
|
|
114
|
-
output += response.
|
|
114
|
+
if !response.nil? && response.data[:headers].include?("x-ms-request-id")
|
|
115
|
+
output += response.data[:headers]["x-ms-request-id"]
|
|
115
116
|
end
|
|
116
117
|
rescue Exception => ex
|
|
117
118
|
@logger.debug("Error while getting reqeust id from success response headers: #{ex.display}")
|
|
@@ -124,12 +125,13 @@ class LogStashEventsBatcher
|
|
|
124
125
|
begin
|
|
125
126
|
output = ""
|
|
126
127
|
if !response.nil?
|
|
127
|
-
if response.headers.include?(
|
|
128
|
-
output += " [ms-error-code header: #{response.
|
|
128
|
+
if response.data[:headers].include?("x-ms-error-code")
|
|
129
|
+
output += " [ms-error-code header: #{response.data[:headers]["x-ms-error-code"]}]"
|
|
129
130
|
end
|
|
130
|
-
if response.headers.include?(
|
|
131
|
-
output += " [x-ms-request-id header: #{response.
|
|
131
|
+
if response.data[:headers].include?("x-ms-request-id")
|
|
132
|
+
output += " [x-ms-request-id header: #{response.data[:headers]["x-ms-request-id"]}]"
|
|
132
133
|
end
|
|
134
|
+
output += " [response body: #{response.data[:body]}]"
|
|
133
135
|
end
|
|
134
136
|
return output
|
|
135
137
|
rescue Exception => ex
|
|
@@ -23,6 +23,12 @@ class LogstashLoganalyticsOutputConfiguration
|
|
|
23
23
|
|
|
24
24
|
# Taking 4K safety buffer
|
|
25
25
|
@MAX_SIZE_BYTES = @loganalytics_api_data_limit - 10000
|
|
26
|
+
|
|
27
|
+
@azure_clouds = {
|
|
28
|
+
"AzureCloud" => {"aad" => "https://login.microsoftonline.com", "monitor" => "https://monitor.azure.com"},
|
|
29
|
+
"AzureChinaCloud" => {"aad" => "https://login.chinacloudapi.cn", "monitor" => "https://monitor.azure.cn"},
|
|
30
|
+
"AzureUSGovernment" => {"aad" => "https://login.microsoftonline.us", "monitor" => "https://monitor.azure.us"}
|
|
31
|
+
}.freeze
|
|
26
32
|
end
|
|
27
33
|
|
|
28
34
|
def validate_configuration()
|
|
@@ -68,6 +74,9 @@ class LogstashLoganalyticsOutputConfiguration
|
|
|
68
74
|
if @key_names.length > 500
|
|
69
75
|
raise ArgumentError, 'There are over 500 key names listed to be included in the events sent to Azure Loganalytics, which exceeds the limit of columns that can be define in each table in log analytics.'
|
|
70
76
|
end
|
|
77
|
+
if !@azure_clouds.key?(@azure_cloud)
|
|
78
|
+
raise ArgumentError, "The specified Azure cloud #{@azure_cloud} is not supported. Supported clouds are: #{@azure_clouds.keys.join(", ")}."
|
|
79
|
+
end
|
|
71
80
|
end
|
|
72
81
|
@logger.info("Azure Loganalytics configuration was found valid.")
|
|
73
82
|
# If all validation pass then configuration is valid
|
|
@@ -159,10 +168,6 @@ class LogstashLoganalyticsOutputConfiguration
|
|
|
159
168
|
@MIN_MESSAGE_AMOUNT
|
|
160
169
|
end
|
|
161
170
|
|
|
162
|
-
def max_items=(new_max_items)
|
|
163
|
-
@max_items = new_max_items
|
|
164
|
-
end
|
|
165
|
-
|
|
166
171
|
def key_names=(new_key_names)
|
|
167
172
|
@key_names = new_key_names
|
|
168
173
|
end
|
|
@@ -218,5 +223,21 @@ class LogstashLoganalyticsOutputConfiguration
|
|
|
218
223
|
def sample_file_path=(new_sample_file_path)
|
|
219
224
|
@sample_file_path = new_sample_file_path
|
|
220
225
|
end
|
|
226
|
+
|
|
227
|
+
def azure_cloud
|
|
228
|
+
@azure_cloud
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
def azure_cloud=(new_azure_cloud)
|
|
232
|
+
@azure_cloud = new_azure_cloud
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
def get_aad_endpoint
|
|
236
|
+
@azure_clouds[@azure_cloud]["aad"]
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
def get_monitor_endpoint
|
|
240
|
+
@azure_clouds[@azure_cloud]["monitor"]
|
|
241
|
+
end
|
|
221
242
|
end
|
|
222
243
|
end ;end ;end
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
module LogStash; module Outputs;
|
|
2
2
|
class MicrosoftSentinelOutputInternal
|
|
3
|
-
VERSION_INFO = [1, 1,
|
|
3
|
+
VERSION_INFO = [1, 1, 4].freeze
|
|
4
4
|
VERSION = VERSION_INFO.map(&:to_s).join('.').freeze
|
|
5
5
|
|
|
6
6
|
def self.version
|
|
7
7
|
VERSION
|
|
8
8
|
end
|
|
9
9
|
end
|
|
10
|
-
end;end
|
|
10
|
+
end;end
|
|
@@ -1,27 +1,27 @@
|
|
|
1
|
-
require File.expand_path('../lib/logstash/sentinel_la/version', __FILE__)
|
|
2
|
-
|
|
3
|
-
Gem::Specification.new do |s|
|
|
4
|
-
s.name = 'microsoft-sentinel-log-analytics-logstash-output-plugin'
|
|
5
|
-
s.version = LogStash::Outputs::MicrosoftSentinelOutputInternal::VERSION
|
|
6
|
-
s.authors = ["Microsoft Sentinel"]
|
|
7
|
-
s.email = 'AzureSentinel@microsoft.com'
|
|
8
|
-
s.summary = %q{Microsoft Sentinel provides a new output plugin for Logstash. Use this output plugin to send any log via Logstash to the Microsoft Sentinel/Log Analytics workspace. This is done with the Log Analytics DCR-based API.}
|
|
9
|
-
s.description = s.summary
|
|
10
|
-
s.homepage = "https://github.com/Azure/Azure-Sentinel"
|
|
11
|
-
s.licenses = ["MIT"]
|
|
12
|
-
s.require_paths = ["lib"]
|
|
13
|
-
|
|
14
|
-
# Files
|
|
15
|
-
s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
|
|
16
|
-
# Tests
|
|
17
|
-
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
|
18
|
-
|
|
19
|
-
# Special flag to let us know this is actually a logstash plugin
|
|
20
|
-
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
|
|
21
|
-
|
|
22
|
-
# Gem dependencies
|
|
23
|
-
s.add_runtime_dependency "
|
|
24
|
-
s.add_runtime_dependency "logstash-
|
|
25
|
-
s.add_runtime_dependency "
|
|
26
|
-
s.add_development_dependency "logstash-devutils"
|
|
27
|
-
end
|
|
1
|
+
require File.expand_path('../lib/logstash/sentinel_la/version', __FILE__)
|
|
2
|
+
|
|
3
|
+
Gem::Specification.new do |s|
|
|
4
|
+
s.name = 'microsoft-sentinel-log-analytics-logstash-output-plugin'
|
|
5
|
+
s.version = LogStash::Outputs::MicrosoftSentinelOutputInternal::VERSION
|
|
6
|
+
s.authors = ["Microsoft Sentinel"]
|
|
7
|
+
s.email = 'AzureSentinel@microsoft.com'
|
|
8
|
+
s.summary = %q{Microsoft Sentinel provides a new output plugin for Logstash. Use this output plugin to send any log via Logstash to the Microsoft Sentinel/Log Analytics workspace. This is done with the Log Analytics DCR-based API.}
|
|
9
|
+
s.description = s.summary
|
|
10
|
+
s.homepage = "https://github.com/Azure/Azure-Sentinel"
|
|
11
|
+
s.licenses = ["MIT"]
|
|
12
|
+
s.require_paths = ["lib"]
|
|
13
|
+
|
|
14
|
+
# Files
|
|
15
|
+
s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
|
|
16
|
+
# Tests
|
|
17
|
+
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
|
18
|
+
|
|
19
|
+
# Special flag to let us know this is actually a logstash plugin
|
|
20
|
+
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
|
|
21
|
+
|
|
22
|
+
# Gem dependencies
|
|
23
|
+
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
|
24
|
+
s.add_runtime_dependency "logstash-codec-plain"
|
|
25
|
+
s.add_runtime_dependency "excon", ">= 0.88.0", "< 1.0.0"
|
|
26
|
+
s.add_development_dependency "logstash-devutils"
|
|
27
|
+
end
|