microsoft-sentinel-log-analytics-logstash-output-plugin 1.1.0

data/lib/logstash/sentinel_la/logsSender.rb

@@ -0,0 +1,47 @@
+# encoding: utf-8
+require "logstash/sentinel_la/logstashLoganalyticsConfiguration"
+require "logstash/sentinel_la/eventsHandler"
+require "logstash/sentinel_la/logStashAutoResizeBuffer"
+require "logstash/sentinel_la/logStashCompressedStream"
+
+module LogStash; module Outputs; class MicrosoftSentinelOutputInternal
+        class LogsSender < EventsHandler
+
+        @thread_batch_map
+
+        def initialize(logstashLogAnalyticsConfiguration)
+          @thread_batch_map = Concurrent::Hash.new
+          @logstashLogAnalyticsConfiguration = logstashLogAnalyticsConfiguration
+          @logger = logstashLogAnalyticsConfiguration.logger
+          super
+        end
+
+        def handle_events(events)
+          t = Thread.current
+          
+          unless @thread_batch_map.include?(t)
+            @thread_batch_map[t] = @logstashLogAnalyticsConfiguration.compress_data ? 
+                                      LogStashCompressedStream::new(@logstashLogAnalyticsConfiguration) :
+                                      LogStashAutoResizeBuffer::new(@logstashLogAnalyticsConfiguration)
+          end
+
+          events.each do |event|
+            # creating document from event
+            document = create_event_document(event)
+
+            # Skip if document doesn't contain any items
+            next if (document.keys).length < 1
+
+            @logger.trace("Adding event document - " + event.to_s)
+            @thread_batch_map[t].batch_event_document(document)
+          end
+        end
+
+        def close
+          @thread_batch_map.each { |thread_id, batcher|
+            batcher.close
+          }
+        end
+
+      end
+end; end; end;

data/lib/logstash/sentinel_la/logstashLoganalyticsConfiguration.rb

@@ -0,0 +1,222 @@
+# encoding: utf-8
+module LogStash; module Outputs; class MicrosoftSentinelOutputInternal
+class LogstashLoganalyticsOutputConfiguration
+    def initialize(client_app_Id, client_app_secret, tenant_id, data_collection_endpoint, dcr_immutable_id, dcr_stream_name, compress_data, create_sample_file, sample_file_path, logger)
+		@client_app_Id = client_app_Id
+        @client_app_secret = client_app_secret
+        @tenant_id = tenant_id
+        @data_collection_endpoint = data_collection_endpoint
+        @dcr_immutable_id = dcr_immutable_id
+        @dcr_stream_name = dcr_stream_name
+        @logger = logger
+	    @compress_data = compress_data
+	    @create_sample_file = create_sample_file
+	    @sample_file_path = sample_file_path
+
+	# Delay between each resending of a message
+        @RETRANSMISSION_DELAY = 2
+        @MIN_MESSAGE_AMOUNT = 100
+        # Maximum of 1 MB per post to Log Analytics Data Collector API V2. 
+        # This is a size limit for a single post. 
+        # If the data from a single post that exceeds 1 MB, you should split it.
+        @loganalytics_api_data_limit = 1 * 1024 * 1024
+
+        # Taking 4K safety buffer
+        @MAX_SIZE_BYTES = @loganalytics_api_data_limit - 10000
+    end
+	
+	def validate_configuration()
+      if @create_sample_file
+          begin
+              if @sample_file_path.nil?
+                  print_missing_parameter_message_and_raise("sample_file_path")
+              end
+              if @sample_file_path.strip == ""
+                  raise ArgumentError, "The setting sample_file_path cannot be empty"
+              end
+              begin
+                  file = java.io.File.new(@sample_file_path)
+                  if !file.exists
+                      raise "Path not exists"
+                  end
+              rescue Exception
+                  raise ArgumentError, "The path #{@sample_file_path} does not exist."
+              end
+          end
+      else
+          required_configs = { "client_app_Id" => @client_app_Id,
+                              "client_app_secret" => @client_app_secret,
+                              "tenant_id" => @tenant_id,
+                              "data_collection_endpoint" => @data_collection_endpoint,
+                              "dcr_immutable_id" => @dcr_immutable_id,
+                              "dcr_stream_name" => @dcr_stream_name }
+          required_configs.each { |name, conf|
+              if conf.nil?
+                  print_missing_parameter_message_and_raise(name)
+              end
+              if conf.empty?
+                  raise ArgumentError, "Malformed configuration , the following arguments can not be null or empty.[client_app_Id, client_app_secret, tenant_id, data_collection_endpoint, dcr_immutable_id, dcr_stream_name]"
+              end
+          }
+
+          if @retransmission_time < 0
+              raise ArgumentError, "retransmission_time must be a positive integer."
+          end
+          if @max_items < @MIN_MESSAGE_AMOUNT
+              raise ArgumentError, "Setting max_items to value must be greater then #{@MIN_MESSAGE_AMOUNT}."
+          end
+          if @key_names.length > 500
+              raise ArgumentError, 'There are over 500 key names listed to be included in the events sent to Azure Loganalytics, which exceeds the limit of columns that can be define in each table in log analytics.'
+          end
+      end
+        @logger.info("Azure Loganalytics configuration was found valid.")
+        # If all validation pass then configuration is valid
+        return  true
+    end # def validate_configuration
+
+
+    def print_missing_parameter_message_and_raise(param_name)
+        @logger.error("Missing a required setting for the microsoft-sentinel-log-analytics-logstash-output-plugin output plugin:
+  output {
+    microsoft-sentinel-log-analytics-logstash-output-plugin {
+      #{param_name} => # SETTING MISSING
+      ...
+    }
+  }
+")
+        raise ArgumentError, "The setting #{param_name} is required."
+    end
+
+    def RETRANSMISSION_DELAY
+        @RETRANSMISSION_DELAY
+    end
+
+    def MAX_SIZE_BYTES
+        @MAX_SIZE_BYTES
+    end
+
+    def amount_resizing
+        @amount_resizing
+    end
+
+    def retransmission_time
+        @retransmission_time
+    end
+
+    def proxy_aad
+        @proxy_aad
+    end
+
+    def proxy_endpoint
+        @proxy_endpoint
+    end
+
+    def logger
+        @logger
+    end
+
+    def decrease_factor
+        @decrease_factor
+    end
+	
+	def client_app_Id
+        @client_app_Id
+    end
+
+    def client_app_secret
+        @client_app_secret
+    end
+
+    def tenant_id
+        @tenant_id
+    end
+
+    def data_collection_endpoint
+        @data_collection_endpoint
+    end
+
+    def dcr_immutable_id
+        @dcr_immutable_id
+    end
+
+    def dcr_stream_name
+        @dcr_stream_name
+    end
+
+    def key_names
+        @key_names
+    end
+
+    def max_items
+        @max_items
+    end
+
+    def plugin_flush_interval
+        @plugin_flush_interval
+    end
+
+    def MIN_MESSAGE_AMOUNT
+        @MIN_MESSAGE_AMOUNT
+    end
+    
+    def max_items=(new_max_items)
+        @max_items = new_max_items
+    end
+
+    def key_names=(new_key_names)
+        @key_names = new_key_names
+    end
+
+    def plugin_flush_interval=(new_plugin_flush_interval)
+        @plugin_flush_interval = new_plugin_flush_interval
+    end
+    
+    def decrease_factor=(new_decrease_factor)
+        @decrease_factor = new_decrease_factor
+    end
+
+    def amount_resizing=(new_amount_resizing)
+        @amount_resizing = new_amount_resizing
+    end
+
+    def max_items=(new_max_items)
+        @max_items = new_max_items
+    end
+    
+    def proxy_aad=(new_proxy_aad)
+        @proxy_aad = new_proxy_aad
+    end
+
+    def proxy_endpoint=(new_proxy_endpoint)
+        @proxy_endpoint = new_proxy_endpoint
+    end
+
+    def retransmission_time=(new_retransmission_time)
+        @retransmission_time = new_retransmission_time
+    end
+
+    def compress_data
+        @compress_data
+    end
+
+    def compress_data=(new_compress_data)
+        @compress_data = new_compress_data
+    end
+
+    def create_sample_file
+        @create_sample_file
+    end
+
+    def create_sample_file=(new_create_sample_file)
+        @create_sample_file = new_create_sample_file
+    end
+
+    def sample_file_path
+        @sample_file_path
+    end
+
+    def sample_file_path=(new_sample_file_path)
+        @sample_file_path = new_sample_file_path
+    end
+end
+end ;end ;end 

data/lib/logstash/sentinel_la/sampleFileCreator.rb

@@ -0,0 +1,61 @@
+# encoding: utf-8
+require "logstash/sentinel_la/logstashLoganalyticsConfiguration"
+require "logstash/sentinel_la/eventsHandler"
+
+module LogStash
+  module Outputs
+    class MicrosoftSentinelOutputInternal
+      class SampleFileCreator < EventsHandler
+
+        def initialize(logstashLogAnalyticsConfiguration)
+          @events_buffer = Concurrent::Array.new
+          @maximum_events_to_sample = 10
+          @was_file_written = false
+          @writing_mutex = Mutex.new
+          super
+        end
+
+        def handle_events(events)
+          events.each do |event|
+            if !@was_file_written
+              filtered_event = create_event_document(event)
+              @events_buffer.push(filtered_event)
+            end
+          end
+          try_writing_events_to_file
+        end
+
+        def close
+          try_writing_events_to_file(true)
+        end
+
+        def try_writing_events_to_file(force = false)
+          if @was_file_written
+            return
+          end
+
+          @writing_mutex.synchronize do
+            #check if file was written during the wait
+            if @was_file_written ||
+               @events_buffer.length == 0 ||
+               (@events_buffer.length <= @maximum_events_to_sample && !force) 
+              return
+            end
+
+            output_path = @logstashLogAnalyticsConfiguration.sample_file_path
+            output_file_name = "sampleFile#{Time.now.to_i}.json"
+            file = java.io.File.new(output_path,output_file_name)
+            fw = java.io.FileWriter.new(file)
+            fw.write(@events_buffer.take(@maximum_events_to_sample).to_json)
+            fw.flush
+            fw.close
+
+            @was_file_written = true
+            @logger.info("Sample file was written in path: #{file.getAbsolutePath}")
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/logstash/sentinel_la/version.rb

@@ -0,0 +1,10 @@
+module LogStash; module Outputs;
+class MicrosoftSentinelOutputInternal
+  VERSION_INFO = [1, 1, 0].freeze
+  VERSION = VERSION_INFO.map(&:to_s).join('.').freeze
+
+  def self.version
+    VERSION
+  end
+end
+end;end

data/microsoft-sentinel-log-analytics-logstash-output-plugin.gemspec

@@ -0,0 +1,27 @@
+require File.expand_path('../lib/logstash/sentinel_la/version', __FILE__)
+
+Gem::Specification.new do |s|
+  s.name = 'microsoft-sentinel-log-analytics-logstash-output-plugin'
+  s.version = LogStash::Outputs::MicrosoftSentinelOutputInternal::VERSION
+  s.authors = ["Microsoft Sentinel"]
+  s.email   = 'AzureSentinel@microsoft.com'
+  s.summary = %q{Microsoft Sentinel provides a new output plugin for Logstash. Use this output plugin to send any log via Logstash to the Microsoft Sentinel/Log Analytics workspace. This is done with the Log Analytics DCR-based API.}
+  s.description = s.summary
+  s.homepage = "https://github.com/Azure/Azure-Sentinel"
+  s.licenses = ["MIT"]
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "rest-client", ">= 2.1.0"
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency "logstash-codec-plain"
+  s.add_development_dependency "logstash-devutils"
+end

metadata

@@ -0,0 +1,127 @@
+--- !ruby/object:Gem::Specification
+name: microsoft-sentinel-log-analytics-logstash-output-plugin
+version: !ruby/object:Gem::Version
+  version: 1.1.0
+platform: ruby
+authors:
+- Microsoft Sentinel
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2023-07-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+- !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Microsoft Sentinel provides a new output plugin for Logstash. Use this
+  output plugin to send any log via Logstash to the Microsoft Sentinel/Log Analytics
+  workspace. This is done with the Log Analytics DCR-based API.
+email: AzureSentinel@microsoft.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- Gemfile
+- LICENSE
+- README.md
+- lib/logstash/outputs/microsoft-sentinel-log-analytics-logstash-output-plugin.rb
+- lib/logstash/sentinel_la/customSizeBasedBuffer.rb
+- lib/logstash/sentinel_la/eventsHandler.rb
+- lib/logstash/sentinel_la/logAnalyticsAadTokenProvider.rb
+- lib/logstash/sentinel_la/logAnalyticsClient.rb
+- lib/logstash/sentinel_la/logStashAutoResizeBuffer.rb
+- lib/logstash/sentinel_la/logStashCompressedStream.rb
+- lib/logstash/sentinel_la/logStashEventsBatcher.rb
+- lib/logstash/sentinel_la/logsSender.rb
+- lib/logstash/sentinel_la/logstashLoganalyticsConfiguration.rb
+- lib/logstash/sentinel_la/sampleFileCreator.rb
+- lib/logstash/sentinel_la/version.rb
+- microsoft-sentinel-log-analytics-logstash-output-plugin.gemspec
+homepage: https://github.com/Azure/Azure-Sentinel
+licenses:
+- MIT
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: output
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.6
+signing_key:
+specification_version: 4
+summary: Microsoft Sentinel provides a new output plugin for Logstash. Use this output
+  plugin to send any log via Logstash to the Microsoft Sentinel/Log Analytics workspace.
+  This is done with the Log Analytics DCR-based API.
+test_files: []