microsoft-logstash-output-azure-loganalytics 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: ce0904868fbfab9bfbc3386a101957a436f9d8c1fde87450ffd2a3a0d9c78574
+  data.tar.gz: 1a69ab08f7bd6a3ea6ee639dfddd5c69333ef53a4dcfd6a08fccd65eb4bd26e5
+SHA512:
+  metadata.gz: 262dd9c81132c1ad32bd5ccab843722ee42cc3d8ebc5cf417d99540c13c49f42f5eee2f987d29e31d03a471feda3a5dee4318f302ebace577333144f1744dd6b
+  data.tar.gz: 20306ed12e1dfde8f1a63caeca7c7b585ed2b50ab8de437fb7d1c37c66fedb2fce6c8d3080e73f2e3c49b276f27f23d3df17c68dffd0a82ebfe4473ad2ea56be

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+## 1.0.0
+
+* Initial release for output plugin for logstash to Azure Sentinel(Log Analytics)

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/README.md

@@ -0,0 +1,69 @@
+# Azure Log Analytics output plugin for Logstash 
+
+Azure Sentinel provides a new output plugin for Logstash. Using this output plugin, you will be able to send any log you want using Logstash to the Azure Sentinel/Log Analytics workspace
+Today you will be able to send messages to custom logs table that you will define in the output plugin. 
+Getting started with Logstash 
+
+Azure Sentinel output plugin uses the rest API integration to Log Analytics, in order to ingest the logs into custom logs tables [What are custom logs tables] 
+
+Plugin version: v1.0.0 
+Released on: 2020-04-30 
+
+## Installation
+
+Azure Sentinel provides Logstash output plugin to Log analytics workspace. 
+Install the microsoft-logstash-output-azure-loganalytics, use Logstash Working with plugins document. 
+For offline setup follow Logstash Offline Plugin Management instruction. 
+
+## Configuration
+
+in your Logstash configuration file, add the Azure Sentinel output plugin to the configuration with following values: 
+- workspace_id – your workspace ID guid 
+- workspace_key – your workspace primary key guid 
+- custom_log_table_name – table name, in which the logs will be ingested, limited to one table, the log table will be presented in the logs blade under the custom logs label, with a _CL suffix. 
+- endpoint – Optional field by default set as log analytics endpoint.  
+- time_generated_field – Optional field, this property is used to override the default TimeGenerated field in Log Analytics. Populate this property with the name of the sent data time field. 
+- key_names – list of Log analytics output schema fields. 
+- plugin_flash_interval – Optional filed, define the maximal time difference (in seconds) between sending two messages to Log Analytics. 
+- Max_items – Optional field, 2000 by default. this parameter will control the maximum batch size. This value will be changed if the user didn’t specify “amount_resizing = false” in the configuration. 
+
+Note: View the GitHub to learn more about the sent message’s configuration, performance settings and mechanism 
+
+## Tests
+
+Here is an example configuration who parse Syslog incoming data into a custom table named "logstashCustomTableName".
+
+### Example Configuration
+<u>Configuration</u>
+```
+input {
+  tcp {
+    port => 514
+    type => syslog
+  }
+}
+
+filter {
+    grok {
+      match => { "message" => "<%{NUMBER:PRI}>1 (?<TIME_TAG>[0-9]{4}-[0-9]{1,2}-[0-9]{1,2}T[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2})[^ ]* (?<HOSTNAME>[^ ]*) %{GREEDYDATA:MSG}" }
+    }
+}
+
+output {
+        microsoft-logstash-output-azure-loganalytics {
+                workspace_id => "<WS_ID>"
+                workspace_key => "${WS_KEY}"
+                custom_log_table_name => "logstashCustomTableName"
+                key_names => ['PRI','TIME_TAG','HOSTNAME','MSG']
+                plugin_flush_interval => 5
+        }
+}
+```
+
+Now you are able to run logstash with the example configuration and send mock data using the 'logger' command.
+
+For example: 
+```
+logger -p local4.warn -t CEF: "0|Microsoft|Device|cef-test|example|data|1|here is some more data for the example" -P 514 -d -n 127.0.0.1
+
+```

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/lib/logstash/logAnalyticsClient/logAnalyticsClient.rb

@@ -0,0 +1,72 @@
+# encoding: utf-8
+require "logstash/logAnalyticsClient/logstashLoganalyticsConfiguration"
+require 'rest-client'
+require 'json'
+require 'openssl'
+require 'base64'
+require 'time'
+
+class LogAnalyticsClient
+  API_VERSION = '2016-04-01'.freeze
+
+  def initialize (logstashLoganalyticsConfiguration)
+    @logstashLoganalyticsConfiguration = logstashLoganalyticsConfiguration
+    set_proxy(@logstashLoganalyticsConfiguration.proxy)
+    @uri = sprintf("https://%s.%s/api/logs?api-version=%s", @logstashLoganalyticsConfiguration.workspace_id, @logstashLoganalyticsConfiguration.endpoint, API_VERSION)
+  end # def initialize
+
+
+  # Post the given json to Azure Loganalytics
+  def post_data(body)
+    raise ConfigError, 'no json_records' if body.empty?
+    # Create REST request header
+    header = get_header(body.bytesize)
+    # Post REST request 
+    response = RestClient.post(@uri, body, header)
+
+    return response
+  end # def post_data
+
+  private 
+
+  # Create a header for the given length 
+  def get_header(body_bytesize_length)
+    # We would like each request to be sent with the current time
+    date = rfc1123date()
+
+    return {
+      'Content-Type' => 'application/json',
+      'Authorization' => signature(date, body_bytesize_length),
+      'Log-Type' => @logstashLoganalyticsConfiguration.custom_log_table_name,
+      'x-ms-date' => date,
+      'time-generated-field' =>  @logstashLoganalyticsConfiguration.time_generated_field,
+      'x-ms-AzureResourceId' => @logstashLoganalyticsConfiguration.azure_resource_id
+    }
+  end # def get_header
+
+  # Setting proxy for the REST client.
+  # This option is not used in the output plugin and will be used 
+  #  
+  def set_proxy(proxy='')
+    RestClient.proxy = proxy.empty? ? ENV['http_proxy'] : proxy
+  end # def set_proxy
+
+  # Return the current data 
+  def rfc1123date()
+    current_time = Time.now
+    
+    return current_time.httpdate()
+  end # def rfc1123date
+
+  def signature(date, body_bytesize_length)
+    sigs = sprintf("POST\n%d\napplication/json\nx-ms-date:%s\n/api/logs", body_bytesize_length, date)
+    utf8_sigs = sigs.encode('utf-8')
+    decoded_shared_key = Base64.decode64(@logstashLoganalyticsConfiguration.workspace_key)
+    hmac_sha256_sigs = OpenSSL::HMAC.digest(OpenSSL::Digest.new('sha256'), decoded_shared_key, utf8_sigs)
+    encoded_hash = Base64.encode64(hmac_sha256_sigs)
+    authorization = sprintf("SharedKey %s:%s", @logstashLoganalyticsConfiguration.workspace_id, encoded_hash)
+    
+    return authorization
+  end # def signature
+
+end # end of class

data/lib/logstash/logAnalyticsClient/logStashAutoResizeBuffer.rb

@@ -0,0 +1,142 @@
+# encoding: utf-8
+require "stud/buffer"
+require "logstash/logAnalyticsClient/logAnalyticsClient"
+require "stud/buffer"
+require "logstash/logAnalyticsClient/logstashLoganalyticsConfiguration"
+
+# LogStashAutoResizeBuffer class setting a resizable buffer which is flushed periodically
+# The buffer resize itself according to Azure Loganalytics  and configuration limitations
+class LogStashAutoResizeBuffer
+    include Stud::Buffer
+
+    def initialize(logstashLoganalyticsConfiguration)
+        @logstashLoganalyticsConfiguration = logstashLoganalyticsConfiguration
+        @logger = @logstashLoganalyticsConfiguration.logger
+        @client=LogAnalyticsClient::new(logstashLoganalyticsConfiguration)
+        buffer_initialize(
+          :max_items => logstashLoganalyticsConfiguration.max_items,
+          :max_interval => logstashLoganalyticsConfiguration.plugin_flush_interval,
+          :logger => @logstashLoganalyticsConfiguration.logger
+        )
+    end # initialize
+
+    # Public methods
+    public
+
+    # Adding an event document into the buffer
+    def add_event_document(event_document)
+        buffer_receive(event_document)
+    end # def add_event_document
+
+    # Flushing all buffer content to Azure Loganalytics.
+    # Called from Stud::Buffer#buffer_flush when there are events to flush
+    def flush (documents, close=false)
+        # Skip in case there are no candidate documents to deliver
+        if documents.length < 1
+            @logger.warn("No documents in batch for log type #{@logstashLoganalyticsConfiguration.custom_log_table_name}. Skipping")
+            return
+        end
+
+        # We send Json in the REST request 
+        documents_json = documents.to_json
+        # Setting resizing to true will cause changing the max size
+        if @logstashLoganalyticsConfiguration.amount_resizing == true
+            # Resizing the amount of messages according to size of message received and amount of messages
+            change_message_limit_size(documents.length, documents_json.bytesize)
+        end
+        send_message_to_loganalytics(documents_json, documents.length)
+    end # def flush
+
+    # Private methods 
+    private 
+
+    # Send documents_json to Azure Loganalytics  
+    def send_message_to_loganalytics(documents_json, amount_of_documents)
+        begin
+            @logger.debug("Posting log batch (log count: #{amount_of_documents}) as log type #{@logstashLoganalyticsConfiguration.custom_log_table_name} to DataCollector API.")
+            response = @client.post_data(documents_json)
+            if is_successfully_posted(response)
+                @logger.info("Successfully posted #{amount_of_documents} logs into custom log analytics table[#{@logstashLoganalyticsConfiguration.custom_log_table_name}].")
+            else
+                @logger.error("DataCollector API request failure: error code: #{response.code}, data=>" + (documents.to_json).to_s)
+                resend_message(documents_json, amount_of_documents, @logstashLoganalyticsConfiguration.retransmission_time)
+            end
+            rescue Exception => ex
+                @logger.error("Exception in posting data to Azure Loganalytics.\n[Exception: '#{ex}]'")
+                @logger.trace("Exception in posting data to Azure Loganalytics.[amount_of_documents=#{amount_of_documents} documents=#{documents_json}]")
+                resend_message(documents_json, amount_of_documents, @logstashLoganalyticsConfiguration.retransmission_time)
+            end
+    end # end send_message_to_loganalytics
+
+    # If sending the message toAzure Loganalytics fails we would like to retry to send it again.
+    # We would like to do it until we reached to the duration 
+    def resend_message(documents_json, amount_of_documents, remaining_duration)
+        if remaining_duration > 0
+            @logger.info("Resending #{amount_of_documents} documents as log type #{@logstashLoganalyticsConfiguration.custom_log_table_name} to DataCollector API in #{@logstashLoganalyticsConfiguration.RETRANSMITION_DELAY} seconds.")
+            sleep @logstashLoganalyticsConfiguration.RETRANSMISSION_DELAY
+            begin
+                response = @client.post_data(documents_json)
+                if is_successfully_posted(response)
+                    @logger.info("Successfully sent #{amount_of_documents} logs into custom log analytics table[#{@logstashLoganalyticsConfiguration.custom_log_table_name}] after resending.")
+                else
+                    @logger.debug("Resending #{amount_of_documents} documents failed, will try to resend for #{(remaining_duration - @logstashLoganalyticsConfiguration.RETRANSMITION_DELAY)}")
+                    resend_message(documents_json, amount_of_documents, (remaining_duration - @logstashLoganalyticsConfiguration.RETRANSMITION_DELAY))
+                end
+            rescue Exception => ex
+                @logger.debug("Resending #{amount_of_documents} documents failed, will try to resend for #{(remaining_duration - @logstashLoganalyticsConfiguration.RETRANSMITION_DELAY)}")
+                resend_message(documents_json, amount_of_documents, (remaining_duration - @logstashLoganalyticsConfiguration.RETRANSMITION_DELAY))
+            end
+        else 
+            @logger.error("Could not resend #{amount_of_documents} documents, message is dropped.")
+            @logger.trace("Documents (#{amount_of_documents}) dropped. [documents_json=#{documents_json}]")
+        end
+    end # def resend_message
+
+    # We would like to change the amount of messages in the buffer (change_max_size)
+    # We change the amount according to the Azure Loganalytics limitation and the amount of messages inserted to the buffer
+    # in one sending window.
+    # Meaning that if we reached the max amount we would like to increase it.
+    # Else we would like to decrease it(to reduce latency for messages)
+    def change_message_limit_size(amount_of_documents, documents_byte_size)
+        new_buffer_size = @logstashLoganalyticsConfiguration.max_items
+        average_document_size = documents_byte_size / amount_of_documents
+        # If window is full we need to increase it 
+        # "amount_of_documents" can be greater since buffer is not synchronized meaning 
+        # that flush can occur after limit was reached.
+        if  amount_of_documents >= @logstashLoganalyticsConfiguration.max_items
+            # if doubling the size wouldn't exceed the API limit
+            if ((2 * @logstashLoganalyticsConfiguration.max_items) * average_document_size) < @logstashLoganalyticsConfiguration.MAX_SIZE_BYTES
+                new_buffer_size = 2 * @logstashLoganalyticsConfiguration.max_items
+            else
+                new_buffer_size = (@logstashLoganalyticsConfiguration.MAX_SIZE_BYTES / average_document_size) -1000
+            end
+
+        # We would like to decrease the window but not more then the MIN_MESSAGE_AMOUNT
+        # We are trying to decrease it slowly to be able to send as much messages as we can in one window 
+        elsif amount_of_documents < @logstashLoganalyticsConfiguration.max_items and  @logstashLoganalyticsConfiguration.max_items != [(@logstashLoganalyticsConfiguration.max_items - @logstashLoganalyticsConfiguration.decrease_factor) ,@logstashLoganalyticsConfiguration.MIN_MESSAGE_AMOUNT].max
+            new_buffer_size = [(@logstashLoganalyticsConfiguration.max_items - @logstashLoganalyticsConfiguration.decrease_factor) ,@logstashLoganalyticsConfiguration.MIN_MESSAGE_AMOUNT].max
+        end
+
+        change_buffer_size(new_buffer_size)
+    end # def change_message_limit_size
+
+    # Receiving new_size as the new max buffer size.
+    # Changing both the buffer, the configuration and logging as necessary
+    def change_buffer_size(new_size)
+        # Change buffer size only if it's needed(new size)
+        if @buffer_config[:max_items] != new_size
+            old_buffer_size = @buffer_config[:max_items]
+            @buffer_config[:max_items] = new_size
+            @logstashLoganalyticsConfiguration.max_items = new_size
+            @logger.info("Changing buffer size.[configuration='#{old_buffer_size}' , new_size='#{new_size}']")
+        else
+            @logger.info("Buffer size wasn't changed.[configuration='#{old_buffer_size}' , new_size='#{new_size}']")
+        end
+    end # def change_buffer_size
+
+    # Function to return if the response is OK or else
+    def is_successfully_posted(response)
+        return (response.code == 200) ? true : false
+      end # def is_successfully_posted
+
+end # LogStashAutoResizeBuffer

data/lib/logstash/logAnalyticsClient/logstashLoganalyticsConfiguration.rb

@@ -0,0 +1,158 @@
+# encoding: utf-8
+class LogstashLoganalyticsOutputConfiguration
+    def initialize(workspace_id, workspace_key, custom_log_table_name, logger)
+        @workspace_id = workspace_id
+        @workspace_key = workspace_key
+        @custom_log_table_name = custom_log_table_name
+        @logger = logger
+
+        # Delay between each resending of a message
+        @RETRANSMISSION_DELAY = 2
+        @MIN_MESSAGE_AMOUNT = 100
+        # Maximum of 30 MB per post to Log Analytics Data Collector API. 
+        # This is a size limit for a single post. 
+        # If the data from a single post that exceeds 30 MB, you should split it.
+        @loganalytics_api_data_limit = 30 * 1000 * 1000
+
+        # Taking 4K safety buffer
+        @MAX_SIZE_BYTES = @loganalytics_api_data_limit - 10000
+    end
+
+    def validate_configuration()
+        if @retransmission_time < 0
+            raise ArgumentError, "Setting retransmission_time which sets the time spent for resending each failed messages must be positive integer. [retransmission_time=#{@retransmission_time}]." 
+        
+        elsif @max_items < @MIN_MESSAGE_AMOUNT
+            raise ArgumentError, "Setting max_items to value must be greater then #{@MIN_MESSAGE_AMOUNT}."
+
+        elsif @workspace_id.empty? or @workspace_key.empty? or @custom_log_table_name.empty? 
+            raise ArgumentError, "Malformed configuration , the following arguments can not be null or empty.[workspace_id=#{@workspace_id} , workspace_key=#{@workspace_key} , custom_log_table_name=#{@custom_log_table_name}]"
+
+        elsif not @custom_log_table_name.match(/^[[:alpha:]]+$/)
+            raise ArgumentError, 'custom_log_table_name must be only alpha characters.' 
+
+        elsif custom_log_table_name.empty?
+            raise ArgumentError, 'custom_log_table_name should not be empty.' 
+            
+        elsif @key_names.length > 500
+            raise ArgumentError, 'Azure Loganalytics limits the amount of columns to 500 in each table.' 
+        end
+
+        @logger.info("Azure Loganalytics configuration was found valid.")
+        
+        # If all validation pass then configuration is valid 
+        return  true
+    end # def validate_configuration
+
+    def azure_resource_id
+        @azure_resource_id
+    end
+
+    def RETRANSMISSION_DELAY
+        @RETRANSMISSION_DELAY
+    end
+
+    def MAX_SIZE_BYTES
+        @MAX_SIZE_BYTES
+    end
+
+    def amount_resizing
+        @amount_resizing
+    end
+
+    def retransmission_time
+        @retransmission_time
+    end
+
+    def proxy
+        @proxy
+    end
+
+    def logger
+        @logger
+    end
+
+    def decrease_factor
+        @decrease_factor
+    end
+
+    def workspace_id
+        @workspace_id
+    end
+
+    def workspace_key
+        @workspace_key
+    end
+
+    def custom_log_table_name
+        @custom_log_table_name
+    end
+
+    def endpoint
+        @endpoint
+    end
+
+    def time_generated_field
+        @time_generated_field
+    end
+
+    def key_names
+        @key_names
+    end
+
+    def max_items
+        @max_items
+    end
+
+    def plugin_flush_interval
+        @plugin_flush_interval
+    end
+
+    def MIN_MESSAGE_AMOUNT
+        @MIN_MESSAGE_AMOUNT
+    end
+    
+    def max_items=(new_max_items)
+        @max_items = new_max_items
+    end
+
+    def endpoint=(new_endpoint)
+        @endpoint = new_endpoint
+    end
+
+    def time_generated_field=(new_time_generated_field)
+        @time_generated_field = new_time_generated_field
+    end
+
+    def key_names=(new_key_names)
+        @key_names = new_key_names
+    end
+
+    def plugin_flush_interval=(new_plugin_flush_interval)
+        @plugin_flush_interval = new_plugin_flush_interval
+    end
+    
+    def decrease_factor=(new_decrease_factor)
+        @decrease_factor = new_decrease_factor
+    end
+
+    def amount_resizing=(new_amount_resizing)
+        @amount_resizing = new_amount_resizing
+    end
+
+    def max_items=(new_max_items)
+        @max_items = new_max_items
+    end
+    
+    def azure_resource_id=(new_azure_resource_id)
+        @azure_resource_id = new_azure_resource_id
+    end
+    
+    def proxy=(new_proxy)
+        @proxy = new_proxy
+    end
+
+    def retransmission_time=(new_retransmission_time)
+        @retransmission_time = new_retransmission_time
+    end
+end

data/lib/logstash/outputs/microsoft-logstash-output-azure-loganalytics.rb

@@ -0,0 +1,130 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/namespace"
+require "stud/buffer"
+require "logstash/logAnalyticsClient/logStashAutoResizeBuffer"
+require "logstash/logAnalyticsClient/logstashLoganalyticsConfiguration"
+
+class LogStash::Outputs::AzureLogAnalytics < LogStash::Outputs::Base
+
+  config_name "microsoft-logstash-output-azure-loganalytics"
+  
+  # Stating that the output plugin will run in concurrent mode
+  concurrency :shared
+
+  # Your Operations Management Suite workspace ID
+  config :workspace_id, :validate => :string, :required => true
+
+  # The primary or the secondary key used for authentication, required by Azure Loganalytics REST API
+  config :workspace_key, :validate => :string, :required => true
+
+  # The name of the event type that is being submitted to Log Analytics. 
+  # This must be only alpha characters.
+  # Table name under custom logs in which the data will be inserted
+  config :custom_log_table_name, :validate => :string, :required => true
+
+  # The service endpoint (Default: ods.opinsights.azure.com)
+  config :endpoint, :validate => :string, :default => 'ods.opinsights.azure.com'
+
+  # The name of the time generated field.
+  # Be careful that the value of field should strictly follow the ISO 8601 format (YYYY-MM-DDThh:mm:ssZ)
+  config :time_generated_field, :validate => :string, :default => ''
+
+  # Subset of keys to send to the Azure Loganalytics workspace
+  config :key_names, :validate => :array, :default => []
+
+  # # Max number of items to buffer before flushing. Default 50.
+  # config :flush_items, :validate => :number, :default => 50
+  
+  # Max number of seconds to wait between flushes. Default 5
+  config :plugin_flush_interval, :validate => :number, :default => 5
+
+  # Factor for adding to the amount of messages sent
+  config :decrease_factor, :validate => :number, :default => 100
+
+  # This will trigger message amount resizing in a REST request to LA
+  config :amount_resizing, :validate => :boolean, :default => true
+
+  # Setting the default amount of messages sent                                                                                                    
+  # it this is set with amount_resizing=false --> each message will have max_items
+  config :max_items, :validate => :number, :default => 2000
+
+  # Setting proxy to be used for the Azure Loganalytics REST client
+  config :proxy, :validate => :string, :default => ''
+
+  # This will set the amount of time given for retransmitting messages once sending is failed
+  config :retransmission_time, :validate => :number, :default => 10
+
+  # Optional to override the resource ID field on the workspace table.
+  # Resource ID provided must be a valid resource ID on azure 
+  config :azure_resource_id, :validate => :string, :default => ''
+
+  public
+  def register
+    @logstash_configuration= build_logstash_configuration()
+    # Validate configuration correctness 
+    @logstash_configuration.validate_configuration()
+    @logger.info("Logstash Azure Loganalytics output plugin configuration was found valid")
+
+    # Initialize the logstash resizable buffer
+    # This buffer will increase and decrease size according to the amount of messages inserted.
+    # If the buffer reached the max amount of messages the amount will be increased until the limit
+    @logstash_resizable_event_buffer=LogStashAutoResizeBuffer::new(@logstash_configuration)
+
+  end # def register
+
+  def multi_receive(events)
+    events.each do |event|
+      # creating document from event
+      document = create_event_document(event)
+      # Skip if document doesn't contain any items  
+      next if (document.keys).length < 1
+      
+      @logger.trace("Adding event document - " + event.to_s)
+      @logstash_resizable_event_buffer.add_event_document(document)
+
+    end
+  end # def multi_receive
+  
+  #private 
+  private
+
+  # In case that the user has defined key_names meaning that he would like to a subset of the data,
+  # we would like to insert only those keys.
+  # If no keys were defined we will send all the data 
+   
+  def create_event_document(event)
+    document = {}
+    event_hash = event.to_hash()
+    if @key_names.length > 0
+      # Get the intersection of key_names and keys of event_hash
+      keys_intersection = @key_names & event_hash.keys
+      keys_intersection.each do |key|
+        document[key] = event_hash[key]
+      end
+    else
+      document = event_hash
+    end
+
+    return document
+  end # def create_event_document
+
+  # Building the logstash object configuration from the output configuration provided by the user
+  # Return LogstashLoganalyticsOutputConfiguration populated with the configuration values
+  def build_logstash_configuration()
+    logstash_configuration= LogstashLoganalyticsOutputConfiguration::new(@workspace_id, @workspace_key, @custom_log_table_name, @logger)    
+    logstash_configuration.endpoint = @endpoint
+    logstash_configuration.time_generated_field = @time_generated_field
+    logstash_configuration.key_names = @key_names
+    logstash_configuration.plugin_flush_interval = @plugin_flush_interval
+    logstash_configuration.decrease_factor = @decrease_factor
+    logstash_configuration.amount_resizing = @amount_resizing
+    logstash_configuration.max_items = @max_items
+    logstash_configuration.azure_resource_id = @azure_resource_id
+    logstash_configuration.proxy = @proxy
+    logstash_configuration.retransmission_time = @retransmission_time
+    
+    return logstash_configuration
+  end # def build_logstash_configuration
+
+end # class LogStash::Outputs::AzureLogAnalytics

data/microsoft-logstash-output-azure-loganalytics.gemspec

@@ -0,0 +1,26 @@
+Gem::Specification.new do |s|
+  s.name = 'microsoft-logstash-output-azure-loganalytics'
+  s.version    =  File.read("VERSION").strip
+  s.authors = ["Ron Marsiano"]
+  s.email = "romarsia@outlook.com"
+  s.summary = %q{Azure Sentinel provides a new output plugin for Logstash. Using this output plugin, you will be able to send any log you want using Logstash to the Azure Sentinel/Log Analytics workspace}
+  s.description = s.summary
+  s.homepage = "https://github.com/Azure/Azure-Sentinel"
+  s.licenses = ["Apache License (2.0)"]
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT', 'VERSION']
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "rest-client", ">= 1.8.0"
+  s.add_runtime_dependency "azure-loganalytics-datacollector-api", ">= 0.1.5"
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency "logstash-codec-plain"
+  s.add_development_dependency "logstash-devutils"
+end

data/spec/outputs/azure_loganalytics_spec.rb

@@ -0,0 +1,78 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/microsoft-logstash-output-azure-loganalytics"
+require "logstash/codecs/plain"
+require "logstash/event"
+
+describe LogStash::Outputs::AzureLogAnalytics do
+
+  let(:workspace_id) { '<Workspace ID identifying your workspace>' }
+  let(:workspace_key) { '<Primary Key for the Azure log analytics workspace>' }
+  let(:custom_log_table_name) { 'ApacheAccessLog' }
+  let(:key_names) { ['logid','date','processing_time','remote','user','method','status','agent','eventtime'] }
+  let(:time_generated_field) { 'eventtime' }
+  let(:amount_resizing) {false}
+
+  # 1 second flush interval
+  let(:plugin_flush_interval) {1}
+
+  let(:azure_loganalytics_config) {
+    { 
+      "workspace_id" => workspace_id, 
+      "workspace_key" => workspace_key,
+      "custom_log_table_name" => custom_log_table_name,
+      "key_names" => key_names,
+      "time_generated_field" => time_generated_field,
+      "plugin_flush_interval" => plugin_flush_interval,
+      "amount_resizing" => amount_resizing
+    }
+  }
+
+  let(:azure_loganalytics) { LogStash::Outputs::AzureLogAnalytics.new(azure_loganalytics_config) }
+
+  before do
+    azure_loganalytics.register
+  end 
+
+  describe "#flush" do
+    it "Should successfully send the event to Azure Log Analytics" do
+      events = []
+      log1 = {
+        :logid => "5cdad72f-c848-4df0-8aaa-ffe033e75d57",
+        :date => "2017-04-22 09:44:32 JST",
+        :processing_time => "372",
+        :remote => "101.202.74.59",
+        :user => "-",
+        :method => "GET / HTTP/1.1",
+        :status => "304",
+        :size => "-",
+        :referer => "-",
+        :agent => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:27.0) Gecko/20100101 Firefox/27.0",
+        :eventtime => "2017-04-22T01:44:32Z"
+      }
+
+      log2 = {
+        :logid => "7260iswx-8034-4cc3-uirtx-f068dd4cd659",
+        :date => "2017-04-22 09:45:14 JST",
+        :processing_time => "105",
+        :remote => "201.78.74.59",
+        :user => "-",
+        :method => "GET /manager/html HTTP/1.1",
+        :status =>"200",
+        :size => "-",
+        :referer => "-",
+        :agent => "Mozilla/5.0 (Windows NT 5.1; rv:5.0) Gecko/20100101 Firefox/5.0",
+        :eventtime => "2017-04-22T01:45:14Z"
+      }
+
+      event1 =  LogStash::Event.new(log1) 
+      event2 =  LogStash::Event.new(log2) 
+      events.push(event1)
+      events.push(event2)
+      expect {azure_loganalytics.multi_receive(events)}.to_not raise_error
+      # Waiting for the data to be sent
+      sleep(plugin_flush_interval + 2)
+    end
+  end
+
+end

metadata

@@ -0,0 +1,136 @@
+--- !ruby/object:Gem::Specification
+name: microsoft-logstash-output-azure-loganalytics
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Ron Marsiano
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2020-05-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.0
+  name: rest-client
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.5
+  name: azure-loganalytics-datacollector-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.5
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+  name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-plain
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Azure Sentinel provides a new output plugin for Logstash. Using this
+  output plugin, you will be able to send any log you want using Logstash to the Azure
+  Sentinel/Log Analytics workspace
+email: romarsia@outlook.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- Gemfile
+- README.md
+- VERSION
+- lib/logstash/logAnalyticsClient/logAnalyticsClient.rb
+- lib/logstash/logAnalyticsClient/logStashAutoResizeBuffer.rb
+- lib/logstash/logAnalyticsClient/logstashLoganalyticsConfiguration.rb
+- lib/logstash/outputs/microsoft-logstash-output-azure-loganalytics.rb
+- microsoft-logstash-output-azure-loganalytics.gemspec
+- spec/outputs/azure_loganalytics_spec.rb
+homepage: https://github.com/Azure/Azure-Sentinel
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: output
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.7.10
+signing_key:
+specification_version: 4
+summary: Azure Sentinel provides a new output plugin for Logstash. Using this output
+  plugin, you will be able to send any log you want using Logstash to the Azure Sentinel/Log
+  Analytics workspace
+test_files:
+- spec/outputs/azure_loganalytics_spec.rb