logstash-input-azureblob 0.9.8 → 0.9.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e72dbef9d73b6b872b786b1cd40ffd3434835ac4
-  data.tar.gz: 9054b485b8073ca0dc9cce2e09c125630caec5fc
+  metadata.gz: 56832bbecde64cfccd20199348a414c0738ed9c1
+  data.tar.gz: fcf9cf2cae426c7f4b4f94dd7151ebac0b2b2964
 SHA512:
-  metadata.gz: 917b7f0ad4b0a552091a6055c9e5fd5eeef38d6ce0ce85166eb432d2a5737a8d49f75c28579ce0119ed1887c952a0c372dc620cdf41bcdfdd9d30e9160e33c0a
-  data.tar.gz: 3f513712c43563b6ea3407b23516e784b1d4d8fc706213247132b57827f7605ed55cbcacb9c357525fb84bc18920ab367a03e411b658d56e4da7506152560767
+  metadata.gz: cae7c2d9420417e758cea143bd9852e112fc98aed2c0e7040923e15e7379c1acb149f94f7eab73c73b50e67dde8869233d10341dfa6f2594078e4104d46c294e
+  data.tar.gz: 0962291e17c6f9e9c3eea68e898cb3ccdd0ba8ded581b7d6d087958155f71100ac4013ea2306264b5b9d36d88567f40804a51cc77b7902c9b4daf257a4064049

data/README.md

@@ -56,8 +56,37 @@ When set to `start_over`, it assumes none of the blob is consumed and it will re
 
 Offsets will be picked up from registry file whenever it exists.
 
+__*file_head_bytes*__
+
+Specifies the header of the file in bytes that does not repeat over records. Usually, these are json opening tags. The default value is `0`.
+
+__*file_tail_bytes*__
+  
+Specifies the tail of the file that does not repeat over records. Usually, these are json closing tags. The defaul tvalue is `0`.
+
+### Advanced tweaking parameters
+
+Keep these parameters default to use under normal situration. Tweak these parameters when dealing with large scale azure blobs and logs.
+
+__*blob_list_page_size*__
+
+Specifies the page-size for returned blob items. Too big number will hit heap overflow; Too small number will leads to too many requests. The default of `100` is good for heap size of 1G.
+
+__*break_json_down_policy*__
+
+Only works when the codec is set to `json`. Sets the policy to break the json object in the array into small events. Break json into small sections will not be as efficient as keep it as a whole, but will reduce the usage of the memory. Possible options: `do_not_break`, `with_head_tail`, `without_head_tail`. 
+
+The default value is: `do_not_break`.
+
+__*break_json_batch_count*__
+
+Only works when the codec is set to `json`. Sets when break json happens, how many json object will be put in 1 batch. The bigger this is set, more memory is taken and the bigger the json will be handing to the codec. This is useful when we need to break the big json array into small pieces. Set to `1` when expect to send json 1 by 1 in the array.
+
 ### Examples
-```
+
+* Bare-bone settings:
+
+```yaml
 input
 {
     azureblob
@@ -69,5 +98,160 @@ input
 }
 ```
 
+* Example for Wad-IIS
+
+```yaml
+input {
+    azureblob
+    {
+        storage_account_name => 'mystorageaccount'
+        storage_access_key => 'VGhpcyBpcyBhIGZha2Uga2V5Lg=='
+        container => 'wad-iis-logfiles'
+        codec => line
+    }
+}    
+filter {
+  ## Ignore the comments that IIS will add to the start of the W3C logs
+  #
+  if [message] =~ "^#" {
+    drop {}
+  }
+
+  grok {
+      # https://grokdebug.herokuapp.com/
+      match => ["message", "%{TIMESTAMP_ISO8601:log_timestamp} %{WORD:sitename} %{WORD:computername} %{IP:server_ip} %{WORD:method} %{URIPATH:uriStem} %{NOTSPACE:uriQuery} %{NUMBER:port} %{NOTSPACE:username} %{IPORHOST:clientIP} %{NOTSPACE:protocolVersion} %{NOTSPACE:userAgent} %{NOTSPACE:cookie} %{NOTSPACE:referer} %{NOTSPACE:requestHost} %{NUMBER:response} %{NUMBER:subresponse} %{NUMBER:win32response} %{NUMBER:bytesSent} %{NUMBER:bytesReceived} %{NUMBER:timetaken}"]
+  }
+
+  ## Set the Event Timesteamp from the log
+  #
+  date {
+    match => [ "log_timestamp", "YYYY-MM-dd HH:mm:ss" ]
+      timezone => "Etc/UTC"
+  }
+
+  ## If the log record has a value for 'bytesSent', then add a new field
+  #   to the event that converts it to kilobytes
+  #
+  if [bytesSent] {
+    ruby {
+      code => "event['kilobytesSent'] = event['bytesSent'].to_i / 1024.0"
+    }
+  }
+
+  ## Do the same conversion for the bytes received value
+  #
+  if [bytesReceived] {
+    ruby {
+      code => "event['kilobytesReceived'] = event['bytesReceived'].to_i / 1024.0"
+    }
+  }
+
+  ## Perform some mutations on the records to prep them for Elastic
+  #
+  mutate {
+    ## Convert some fields from strings to integers
+    #
+    convert => ["bytesSent", "integer"]
+    convert => ["bytesReceived", "integer"]
+    convert => ["timetaken", "integer"]
+
+    ## Create a new field for the reverse DNS lookup below
+    #
+    add_field => { "clientHostname" => "%{clientIP}" }
+
+    ## Finally remove the original log_timestamp field since the event will
+    #   have the proper date on it
+    #
+    remove_field => [ "log_timestamp"]
+  }
+
+  ## Do a reverse lookup on the client IP to get their hostname.
+  #
+  dns {
+    ## Now that we've copied the clientIP into a new field we can
+    #   simply replace it here using a reverse lookup
+    #
+    action => "replace"
+    reverse => ["clientHostname"]
+  }
+
+  ## Parse out the user agent
+  #
+  useragent {
+    source=> "useragent"
+    prefix=> "browser"
+  }
+}
+output {
+    file {
+        path => '/var/tmp/logstash-file-output'
+        codec => rubydebug
+    }
+    stdout { 
+        codec => rubydebug
+    }
+}
+```
+
+* NSG Logs
+
+```yaml
+input {
+   azureblob
+     {
+         storage_account_name => "mystorageaccount"
+         storage_access_key => "VGhpcyBpcyBhIGZha2Uga2V5Lg=="
+         container => "insights-logs-networksecuritygroupflowevent"
+         codec => "json"
+         # Refer https://docs.microsoft.com/en-us/azure/network-watcher/network-watcher-read-nsg-flow-logs
+         # Typical numbers could be 21/9 or 12/2 depends on the nsg log file types
+         file_head_bytes => 21
+         file_tail_bytes => 9
+         # Enable / tweak these settings when event is too big for codec to handle.
+         # break_json_down_policy => "with_head_tail"
+         # break_json_batch_count => 2
+     }
+   }
+
+   filter {
+     split { field => "[records]" }
+     split { field => "[records][properties][flows]"}
+     split { field => "[records][properties][flows][flows]"}
+     split { field => "[records][properties][flows][flows][flowTuples]"}
+
+  mutate{
+   split => { "[records][resourceId]" => "/"}
+   add_field => {"Subscription" => "%{[records][resourceId][2]}"
+                 "ResourceGroup" => "%{[records][resourceId][4]}"
+                 "NetworkSecurityGroup" => "%{[records][resourceId][8]}"}
+   convert => {"Subscription" => "string"}
+   convert => {"ResourceGroup" => "string"}
+   convert => {"NetworkSecurityGroup" => "string"}
+   split => { "[records][properties][flows][flows][flowTuples]" => ","}
+   add_field => {
+               "unixtimestamp" => "%{[records][properties][flows][flows][flowTuples][0]}"
+               "srcIp" => "%{[records][properties][flows][flows][flowTuples][1]}"
+               "destIp" => "%{[records][properties][flows][flows][flowTuples][2]}"
+               "srcPort" => "%{[records][properties][flows][flows][flowTuples][3]}"
+               "destPort" => "%{[records][properties][flows][flows][flowTuples][4]}"
+               "protocol" => "%{[records][properties][flows][flows][flowTuples][5]}"
+               "trafficflow" => "%{[records][properties][flows][flows][flowTuples][6]}"
+               "traffic" => "%{[records][properties][flows][flows][flowTuples][7]}"
+                }
+   convert => {"unixtimestamp" => "integer"}
+   convert => {"srcPort" => "integer"}
+   convert => {"destPort" => "integer"}        
+  }
+
+  date{
+    match => ["unixtimestamp" , "UNIX"]
+  }
+ }
+
+ output {
+   stdout { codec => rubydebug }
+ } 
+```
+
 ## More information
 The source code of this plugin is hosted in GitHub repo [Microsoft Azure Diagnostics with ELK](https://github.com/Azure/azure-diagnostics-tools). We welcome you to provide feedback and/or contribute to the project.

data/lib/logstash/inputs/azureblob.rb

@@ -77,6 +77,28 @@ class LogStash::Inputs::LogstashInputAzureblob < LogStash::Inputs::Base
   # When set to `start_over`, it will read all log files from begining.
   config :registry_create_policy, :validate => :string, :default => 'resume'
 
+  # Sets the header of the file that does not repeat over records. Usually, these are json opening tags.
+  config :file_head_bytes, :validate => :number, :default => 0
+
+  # Sets the tail of the file that does not repeat over records. Usually, these are json closing tags.
+  config :file_tail_bytes, :validate => :number, :default => 0
+
+  # Sets how to break json
+  #
+  # Only works when the codec is set to `json`. Sets the policy to break the json object in the array into small events.
+  # Break json into small sections will not be as efficient as keep it as a whole, but will reduce the usage of 
+  # the memory. 
+  # Possible options: `do_not_break`, `with_head_tail`, `without_head_tail`
+  config :break_json_down_policy, :validate => :string, :default => 'do_not_break'
+
+  # Sets when break json happens, how many json object will be put in 1 batch
+  config :break_json_batch_count, :validate => :number, :default => 10
+  
+  # Sets the page-size for returned blob items. Too big number will hit heap overflow; Too small number will leads to too many requests.
+  #
+  # The default, `100` is good for default heap size of 1G.
+  config :blob_list_page_size, :validate => :number, :default => 100
+
   # Constant of max integer
   MAX = 2 ** ([42].pack('i').size * 16 -2 ) -1
 
@@ -98,6 +120,7 @@ class LogStash::Inputs::LogstashInputAzureblob < LogStash::Inputs::Base
     # we can abort the loop if stop? becomes true
     while !stop?
       process(queue)
+      @logger.debug("Hitting interval of #{@interval}ms . . .")
       Stud.stoppable_sleep(@interval) { stop? }
     end # loop
   end # def run
@@ -117,15 +140,63 @@ class LogStash::Inputs::LogstashInputAzureblob < LogStash::Inputs::Base
           # Work-around: After returned by get_blob, the etag will contains quotes.
           new_etag = blob.properties[:etag]
           # ~ Work-around
-          blob, content = @azure_blob.get_blob(@container, blob_name, {:start_range=>start_index} )
-          
-          @codec.decode(content) do |event|
-            decorate(event)
-            queue << event
-          end # decode
+
+          blob, header = @azure_blob.get_blob(@container, blob_name, {:end_range => (@file_head_bytes-1) }) if header.nil? unless @file_head_bytes.nil? or @file_head_bytes <= 0
+
+          if start_index == 0
+            # Skip the header since it is already read.
+            start_index = @file_head_bytes
+          else
+            # Adjust the offset when it is other than first time, then read till the end of the file, including the tail.
+            start_index = start_index - @file_tail_bytes
+            start_index = 0 if start_index < 0
+          end
+
+          blob, content = @azure_blob.get_blob(@container, blob_name, {:start_range => start_index} )
+
+          # content will be used to calculate the new offset. Create a new variable for processed content.
+          processed_content = content
+
+          is_json_codec = (defined?(LogStash::Codecs::JSON) == 'constant') && (@codec.is_a? LogStash::Codecs::JSON)
+          if (is_json_codec)
+            skip = processed_content.index '{'
+            processed_content = processed_content[skip..-1] unless skip.nil?
+          end #if
+
+          if is_json_codec && (@break_json_down_policy != 'do_not_break')
+            @logger.debug("codec is json and policy is not do_not_break")
+            
+            @break_json_batch_count = 1 if break_json_batch_count <= 0
+            tail = processed_content[-@file_tail_bytes..-1]
+            while (!processed_content.nil? && processed_content.length > @file_tail_bytes)
+              json_event, processed_content = get_jsons(processed_content, @break_json_batch_count)
+              @logger.debug("Got json: ========================")
+              @logger.debug("#{json_event[0..50]}...#{json_event[-50..-1]}")
+              @logger.debug("End got json: ========================")
+              @logger.debug("Processed content: #{processed_content[0..50]}...")
+              break if json_event.nil?
+              if @break_json_down_policy == 'with_head_tail'
+                @logger.debug("Adding json head/tails.")
+                json_event = "#{header}#{json_event}#{tail}"
+              end #if
+              @codec.decode(json_event) do |event|
+                decorate(event)
+                queue << event
+              end # decode
+            end
+          else
+            @logger.debug("Non-json codec or the policy is do not break")
+            # Putting header and content and tail together before pushing into event queue
+            processed_content = "#{header}#{processed_content}" unless header.nil? || header.length == 0
+            @codec.decode(processed_content) do |event|
+              decorate(event)
+              queue << event
+            end # decode
+          end #if
         ensure
           # Making sure the reader is removed from the registry even when there's exception.
           new_offset = start_index
+          new_offset = 0 if start_index == @file_head_bytes && content.nil? # Reset the offset when nothing has been read.
           new_offset = new_offset + content.length unless content.nil?
           new_registry_item = LogStash::Inputs::RegistryItem.new(blob_name, new_etag, nil, new_offset, gen)
           update_registry(new_registry_item)
@@ -135,6 +206,47 @@ class LogStash::Inputs::LogstashInputAzureblob < LogStash::Inputs::Base
       @logger.error("Oh My, An error occurred. \nError:#{e}:\nTrace:\n#{e.backtrace}", :exception => e)
     end # begin
   end # process
+  
+  # Get first json object out of a string, return the rest of the string
+  def get_jsons(content, batch_size)
+    return nil, content, 0 if content.nil? || content.length == 0
+    return nil, content, 0 if (content.index '{').nil?
+
+    hit = 0
+    count = 0
+    index = 0
+    first = content.index('{')
+    move_opening = true
+    move_closing = true
+    while(hit < batch_size)
+        inIndex = content.index('{', index) if move_opening
+        outIndex = content.index('}', index) if move_closing
+
+        # TODO: Fix the ending condition
+        break if count == 0 && (inIndex.nil? || outIndex.nil?)
+        
+        if(inIndex.nil?)
+            index = outIndex
+        elsif(outIndex.nil?)
+            index = inIndex
+        else
+            index = [inIndex, outIndex].min
+        end #if
+        if content[index] == '{'
+            count += 1
+            move_opening = true
+            move_closing = false
+        elsif content[index] == '}'
+            count -= 1
+            move_closing = true
+            move_opening = false
+        end #if
+        index += 1
+        hit += 1 if count == 0
+    end
+    
+    return content[first..index-1], content[index..-1], hit
+  end #def get_first_json
 
   # Deserialize registry hash from json string.
   def deserialize_registry_hash (json_string)
@@ -150,8 +262,10 @@ class LogStash::Inputs::LogstashInputAzureblob < LogStash::Inputs::Base
   def list_all_blobs
     blobs = Set.new []
     continuation_token = NIL
+    @blob_list_page_size = 100 if @blob_list_page_size <= 0
     loop do
-      entries = @azure_blob.list_blobs(@container, { :timeout => 10, :marker => continuation_token})
+      # Need to limit the returned number of the returned entries to avoid out of memory exception.
+      entries = @azure_blob.list_blobs(@container, { :timeout => 10, :marker => continuation_token, :max_results => @blob_list_page_size })
       entries.each do |entry|
         blobs << entry
       end # each

data/logstash-input-azureblob.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-input-azureblob'
-  s.version       = '0.9.8'
+  s.version       = '0.9.9'
   s.licenses      = ['Apache License (2.0)']
   s.summary       = 'This plugin collects Microsoft Azure Diagnostics data from Azure Storage Blobs.'
   s.description   = 'This gem is a Logstash plugin. It reads and parses data from Azure Storage Blobs.'
@@ -21,6 +21,6 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "logstash-core-plugin-api", '>= 1.60', '<= 2.99'
   s.add_runtime_dependency 'logstash-codec-json_lines'
   s.add_runtime_dependency 'stud', '>= 0.0.22'
-  s.add_runtime_dependency 'azure-storage', '~> 0.11.4.preview'
+  s.add_runtime_dependency 'azure-storage', '~> 0.12.3.preview'
   s.add_development_dependency 'logstash-devutils'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-azureblob
 version: !ruby/object:Gem::Version
-  version: 0.9.8
+  version: 0.9.9
 platform: ruby
 authors:
 - Microsoft Corporation
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-07-30 00:00:00.000000000 Z
+date: 2017-08-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -63,7 +63,7 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.11.4.preview
+        version: 0.12.3.preview
   name: azure-storage
   prerelease: false
   type: :runtime
@@ -71,7 +71,7 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.11.4.preview
+        version: 0.12.3.preview
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements: