logstash-input-azureblob-json-head-tail 0.9.9

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 36d2a3fcea0fdf5f3af0b3175acab68a94901b7b
+  data.tar.gz: 85b71255879d3b0b15c6277b4aa9530406982255
+SHA512:
+  metadata.gz: c7e8986e035347e2a482877ae18683fbd1001cc7c0b7bdbf67077090a718d81be5da4a05fa1ba6f388874bc28bb9183927b22faaab8840db6bc69e29c282c607
+  data.tar.gz: 6b99f505724ed327e11eb38c5b971d970691ff9903668482936ee01ab3a89920174218e2b0432e09d56fb9dac0e4f2e79ce466488252ee47fab1586d437bafe7

data/CHANGELOG.md

@@ -0,0 +1,7 @@
+## 2016.08.17
+* Added a new configuration parameter for custom endpoint.
+
+## 2016.05.05
+* Made the plugin to respect Logstash shutdown signal.
+* Updated the *logstash-core* runtime dependency requirement to '~> 2.0'.
+* Updated the *logstash-devutils* development dependency requirement to '>= 0.0.16'

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,17 @@
+
+Copyright (c) Microsoft.  All rights reserved.
+Microsoft would like to thank its contributors, a list
+of whom are at http://aka.ms/entlib-contributors
+
+Licensed under the Apache License, Version 2.0 (the "License"); you
+may not use this file except in compliance with the License. You may
+obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied. See the License for the specific language governing permissions
+and limitations under the License.
+

data/README.md

@@ -0,0 +1,243 @@
+# Logstash input plugin for Azure Storage Blobs
+
+## Summary
+This plugin reads and parses data from Azure Storage Blobs.
+
+## Installation
+You can install this plugin using the Logstash "plugin" or "logstash-plugin" (for newer versions of Logstash) command:
+```sh
+logstash-plugin install logstash-input-azureblob
+```
+For more information, see Logstash reference [Working with plugins](https://www.elastic.co/guide/en/logstash/current/working-with-plugins.html).
+
+## Configuration
+### Required Parameters
+__*storage_account_name*__
+
+The storage account name.
+
+__*storage_access_key*__
+
+The access key to the storage account.
+
+__*container*__
+
+The blob container name.
+
+### Optional Parameters
+__*endpoint*__
+
+Specifies the endpoint of Azure Service Management. The default value is `core.windows.net`. 
+
+__*registry_path*__
+
+Specifies the file path for the registry file to record offsets and coordinate between multiple clients. The default value is `data/registry`.
+
+Overwrite this value when there happen to be a file at the path of `data/registry` in the azure blob container.
+
+__*interval*__
+
+Set how many seconds to idle before checking for new logs. The default, `30`, means idle for `30` seconds.
+
+__*registry_create_policy*__
+
+Specifies the way to initially set offset for existing blob files.
+
+This option only applies for registry creation. 
+
+Valid values include:
+
+  - resume
+  - start_over
+
+The default, `resume`, means when the registry is initially created, it assumes all blob has been consumed and it will start to pick up any new content in the blobs.
+
+When set to `start_over`, it assumes none of the blob is consumed and it will read all blob files from begining.
+
+Offsets will be picked up from registry file whenever it exists.
+
+__*file_head_bytes*__
+
+Specifies the header of the file in bytes that does not repeat over records. Usually, these are json opening tags. The default value is `0`.
+
+__*file_tail_bytes*__
+  
+Specifies the tail of the file that does not repeat over records. Usually, these are json closing tags. The defaul tvalue is `0`.
+
+__*record_preprocess_reg_exp*__
+
+Specifies the regular expression to process content before pushing the event. The matched will be removed. For example, `^\s*,` will removing the leading `,` from the content. The regular expression uses multiline mode.
+
+__*blob_list_page_size*__
+
+Specifies the page-size for returned blob items. Too big number will hit heap overflow; Too small number will leads to too many requests. The default of `100` is good for heap size of 1G.
+
+### Examples
+
+* Bare-bone settings:
+
+```yaml
+input
+{
+    azureblob
+    {
+        storage_account_name => "mystorageaccount"
+        storage_access_key => "VGhpcyBpcyBhIGZha2Uga2V5Lg=="
+        container => "mycontainer"
+    }
+}
+```
+
+* Example for Wad-IIS
+
+```yaml
+input {
+    azureblob
+    {
+        storage_account_name => 'mystorageaccount'
+        storage_access_key => 'VGhpcyBpcyBhIGZha2Uga2V5Lg=='
+        container => 'wad-iis-logfiles'
+        codec => line
+    }
+}    
+filter {
+  ## Ignore the comments that IIS will add to the start of the W3C logs
+  #
+  if [message] =~ "^#" {
+    drop {}
+  }
+
+  grok {
+      # https://grokdebug.herokuapp.com/
+      match => ["message", "%{TIMESTAMP_ISO8601:log_timestamp} %{WORD:sitename} %{WORD:computername} %{IP:server_ip} %{WORD:method} %{URIPATH:uriStem} %{NOTSPACE:uriQuery} %{NUMBER:port} %{NOTSPACE:username} %{IPORHOST:clientIP} %{NOTSPACE:protocolVersion} %{NOTSPACE:userAgent} %{NOTSPACE:cookie} %{NOTSPACE:referer} %{NOTSPACE:requestHost} %{NUMBER:response} %{NUMBER:subresponse} %{NUMBER:win32response} %{NUMBER:bytesSent} %{NUMBER:bytesReceived} %{NUMBER:timetaken}"]
+  }
+
+  ## Set the Event Timesteamp from the log
+  #
+  date {
+    match => [ "log_timestamp", "YYYY-MM-dd HH:mm:ss" ]
+      timezone => "Etc/UTC"
+  }
+
+  ## If the log record has a value for 'bytesSent', then add a new field
+  #   to the event that converts it to kilobytes
+  #
+  if [bytesSent] {
+    ruby {
+      code => "event['kilobytesSent'] = event['bytesSent'].to_i / 1024.0"
+    }
+  }
+
+  ## Do the same conversion for the bytes received value
+  #
+  if [bytesReceived] {
+    ruby {
+      code => "event['kilobytesReceived'] = event['bytesReceived'].to_i / 1024.0"
+    }
+  }
+
+  ## Perform some mutations on the records to prep them for Elastic
+  #
+  mutate {
+    ## Convert some fields from strings to integers
+    #
+    convert => ["bytesSent", "integer"]
+    convert => ["bytesReceived", "integer"]
+    convert => ["timetaken", "integer"]
+
+    ## Create a new field for the reverse DNS lookup below
+    #
+    add_field => { "clientHostname" => "%{clientIP}" }
+
+    ## Finally remove the original log_timestamp field since the event will
+    #   have the proper date on it
+    #
+    remove_field => [ "log_timestamp"]
+  }
+
+  ## Do a reverse lookup on the client IP to get their hostname.
+  #
+  dns {
+    ## Now that we've copied the clientIP into a new field we can
+    #   simply replace it here using a reverse lookup
+    #
+    action => "replace"
+    reverse => ["clientHostname"]
+  }
+
+  ## Parse out the user agent
+  #
+  useragent {
+    source=> "useragent"
+    prefix=> "browser"
+  }
+}
+output {
+    file {
+        path => '/var/tmp/logstash-file-output'
+        codec => rubydebug
+    }
+    stdout { 
+        codec => rubydebug
+    }
+}
+```
+
+* NSG Logs
+
+```yaml
+input {
+   azureblob
+     {
+         storage_account_name => "mystorageaccount"
+         storage_access_key => "VGhpcyBpcyBhIGZha2Uga2V5Lg=="
+         container => "insights-logs-networksecuritygroupflowevent"
+         codec => "json"
+         file_head_bytes => 21
+         file_tail_bytes => 9
+         record_preprocess_reg_exp => "^\s*,"
+     }
+   }
+
+   filter {
+     split { field => "[records]" }
+     split { field => "[records][properties][flows]"}
+     split { field => "[records][properties][flows][flows]"}
+     split { field => "[records][properties][flows][flows][flowTuples]"}
+
+  mutate{
+   split => { "[records][resourceId]" => "/"}
+   add_field => {"Subscription" => "%{[records][resourceId][2]}"
+                 "ResourceGroup" => "%{[records][resourceId][4]}"
+                 "NetworkSecurityGroup" => "%{[records][resourceId][8]}"}
+   convert => {"Subscription" => "string"}
+   convert => {"ResourceGroup" => "string"}
+   convert => {"NetworkSecurityGroup" => "string"}
+   split => { "[records][properties][flows][flows][flowTuples]" => ","}
+   add_field => {
+               "unixtimestamp" => "%{[records][properties][flows][flows][flowTuples][0]}"
+               "srcIp" => "%{[records][properties][flows][flows][flowTuples][1]}"
+               "destIp" => "%{[records][properties][flows][flows][flowTuples][2]}"
+               "srcPort" => "%{[records][properties][flows][flows][flowTuples][3]}"
+               "destPort" => "%{[records][properties][flows][flows][flowTuples][4]}"
+               "protocol" => "%{[records][properties][flows][flows][flowTuples][5]}"
+               "trafficflow" => "%{[records][properties][flows][flows][flowTuples][6]}"
+               "traffic" => "%{[records][properties][flows][flows][flowTuples][7]}"
+                }
+   convert => {"unixtimestamp" => "integer"}
+   convert => {"srcPort" => "integer"}
+   convert => {"destPort" => "integer"}        
+  }
+
+  date{
+    match => ["unixtimestamp" , "UNIX"]
+  }
+ }
+
+ output {
+   stdout { codec => rubydebug }
+ } 
+```
+
+## More information
+The source code of this plugin is hosted in GitHub repo [Microsoft Azure Diagnostics with ELK](https://github.com/Azure/azure-diagnostics-tools). We welcome you to provide feedback and/or contribute to the project.

data/lib/logstash/inputs/azureblob.rb

@@ -0,0 +1,383 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+
+# Azure Storage SDK for Ruby
+require "azure/storage"
+require 'json' # for registry content
+require "securerandom" # for generating uuid.
+
+# Registry item to coordinate between mulitple clients
+class LogStash::Inputs::RegistryItem
+  attr_accessor :file_path, :etag, :offset, :reader, :gen
+  # Allow json serialization.
+  def as_json(options={})
+    {
+      file_path: @file_path,
+      etag: @etag,
+      reader: @reader,
+      offset: @offset,
+      gen: @gen
+    }
+  end # as_json
+
+  def to_json(*options)
+    as_json(*options).to_json(*options)
+  end # to_json
+
+  def initialize(file_path, etag, reader, offset = 0, gen = 0)
+    @file_path = file_path
+    @etag = etag
+    @reader = reader
+    @offset = offset
+    @gen = gen
+  end # initialize
+end # class RegistryItem
+
+
+# Logstash input plugin for Azure Blobs
+#
+# This logstash plugin gathers data from Microsoft Azure Blobs
+class LogStash::Inputs::LogstashInputAzureblob < LogStash::Inputs::Base
+  config_name "azureblob"
+
+  # If undefined, Logstash will complain, even if codec is unused.
+  default :codec, "json_lines"
+
+  # Set the account name for the azure storage account.
+  config :storage_account_name, :validate => :string
+  
+  # Set the key to access the storage account.
+  config :storage_access_key, :validate => :string
+  
+  # Set the container of the blobs.
+  config :container, :validate => :string
+  
+  # Set the endpoint for the blobs.
+  #
+  # The default, `core.windows.net` targets the public azure.
+  config :endpoint, :validate => :string, :default => 'core.windows.net'
+
+  # Set the value of using backup mode.
+  config :backupmode, :validate => :boolean, :default => false, :deprecated => true, :obsolete => 'This option is obsoleted and the settings will be ignored.'
+  
+  # Set the value for the registry file.
+  #
+  # The default, `data/registry`, is used to coordinate readings for various instances of the clients.
+  config :registry_path, :validate => :string, :default => 'data/registry'
+  
+  # Set how many seconds to keep idle before checking for new logs.
+  #
+  # The default, `30`, means trigger a reading for the log every 30 seconds after entering idle.
+  config :interval, :validate => :number, :default => 30
+
+  # Set the registry create mode
+  #
+  # The default, `resume`, means when the registry is initially created, it assumes all logs has been handled.
+  # When set to `start_over`, it will read all log files from begining.
+  config :registry_create_policy, :validate => :string, :default => 'resume'
+
+  # Sets the header of the file that does not repeat over records. Usually, these are json opening tags.
+  config :file_head_bytes, :validate => :number, :default => 0
+
+  # Sets the tail of the file that does not repeat over records. Usually, these are json closing tags.
+  config :file_tail_bytes, :validate => :number, :default => 0
+
+  # Sets the regular expression to process content before pushing the event.
+  config :record_preprocess_reg_exp, :validate => :string
+
+  # Sets the page-size for returned blob items. Too big number will hit heap overflow; Too small number will leads to too many requests.
+  #
+  # The default, `100` is good for default heap size of 1G.
+  config :blob_list_page_size, :validate => :number, :default => 100
+
+  # Constant of max integer
+  MAX = 2 ** ([42].pack('i').size * 16 -2 ) -1
+
+  public
+  def register
+    # this is the reader # for this specific instance.
+    @reader = SecureRandom.uuid
+    @registry_locker = "#{@registry_path}.lock"
+   
+    # Setup a specific instance of an Azure::Storage::Client
+    client = Azure::Storage::Client.create(:storage_account_name => @storage_account_name, :storage_access_key => @storage_access_key, :storage_blob_host => "https://#{@storage_account_name}.blob.#{@endpoint}")
+    # Get an azure storage blob service object from a specific instance of an Azure::Storage::Client
+    @azure_blob = client.blob_client
+    # Add retry filter to the service object
+    @azure_blob.with_filter(Azure::Storage::Core::Filter::ExponentialRetryPolicyFilter.new)
+  end # def register
+
+  def run(queue)
+    # we can abort the loop if stop? becomes true
+    while !stop?
+      process(queue)
+      Stud.stoppable_sleep(@interval) { stop? }
+    end # loop
+  end # def run
+
+  def stop
+    cleanup_registry
+  end # def stop
+  
+  # Start processing the next item.
+  def process(queue)
+    begin
+      blob, start_index, gen = register_for_read
+
+      if(!blob.nil?)
+        begin
+          blob_name = blob.name
+          # Work-around: After returned by get_blob, the etag will contains quotes.
+          new_etag = blob.properties[:etag]
+          # ~ Work-around
+
+          blob, header = @azure_blob.get_blob(@container, blob_name, {:end_range => @file_head_bytes}) if header.nil? unless @file_head_bytes.nil? or @file_head_bytes <= 0
+
+          if start_index == 0
+            # Skip the header since it is already read.
+            start_index = start_index + @file_head_bytes
+          else
+            # Adjust the offset when it is other than first time, then read till the end of the file, including the tail.
+            start_index = start_index - @file_tail_bytes
+            start_index = 0 if start_index < 0
+          end
+
+          blob, content = @azure_blob.get_blob(@container, blob_name, {:start_range => start_index} )
+          
+          # content will be used to calculate the new offset. Create a new variable for processed content.
+          processed_content = content
+          if(!@record_preprocess_reg_exp.nil?)
+            reg_exp = Regexp.new(@record_preprocess_reg_exp, Regexp::MULTILINE)
+            processed_content = content.sub(reg_exp, '')
+          end
+
+          # Putting header and content and tail together before pushing into event queue
+          processed_content = "#{header}#{processed_content}" unless header.nil? || header.length == 0
+                              
+          @codec.decode(processed_content) do |event|
+            decorate(event)
+            queue << event
+          end # decode
+        ensure
+          # Making sure the reader is removed from the registry even when there's exception.
+          new_offset = start_index
+          new_offset = new_offset + content.length unless content.nil?
+          new_registry_item = LogStash::Inputs::RegistryItem.new(blob_name, new_etag, nil, new_offset, gen)
+          update_registry(new_registry_item)
+        end # begin
+      end # if
+    rescue StandardError => e
+      @logger.error("Oh My, An error occurred. \nError:#{e}:\nTrace:\n#{e.backtrace}", :exception => e)
+    end # begin
+  end # process
+
+  # Deserialize registry hash from json string.
+  def deserialize_registry_hash (json_string)
+    result = Hash.new
+    temp_hash = JSON.parse(json_string)
+    temp_hash.values.each { |kvp|
+      result[kvp['file_path']] = LogStash::Inputs::RegistryItem.new(kvp['file_path'], kvp['etag'], kvp['reader'], kvp['offset'], kvp['gen'])
+    }
+    return result
+  end #deserialize_registry_hash
+
+  # List all the blobs in the given container.
+  def list_all_blobs
+    blobs = Set.new []
+    continuation_token = NIL
+    @blob_list_page_size = 100 if @blob_list_page_size <= 0
+    loop do
+      # Need to limit the returned number of the returned entries to avoid out of memory exception.
+      entries = @azure_blob.list_blobs(@container, { :timeout => 10, :marker => continuation_token, :max_results => @blob_list_page_size })
+      entries.each do |entry|
+        blobs << entry
+      end # each
+      continuation_token = entries.continuation_token
+      break if continuation_token.empty?
+    end # loop
+    return blobs
+  end # def list_blobs
+
+  # Raise generation for blob in registry
+  def raise_gen(registry_hash, file_path)
+    begin
+      target_item = registry_hash[file_path]
+      begin
+        target_item.gen += 1
+        # Protect gen from overflow.
+        target_item.gen = target_item.gen / 2 if target_item.gen == MAX
+      rescue StandardError => e
+        @logger.error("Fail to get the next generation for target item #{target_item}.", :exception => e)
+        target_item.gen = 0
+      end
+
+      min_gen_item = registry_hash.values.min_by { |x| x.gen }
+      while min_gen_item.gen > 0
+        registry_hash.values.each { |value| 
+          value.gen -= 1
+        }
+        min_gen_item = registry_hash.values.min_by { |x| x.gen }
+      end
+    end
+  end # raise_gen
+
+  # Acquire a lease on a blob item with retries.
+  #
+  # By default, it will retry 30 times with 1 second interval.
+  def acquire_lease(blob_name, retry_times = 30, interval_sec = 1)
+    lease = nil;
+    retried = 0;
+    while lease.nil? do
+      begin
+        lease = @azure_blob.acquire_blob_lease(@container, blob_name, {:timeout => 10})
+      rescue StandardError => e
+        if(e.type == 'LeaseAlreadyPresent')
+            if (retried > retry_times)
+                raise
+            end
+            retried += 1
+            sleep interval_sec
+        end
+      end
+    end #while
+    return lease
+  end # acquire_lease
+
+  # Return the next blob for reading as well as the start index.
+  def register_for_read
+    begin
+      all_blobs = list_all_blobs
+      registry = all_blobs.find { |item| item.name.downcase == @registry_path  }
+      registry_locker = all_blobs.find { |item| item.name.downcase == @registry_locker }
+
+      candidate_blobs = all_blobs.select { |item| (item.name.downcase != @registry_path) && ( item.name.downcase != @registry_locker ) }
+      
+      start_index = 0
+      gen = 0
+      lease = nil
+
+      # Put lease on locker file than the registy file to allow update of the registry as a workaround for Azure Storage Ruby SDK issue # 16.
+      # Workaround: https://github.com/Azure/azure-storage-ruby/issues/16
+      registry_locker = @azure_blob.create_block_blob(@container, @registry_locker, @reader) if registry_locker.nil?
+      lease = acquire_lease(@registry_locker)
+      # ~ Workaround
+
+      if(registry.nil?)
+        registry_hash = create_registry(candidate_blobs)
+      else
+        registry_hash = load_registry
+      end #if
+        
+      picked_blobs = Set.new []
+      # Pick up the next candidate
+      picked_blob = nil
+      candidate_blobs.each { |candidate_blob|
+        registry_item = registry_hash[candidate_blob.name]
+
+        # Appending items that doesn't exist in the hash table
+        if registry_item.nil?
+          registry_item = LogStash::Inputs::RegistryItem.new(candidate_blob.name, candidate_blob.properties[:etag], nil, 0, 0)
+          registry_hash[candidate_blob.name] = registry_item
+        end # if
+        
+        if ((registry_item.offset < candidate_blob.properties[:content_length]) && (registry_item.reader.nil? || registry_item.reader == @reader))
+          picked_blobs << candidate_blob
+        end
+      }
+
+      picked_blob = picked_blobs.min_by { |b| registry_hash[b.name].gen }
+      if !picked_blob.nil?
+        registry_item = registry_hash[picked_blob.name]
+        registry_item.reader = @reader
+        registry_hash[picked_blob.name] = registry_item
+        start_index = registry_item.offset
+        raise_gen(registry_hash, picked_blob.name)
+        gen = registry_item.gen
+      end #if
+
+      # Save the chnage for the registry
+      save_registry(registry_hash)
+      
+      @azure_blob.release_blob_lease(@container, @registry_locker, lease)
+      lease = nil;
+
+      return picked_blob, start_index, gen
+    rescue StandardError => e
+      @logger.error("Oh My, An error occurred. #{e}:\n#{e.backtrace}", :exception => e)
+      return nil, nil, nil
+    ensure
+      @azure_blob.release_blob_lease(@container, @registry_locker, lease) unless lease.nil?
+      lease = nil
+    end # rescue
+  end #register_for_read
+
+  # Update the registry
+  def update_registry (registry_item)
+    begin
+      lease = nil
+      lease = acquire_lease(@registry_locker)
+      registry_hash = load_registry
+      registry_hash[registry_item.file_path] = registry_item
+      save_registry(registry_hash)
+      @azure_blob.release_blob_lease(@container, @registry_locker, lease)
+      lease = nil
+    rescue StandardError => e
+      @logger.error("Oh My, An error occurred. #{e}:\n#{e.backtrace}", :exception => e)
+    ensure
+      @azure_blob.release_blob_lease(@container, @registry_locker, lease) unless lease.nil?
+      lease = nil
+    end #rescue
+  end # def update_registry
+
+  # Clean up the registry.
+  def cleanup_registry
+    begin
+      lease = nil
+      lease = acquire_lease(@registry_locker)
+      registry_hash = load_registry
+      registry_hash.each { | key, registry_item|
+        registry_item.reader = nil if registry_item.reader == @reader
+      }
+      save_registry(registry_hash)
+      @azure_blob.release_blob_lease(@container, @registry_locker, lease)
+      lease = nil
+    rescue StandardError => e
+      @logger.error("Oh My, An error occurred. #{e}:\n#{e.backtrace}", :exception => e)
+    ensure
+      @azure_blob.release_blob_lease(@container, @registry_locker, lease) unless lease.nil?
+      lease = nil
+    end #rescue
+  end # def cleanup_registry
+
+  # Create a registry file to coordinate between multiple azure blob inputs.
+  def create_registry (blob_items)
+    registry_hash = Hash.new
+
+    blob_items.each do |blob_item|
+        initial_offset = 0
+        initial_offset = blob_item.properties[:content_length] if @registry_create_policy == 'resume'
+        registry_item = LogStash::Inputs::RegistryItem.new(blob_item.name, blob_item.properties[:etag], nil, initial_offset, 0)
+      registry_hash[blob_item.name] = registry_item
+    end # each
+    save_registry(registry_hash)
+    return registry_hash
+  end # create_registry
+
+  # Load the content of the registry into the registry hash and return it.
+  def load_registry
+    # Get content
+    registry_blob, registry_blob_body = @azure_blob.get_blob(@container, @registry_path)
+    registry_hash = deserialize_registry_hash(registry_blob_body)
+    return registry_hash
+  end # def load_registry
+
+  # Serialize the registry hash and save it.
+  def save_registry(registry_hash)
+    # Serialize hash to json
+    registry_hash_json = JSON.generate(registry_hash)
+
+    # Upload registry to blob
+    @azure_blob.create_block_blob(@container, @registry_path, registry_hash_json)
+  end # def save_registry
+end # class LogStash::Inputs::LogstashInputAzureblob

data/logstash-input-azureblob.gemspec

@@ -0,0 +1,26 @@
+Gem::Specification.new do |s|
+  s.name          = 'logstash-input-azureblob-json-head-tail'
+  s.version       = '0.9.9'
+  s.licenses      = ['Apache License (2.0)']
+  s.summary       = 'This plugin collects Microsoft Azure Diagnostics data from Azure Storage Blobs.'
+  s.description   = 'This gem is a Logstash plugin. It reads and parses data from Azure Storage Blobs.'
+  s.homepage      = 'https://github.com/Azure/azure-diagnostics-tools'
+  s.authors       = ['Microsoft Corporation']
+  s.email         = 'azdiag@microsoft.com'
+  s.require_paths = ['lib']
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','Gemfile','LICENSE']
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", '>= 1.60', '<= 2.99'
+  s.add_runtime_dependency 'logstash-codec-json_lines'
+  s.add_runtime_dependency 'stud', '>= 0.0.22'
+  s.add_runtime_dependency 'azure-storage', '~> 0.12.3.preview'
+  s.add_development_dependency 'logstash-devutils'
+end

data/spec/inputs/azureblob_spec.rb

@@ -0,0 +1 @@
+require "logstash/devutils/rspec/spec_helper"

metadata

@@ -0,0 +1,131 @@
+--- !ruby/object:Gem::Specification
+name: logstash-input-azureblob-json-head-tail
+version: !ruby/object:Gem::Version
+  version: 0.9.9
+platform: ruby
+authors:
+- Microsoft Corporation
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-08-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - <=
+      - !ruby/object:Gem::Version
+        version: '2.99'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - <=
+      - !ruby/object:Gem::Version
+        version: '2.99'
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-json_lines
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: stud
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.0.22
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.0.22
+- !ruby/object:Gem::Dependency
+  name: azure-storage
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.12.3.preview
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.12.3.preview
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a Logstash plugin. It reads and parses data from Azure Storage
+  Blobs.
+email: azdiag@microsoft.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/logstash/inputs/azureblob.rb
+- spec/inputs/azureblob_spec.rb
+- logstash-input-azureblob.gemspec
+- CHANGELOG.md
+- README.md
+- Gemfile
+- LICENSE
+homepage: https://github.com/Azure/azure-diagnostics-tools
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: input
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14.1
+signing_key: 
+specification_version: 4
+summary: This plugin collects Microsoft Azure Diagnostics data from Azure Storage
+  Blobs.
+test_files:
+- spec/inputs/azureblob_spec.rb