logstash-input-azureblob 0.9.7 → 0.9.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cdf42bfd9738cdc286425cd943c5953cc496c23d
-  data.tar.gz: 2a71a8791f65b51dbf7293ac4885c2006043128a
+  metadata.gz: e72dbef9d73b6b872b786b1cd40ffd3434835ac4
+  data.tar.gz: 9054b485b8073ca0dc9cce2e09c125630caec5fc
 SHA512:
-  metadata.gz: c819309d516003a9efbe45c9760a350be800f62d9331ab69fc4d22b9fe0c57062016d81120eb6409411432c4a1ceaeca474a56038ecde1560a7ae545fc7ac211
-  data.tar.gz: 107f91f5494a2465ef6eae5114c5b352043ddb0996d5be052ab76d0d51b314897266778f978efc4c07477a49ba75c0c2864459c11d3432e28307d30a55bbd081
+  metadata.gz: 917b7f0ad4b0a552091a6055c9e5fd5eeef38d6ce0ce85166eb432d2a5737a8d49f75c28579ce0119ed1887c952a0c372dc620cdf41bcdfdd9d30e9160e33c0a
+  data.tar.gz: 3f513712c43563b6ea3407b23516e784b1d4d8fc706213247132b57827f7605ed55cbcacb9c357525fb84bc18920ab367a03e411b658d56e4da7506152560767

data/README.md

@@ -27,7 +27,34 @@ The blob container name.
 ### Optional Parameters
 __*endpoint*__
 
-Specifies the endpoint of Azure Service Management. The default value is "core.windows.net". 
+Specifies the endpoint of Azure Service Management. The default value is `core.windows.net`. 
+
+__*registry_path*__
+
+Specifies the file path for the registry file to record offsets and coordinate between multiple clients. The default value is `data/registry`.
+
+Overwrite this value when there happen to be a file at the path of `data/registry` in the azure blob container.
+
+__*interval*__
+
+Set how many seconds to idle before checking for new logs. The default, `30`, means idle for `30` seconds.
+
+__*registry_create_policy*__
+
+Specifies the way to initially set offset for existing blob files.
+
+This option only applies for registry creation. 
+
+Valid values include:
+
+  - resume
+  - start_over
+
+The default, `resume`, means when the registry is initially created, it assumes all blob has been consumed and it will start to pick up any new content in the blobs.
+
+When set to `start_over`, it assumes none of the blob is consumed and it will read all blob files from begining.
+
+Offsets will be picked up from registry file whenever it exists.
 
 ### Examples
 ```

data/lib/logstash/inputs/azureblob.rb

@@ -2,98 +2,344 @@
 require "logstash/inputs/base"
 require "logstash/namespace"
 
-require "azure"
-require "securerandom"
+# Azure Storage SDK for Ruby
+require "azure/storage"
+require 'json' # for registry content
+require "securerandom" # for generating uuid.
 
-# Reads events from Azure Blobs
-class LogStash::Inputs::Azureblob < LogStash::Inputs::Base
+# Registry item to coordinate between mulitple clients
+class LogStash::Inputs::RegistryItem
+  attr_accessor :file_path, :etag, :offset, :reader, :gen
+  # Allow json serialization.
+  def as_json(options={})
+    {
+      file_path: @file_path,
+      etag: @etag,
+      reader: @reader,
+      offset: @offset,
+      gen: @gen
+    }
+  end # as_json
 
+  def to_json(*options)
+    as_json(*options).to_json(*options)
+  end # to_json
+
+  def initialize(file_path, etag, reader, offset = 0, gen = 0)
+    @file_path = file_path
+    @etag = etag
+    @reader = reader
+    @offset = offset
+    @gen = gen
+  end # initialize
+end # class RegistryItem
+
+
+# Logstash input plugin for Azure Blobs
+#
+# This logstash plugin gathers data from Microsoft Azure Blobs
+class LogStash::Inputs::LogstashInputAzureblob < LogStash::Inputs::Base
   config_name "azureblob"
-  milestone 1
-  
+
+  # If undefined, Logstash will complain, even if codec is unused.
   default :codec, "json_lines"
-  
+
+  # Set the account name for the azure storage account.
   config :storage_account_name, :validate => :string
+  
+  # Set the key to access the storage account.
   config :storage_access_key, :validate => :string
   
+  # Set the container of the blobs.
   config :container, :validate => :string
-  config :sleep_time, :validate => :number, :default => 10
-  config :endpoint, :validate => :string, :default => "core.windows.net"
   
-  def initialize(*args)
-    super(*args)
-  end # def initialize
+  # Set the endpoint for the blobs.
+  #
+  # The default, `core.windows.net` targets the public azure.
+  config :endpoint, :validate => :string, :default => 'core.windows.net'
+
+  # Set the value of using backup mode.
+  config :backupmode, :validate => :boolean, :default => false, :deprecated => true, :obsolete => 'This option is obsoleted and the settings will be ignored.'
+  
+  # Set the value for the registry file.
+  #
+  # The default, `data/registry`, is used to coordinate readings for various instances of the clients.
+  config :registry_path, :validate => :string, :default => 'data/registry'
   
+  # Set how many seconds to keep idle before checking for new logs.
+  #
+  # The default, `30`, means trigger a reading for the log every 30 seconds after entering idle.
+  config :interval, :validate => :number, :default => 30
+
+  # Set the registry create mode
+  #
+  # The default, `resume`, means when the registry is initially created, it assumes all logs has been handled.
+  # When set to `start_over`, it will read all log files from begining.
+  config :registry_create_policy, :validate => :string, :default => 'resume'
+
+  # Constant of max integer
+  MAX = 2 ** ([42].pack('i').size * 16 -2 ) -1
+
   public
   def register
-    Azure.configure do |config|
-      config.storage_account_name = @storage_account_name
-      config.storage_access_key = @storage_access_key
-      config.storage_blob_host = "https://#{@storage_account_name}.blob.#{@endpoint}"
-    end
-    @azure_blob = Azure::Blob::BlobService.new
+    # this is the reader # for this specific instance.
+    @reader = SecureRandom.uuid
+    @registry_locker = "#{@registry_path}.lock"
+   
+    # Setup a specific instance of an Azure::Storage::Client
+    client = Azure::Storage::Client.create(:storage_account_name => @storage_account_name, :storage_access_key => @storage_access_key, :storage_blob_host => "https://#{@storage_account_name}.blob.#{@endpoint}")
+    # Get an azure storage blob service object from a specific instance of an Azure::Storage::Client
+    @azure_blob = client.blob_client
+    # Add retry filter to the service object
+    @azure_blob.with_filter(Azure::Storage::Core::Filter::ExponentialRetryPolicyFilter.new)
   end # def register
+
+  def run(queue)
+    # we can abort the loop if stop? becomes true
+    while !stop?
+      process(queue)
+      Stud.stoppable_sleep(@interval) { stop? }
+    end # loop
+  end # def run
+
+  def stop
+    cleanup_registry
+  end # def stop
   
-  def list_blob_names
-    blob_names = Set.new []
+  # Start processing the next item.
+  def process(queue)
+    begin
+      blob, start_index, gen = register_for_read
+
+      if(!blob.nil?)
+        begin
+          blob_name = blob.name
+          # Work-around: After returned by get_blob, the etag will contains quotes.
+          new_etag = blob.properties[:etag]
+          # ~ Work-around
+          blob, content = @azure_blob.get_blob(@container, blob_name, {:start_range=>start_index} )
+          
+          @codec.decode(content) do |event|
+            decorate(event)
+            queue << event
+          end # decode
+        ensure
+          # Making sure the reader is removed from the registry even when there's exception.
+          new_offset = start_index
+          new_offset = new_offset + content.length unless content.nil?
+          new_registry_item = LogStash::Inputs::RegistryItem.new(blob_name, new_etag, nil, new_offset, gen)
+          update_registry(new_registry_item)
+        end # begin
+      end # if
+    rescue StandardError => e
+      @logger.error("Oh My, An error occurred. \nError:#{e}:\nTrace:\n#{e.backtrace}", :exception => e)
+    end # begin
+  end # process
+
+  # Deserialize registry hash from json string.
+  def deserialize_registry_hash (json_string)
+    result = Hash.new
+    temp_hash = JSON.parse(json_string)
+    temp_hash.values.each { |kvp|
+      result[kvp['file_path']] = LogStash::Inputs::RegistryItem.new(kvp['file_path'], kvp['etag'], kvp['reader'], kvp['offset'], kvp['gen'])
+    }
+    return result
+  end #deserialize_registry_hash
+
+  # List all the blobs in the given container.
+  def list_all_blobs
+    blobs = Set.new []
     continuation_token = NIL
     loop do
       entries = @azure_blob.list_blobs(@container, { :timeout => 10, :marker => continuation_token})
       entries.each do |entry|
-        blob_names << entry.name
-      end
+        blobs << entry
+      end # each
       continuation_token = entries.continuation_token
       break if continuation_token.empty?
-    end
-    return blob_names
+    end # loop
+    return blobs
   end # def list_blobs
-  
-  def acquire_lock(blob_name)
-    @azure_blob.create_page_blob(@container, blob_name, 512)
-    @azure_blob.acquire_lease(@container, blob_name,{:duration=>60, :timeout=>10, :proposed_lease_id=>SecureRandom.uuid})
-    return true
-  rescue LogStash::ShutdownSignal => e
-    raise e
-  rescue => e
-    @logger.error("Caught exception while locking", :exception => e)
-    return false
-  end # def acquire_lock
-  
-  def lock_blob(blob_names)
-    real_blob_names = blob_names.select { |name| !name.end_with?(".lock") }
-    real_blob_names.each do |blob_name|
-      if !blob_names.include?(blob_name + ".lock")
-        if acquire_lock(blob_name + ".lock")
-          return blob_name
-        end
+
+  # Raise generation for blob in registry
+  def raise_gen(registry_hash, file_path)
+    begin
+      target_item = registry_hash[file_path]
+      begin
+        target_item.gen += 1
+        # Protect gen from overflow.
+        target_item.gen = target_item.gen / 2 if target_item.gen == MAX
+      rescue StandardError => e
+        @logger.error("Fail to get the next generation for target item #{target_item}.", :exception => e)
+        target_item.gen = 0
+      end
+
+      min_gen_item = registry_hash.values.min_by { |x| x.gen }
+      while min_gen_item.gen > 0
+        registry_hash.values.each { |value| 
+          value.gen -= 1
+        }
+        min_gen_item = registry_hash.values.min_by { |x| x.gen }
       end
     end
-    return NIL
-  end # def lock_blob
-  
-  def process(output_queue)
-    blob_names = list_blob_names
-    blob_name = lock_blob(blob_names)
-    return if !blob_name
-    blob, content = @azure_blob.get_blob(@container, blob_name)
-    @codec.decode(content) do |event|
-      decorate(event)
-      output_queue << event
-    end
-  rescue LogStash::ShutdownSignal => e
-    raise e
-  rescue => e
-    @logger.error("Oh My, An error occurred.", :exception => e)
-  end # def process
-  
-  public
-  def run(output_queue)
-    while !stop?
-      process(output_queue)
-    end # loop
-  end # def run
- 
-  public
-  def teardown
-  end # def teardown
-end # class LogStash::Inputs::Azuretopic
+  end # raise_gen
+
+  # Acquire a lease on a blob item with retries.
+  #
+  # By default, it will retry 30 times with 1 second interval.
+  def acquire_lease(blob_name, retry_times = 30, interval_sec = 1)
+    lease = nil;
+    retried = 0;
+    while lease.nil? do
+      begin
+        lease = @azure_blob.acquire_blob_lease(@container, blob_name, {:timeout => 10})
+      rescue StandardError => e
+        if(e.type == 'LeaseAlreadyPresent')
+            if (retried > retry_times)
+                raise
+            end
+            retried += 1
+            sleep interval_sec
+        end
+      end
+    end #while
+    return lease
+  end # acquire_lease
+
+  # Return the next blob for reading as well as the start index.
+  def register_for_read
+    begin
+      all_blobs = list_all_blobs
+      registry = all_blobs.find { |item| item.name.downcase == @registry_path  }
+      registry_locker = all_blobs.find { |item| item.name.downcase == @registry_locker }
+
+      candidate_blobs = all_blobs.select { |item| (item.name.downcase != @registry_path) && ( item.name.downcase != @registry_locker ) }
+      
+      start_index = 0
+      gen = 0
+      lease = nil
+
+      # Put lease on locker file than the registy file to allow update of the registry as a workaround for Azure Storage Ruby SDK issue # 16.
+      # Workaround: https://github.com/Azure/azure-storage-ruby/issues/16
+      registry_locker = @azure_blob.create_block_blob(@container, @registry_locker, @reader) if registry_locker.nil?
+      lease = acquire_lease(@registry_locker)
+      # ~ Workaround
+
+      if(registry.nil?)
+        registry_hash = create_registry(candidate_blobs)
+      else
+        registry_hash = load_registry
+      end #if
+        
+      picked_blobs = Set.new []
+      # Pick up the next candidate
+      picked_blob = nil
+      candidate_blobs.each { |candidate_blob|
+        registry_item = registry_hash[candidate_blob.name]
+
+        # Appending items that doesn't exist in the hash table
+        if registry_item.nil?
+          registry_item = LogStash::Inputs::RegistryItem.new(candidate_blob.name, candidate_blob.properties[:etag], nil, 0, 0)
+          registry_hash[candidate_blob.name] = registry_item
+        end # if
+        
+        if ((registry_item.offset < candidate_blob.properties[:content_length]) && (registry_item.reader.nil? || registry_item.reader == @reader))
+          picked_blobs << candidate_blob
+        end
+      }
+
+      picked_blob = picked_blobs.min_by { |b| registry_hash[b.name].gen }
+      if !picked_blob.nil?
+        registry_item = registry_hash[picked_blob.name]
+        registry_item.reader = @reader
+        registry_hash[picked_blob.name] = registry_item
+        start_index = registry_item.offset
+        raise_gen(registry_hash, picked_blob.name)
+        gen = registry_item.gen
+      end #if
+
+      # Save the chnage for the registry
+      save_registry(registry_hash)
+      
+      @azure_blob.release_blob_lease(@container, @registry_locker, lease)
+      lease = nil;
+
+      return picked_blob, start_index, gen
+    rescue StandardError => e
+      @logger.error("Oh My, An error occurred. #{e}:\n#{e.backtrace}", :exception => e)
+      return nil, nil, nil
+    ensure
+      @azure_blob.release_blob_lease(@container, @registry_locker, lease) unless lease.nil?
+      lease = nil
+    end # rescue
+  end #register_for_read
+
+  # Update the registry
+  def update_registry (registry_item)
+    begin
+      lease = nil
+      lease = acquire_lease(@registry_locker)
+      registry_hash = load_registry
+      registry_hash[registry_item.file_path] = registry_item
+      save_registry(registry_hash)
+      @azure_blob.release_blob_lease(@container, @registry_locker, lease)
+      lease = nil
+    rescue StandardError => e
+      @logger.error("Oh My, An error occurred. #{e}:\n#{e.backtrace}", :exception => e)
+    ensure
+      @azure_blob.release_blob_lease(@container, @registry_locker, lease) unless lease.nil?
+      lease = nil
+    end #rescue
+  end # def update_registry
+
+  # Clean up the registry.
+  def cleanup_registry
+    begin
+      lease = nil
+      lease = acquire_lease(@registry_locker)
+      registry_hash = load_registry
+      registry_hash.each { | key, registry_item|
+        registry_item.reader = nil if registry_item.reader == @reader
+      }
+      save_registry(registry_hash)
+      @azure_blob.release_blob_lease(@container, @registry_locker, lease)
+      lease = nil
+    rescue StandardError => e
+      @logger.error("Oh My, An error occurred. #{e}:\n#{e.backtrace}", :exception => e)
+    ensure
+      @azure_blob.release_blob_lease(@container, @registry_locker, lease) unless lease.nil?
+      lease = nil
+    end #rescue
+  end # def cleanup_registry
+
+  # Create a registry file to coordinate between multiple azure blob inputs.
+  def create_registry (blob_items)
+    registry_hash = Hash.new
+
+    blob_items.each do |blob_item|
+        initial_offset = 0
+        initial_offset = blob_item.properties[:content_length] if @registry_create_policy == 'resume'
+        registry_item = LogStash::Inputs::RegistryItem.new(blob_item.name, blob_item.properties[:etag], nil, initial_offset, 0)
+      registry_hash[blob_item.name] = registry_item
+    end # each
+    save_registry(registry_hash)
+    return registry_hash
+  end # create_registry
+
+  # Load the content of the registry into the registry hash and return it.
+  def load_registry
+    # Get content
+    registry_blob, registry_blob_body = @azure_blob.get_blob(@container, @registry_path)
+    registry_hash = deserialize_registry_hash(registry_blob_body)
+    return registry_hash
+  end # def load_registry
+
+  # Serialize the registry hash and save it.
+  def save_registry(registry_hash)
+    # Serialize hash to json
+    registry_hash_json = JSON.generate(registry_hash)
+
+    # Upload registry to blob
+    @azure_blob.create_block_blob(@container, @registry_path, registry_hash_json)
+  end # def save_registry
+end # class LogStash::Inputs::LogstashInputAzureblob

data/logstash-input-azureblob.gemspec

@@ -1,13 +1,13 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-input-azureblob'
-  s.version       = '0.9.7'
+  s.version       = '0.9.8'
   s.licenses      = ['Apache License (2.0)']
-  s.summary       = "This plugin collects Microsoft Azure Diagnostics data from Azure Storage Blobs."
-  s.description   = "This gem is a Logstash plugin. It reads and parses data from Azure Storage Blobs."
-  s.authors       = ["Microsoft Corporation"]
+  s.summary       = 'This plugin collects Microsoft Azure Diagnostics data from Azure Storage Blobs.'
+  s.description   = 'This gem is a Logstash plugin. It reads and parses data from Azure Storage Blobs.'
+  s.homepage      = 'https://github.com/Azure/azure-diagnostics-tools'
+  s.authors       = ['Microsoft Corporation']
   s.email         = 'azdiag@microsoft.com'
-  s.homepage      = "https://github.com/Azure/azure-diagnostics-tools"
-  s.require_paths = ["lib"]
+  s.require_paths = ['lib']
 
   # Files
   s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','Gemfile','LICENSE']
@@ -18,7 +18,9 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
 
   # Gem dependencies
-  s.add_runtime_dependency 'logstash-core-plugin-api', '>= 1.60', '<= 2.99'
-  s.add_runtime_dependency 'azure', '~> 0.7.1'
+  s.add_runtime_dependency "logstash-core-plugin-api", '>= 1.60', '<= 2.99'
+  s.add_runtime_dependency 'logstash-codec-json_lines'
+  s.add_runtime_dependency 'stud', '>= 0.0.22'
+  s.add_runtime_dependency 'azure-storage', '~> 0.11.4.preview'
   s.add_development_dependency 'logstash-devutils'
 end

metadata

@@ -1,18 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-azureblob
 version: !ruby/object:Gem::Version
-  version: 0.9.7
+  version: 0.9.8
 platform: ruby
 authors:
 - Microsoft Corporation
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-02-08 00:00:00.000000000 Z
+date: 2017-07-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: logstash-core-plugin-api
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
@@ -20,7 +19,10 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
-  requirement: !ruby/object:Gem::Requirement
+  name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
@@ -28,36 +30,62 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-json_lines
   prerelease: false
   type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: azure
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.22
+  name: stud
+  prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.7.1
+        version: 0.0.22
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.7.1
+        version: 0.11.4.preview
+  name: azure-storage
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.11.4.preview
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-devutils
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: This gem is a Logstash plugin. It reads and parses data from Azure Storage Blobs.
 email: azdiag@microsoft.com
 executables: []