logstash-output-azure 0.3.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 570190e66cae98abbd38587c21422274d212ce6d
-  data.tar.gz: a40db86117fc7b8b53b47d45013df5899e21c8e8
+SHA256:
+  metadata.gz: a0957c333ee69df3b98978bda9a84c26d0d2752f1904fbd104d7d4d58e249095
+  data.tar.gz: dc9b4e1a009d29a98183bcc984825e41680ba83bd51f5968e7933097f532619e
 SHA512:
-  metadata.gz: 9e652e68fc1b07aeb1cd9abed642bf3006eda3f13fbded985e37d5d2446cfdcbf9fec7369c39395d4528c4c3d02d9a516a35678bd9690e9a65ec54e88a154cc4
-  data.tar.gz: 2fe605a3752c850d43ea945fbae618ad1c96ae3ae10a9dc3838322f5bf4eef046a2ec01fa5fa5d6c4e38bb016eac977838f0a4b88cbd61203387575dbb137e90
+  metadata.gz: 65aa1e9eba3e573294306710165a91f06b4f5c304a8020f8f1abd5ce5dde5c3c45d6931665baad8a014a66eec52fe0a81b190e752ce23dedca26aa0d9f724f3b
+  data.tar.gz: 5fcdca862617607b619586f854331aa1eda6a47af66f508d3f04bcd9fb5f953d9a2e966a5cfb141763aab0e5d590f4e15d5b7c3678c71c505c9cc1ad2f902ac2

data/CHANGELOG.md

@@ -1,2 +1,5 @@
-## 0.1.0
+## 0.3.0
   - Plugin created with the logstash plugin generator
+## 1.0.0
+  - updated specs
+  - Documented methods

data/CONTRIBUTORS

@@ -3,6 +3,7 @@ reports, or in general have helped logstash along its way.
 
 Contributors:
 * Tuffk - tuffkmulhall@gmail.com
+* BrunoLerner - bru.lerner@gmail.com
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/lib/logstash/outputs/azure.rb

@@ -1,10 +1,7 @@
-# encoding: utf-8
-
 require 'logstash/outputs/base'
 require 'logstash/namespace'
 require 'azure'
 require 'tmpdir'
-require 'pry'
 
 # Logstash outout plugin that uploads the logs to Azure blobs.
 # The logs are stored on local temporary file which is uploaded as a blob
@@ -32,7 +29,7 @@ require 'pry'
 #   upload que size
 # @!attribute upload workers count
 #   how much workers for uplaod
-# @!attribute rotation_strategy
+# @!attribute rotation_strategy_val
 #   what will be considered to do the tmeporary file rotation
 # @!attribute tags
 #   tags for the files
@@ -40,10 +37,10 @@ require 'pry'
 #   the encoding of the files
 # @example basic configuration
 #    output {
-#      logstash_output_azure {
-#        storage_account_name => "my-azure-account"    # requiered
-#        storage_access_key => "my-super-secret-key"   # requiered
-#        contianer_name => "my-contianer"              # requiered
+#      azure {
+#        storage_account_name => "my-azure-account"    # required
+#        storage_access_key => "my-super-secret-key"   # required
+#        contianer_name => "my-contianer"              # required
 #        size_file => 1024*1024*5                      # optional
 #        time_file => 10                               # optional
 #        restore => true                               # optional
@@ -51,15 +48,14 @@ require 'pry'
 #        prefix => "a_prefix"                          # optional
 #        upload_queue_size => 2                        # optional
 #        upload_workers_count => 1                     # optional
-#        rotation_strategy => "size_and_time"          # optional
+#        rotation_strategy_val => "size_and_time"      # optional
 #        tags => []                                    # optional
 #        encoding => "none"                            # optional
 #      }
 #    }
 class LogStash::Outputs::LogstashAzureBlobOutput < LogStash::Outputs::Base
   # name for the namespace under output for logstash configuration
-  config_name "azure"
-
+  config_name 'azure'
 
   require 'logstash/outputs/blob/writable_directory_validator'
   require 'logstash/outputs/blob/path_validator'
@@ -71,14 +67,11 @@ class LogStash::Outputs::LogstashAzureBlobOutput < LogStash::Outputs::Base
   require 'logstash/outputs/blob/uploader'
   require 'logstash/outputs/blob/file_repository'
 
-  PREFIX_KEY_NORMALIZE_CHARACTER = "_"
+  PREFIX_KEY_NORMALIZE_CHARACTER = '_'.freeze
   PERIODIC_CHECK_INTERVAL_IN_SECONDS = 15
-  CRASH_RECOVERY_THREADPOOL = Concurrent::ThreadPoolExecutor.new({
-                                                                   :min_threads => 1,
-                                                                   :max_threads => 2,
-                                                                   :fallback_policy => :caller_runs
-                                                                 })
-
+  CRASH_RECOVERY_THREADPOOL = Concurrent::ThreadPoolExecutor.new(min_threads: 1,
+                                                                 max_threads: 2,
+                                                                 fallback_policy: :caller_runs)
 
   # azure contianer
   config :storage_account_name, validate: :string, required: false
@@ -97,32 +90,30 @@ class LogStash::Outputs::LogstashAzureBlobOutput < LogStash::Outputs::Base
   config :prefix, validate: :string, default: ''
   config :upload_queue_size, validate: :number, default: 2 * (Concurrent.processor_count * 0.25).ceil
   config :upload_workers_count, validate: :number, default: (Concurrent.processor_count * 0.5).ceil
-  config :rotation_strategy, validate: %w[size_and_time size time], default: 'size_and_time'
-  config :tags, :validate => :array, :default => []
-  config :encoding, :validate => ["none", "gzip"], :default => "none"
+  config :rotation_strategy_val, validate: %w[size_and_time size time], default: 'size_and_time'
+  config :tags, validate: :array, default: []
+  config :encoding, validate: %w[none gzip], default: 'none'
 
-  attr_accessor :storage_account_name, :storage_access_key,:container_name,
-    :size_file,:time_file,:restore,:temporary_directory,:prefix,:upload_queue_size,
-    :upload_workers_count,:rotation_strategy,:tags,:encoding
-
-  public
+  attr_accessor :storage_account_name, :storage_access_key, :container_name,
+                :size_file, :time_file, :restore, :temporary_directory, :prefix, :upload_queue_size,
+                :upload_workers_count, :rotation_strategy_val, :tags, :encoding
 
   # initializes the +LogstashAzureBlobOutput+ instances
-  # validates all canfig parameters
+  # validates all config parameters
   # initializes the uploader
   def register
     unless @prefix.empty?
       unless PathValidator.valid?(prefix)
-        raise LogStash::ConfigurationError, "Prefix must not contains: #{PathValidator::INVALID_CHARACTERS}"
+        raise LogStash::ConfigurationError.new("Prefix must not contains: #{PathValidator::INVALID_CHARACTERS}")
       end
     end
 
     unless WritableDirectoryValidator.valid?(@temporary_directory)
-      raise LogStash::ConfigurationError, "Logstash must have the permissions to write to the temporary directory: #{@temporary_directory}"
+      raise LogStash::ConfigurationError.new("Logstash must have the permissions to write to the temporary directory: #{@temporary_directory}")
     end
 
-    if @time_file.nil? && @size_file.nil? || @size_file == 0 && @time_file == 0
-      raise LogStash::ConfigurationError, 'at least one of time_file or size_file set to a value greater than 0'
+    if @time_file.nil? && @size_file.nil? || @size_file.zero? && @time_file.zero?
+      raise LogStash::ConfigurationError.new('at least one of time_file or size_file set to a value greater than 0')
     end
 
     @file_repository = FileRepository.new(@tags, @encoding, @temporary_directory)
@@ -138,8 +129,10 @@ class LogStash::Outputs::LogstashAzureBlobOutput < LogStash::Outputs::Base
 
     restore_from_crash if @restore
     start_periodic_check if @rotation.needs_periodic?
-  end # def register
+  end
 
+  # Receives multiple events and check if there is space in temporary directory
+  # @param events_and_encoded [Object]
   def multi_receive_encoded(events_and_encoded)
     prefix_written_to = Set.new
 
@@ -152,7 +145,7 @@ class LogStash::Outputs::LogstashAzureBlobOutput < LogStash::Outputs::Base
         # The output should stop accepting new events coming in, since it cannot do anything with them anymore.
         # Log the error and rethrow it.
       rescue Errno::ENOSPC => e
-        @logger.error('S3: No space left in temporary directory', temporary_directory: @temporary_directory)
+        @logger.error('Azure: No space left in temporary directory', temporary_directory: @temporary_directory)
         raise e
       end
     end
@@ -161,7 +154,7 @@ class LogStash::Outputs::LogstashAzureBlobOutput < LogStash::Outputs::Base
     rotate_if_needed(prefix_written_to)
   end
 
-  # close the tmeporary file and uploads the content to Azure
+  # close the temporary file and uploads the content to Azure
   def close
     stop_periodic_check if @rotation.needs_periodic?
 
@@ -182,21 +175,18 @@ class LogStash::Outputs::LogstashAzureBlobOutput < LogStash::Outputs::Base
     @crash_uploader.stop if @restore # we might have still work to do for recovery so wait until we are done
   end
 
+  # Validates and normalize prefix key
+  # @param prefix_key [String]
   def normalize_key(prefix_key)
     prefix_key.gsub(PathValidator.matches_re, PREFIX_KEY_NORMALIZE_CHARACTER)
   end
 
-  def upload_options
-    {
-    }
-  end
-
   # checks periodically the tmeporary file if it needs to be rotated
   def start_periodic_check
-    @logger.debug("Start periodic rotation check")
+    @logger.debug('Start periodic rotation check')
 
-    @periodic_check = Concurrent::TimerTask.new(:execution_interval => PERIODIC_CHECK_INTERVAL_IN_SECONDS) do
-      @logger.debug("Periodic check for stale files")
+    @periodic_check = Concurrent::TimerTask.new(execution_interval: PERIODIC_CHECK_INTERVAL_IN_SECONDS) do
+      @logger.debug('Periodic check for stale files')
 
       rotate_if_needed(@file_repository.keys)
     end
@@ -208,21 +198,23 @@ class LogStash::Outputs::LogstashAzureBlobOutput < LogStash::Outputs::Base
     @periodic_check.shutdown
   end
 
-  # login to azure cloud using azure gem and get the contianer if exist or create
-  # the continer if it doesn't
+  # login to azure cloud using azure gem and create the contianer if it doesn't exist
+  # @return [Object] the azure_blob_service object, which is the endpoint to azure gem
   def blob_container_resource
     Azure.config.storage_account_name = storage_account_name
     Azure.config.storage_access_key = storage_access_key
     azure_blob_service = Azure::Blob::BlobService.new
-    list = azure_blob_service.list_containers()
+    list = azure_blob_service.list_containers
     list.each do |item|
       @container = item if item.name == container_name
     end
 
     azure_blob_service.create_container(container_name) unless @container
-    return azure_blob_service
+    azure_blob_service
   end
 
+  # check if it needs to rotate according to rotation policy and rotates it if it needs
+  # @param prefixes [String]
   def rotate_if_needed(prefixes)
     prefixes.each do |prefix|
       # Each file access is thread safe,
@@ -232,10 +224,10 @@ class LogStash::Outputs::LogstashAzureBlobOutput < LogStash::Outputs::Base
         temp_file = factory.current
 
         if @rotation.rotate?(temp_file)
-          @logger.debug("Rotate file",
-                        :strategy => @rotation.class.name,
-                        :key => temp_file.key,
-                        :path => temp_file.path)
+          @logger.debug('Rotate file',
+                        strategy: @rotation.class.name,
+                        key: temp_file.key,
+                        path: temp_file.path)
 
           upload_file(temp_file)
           factory.rotate!
@@ -246,60 +238,46 @@ class LogStash::Outputs::LogstashAzureBlobOutput < LogStash::Outputs::Base
 
   # uploads the file using the +Uploader+
   def upload_file(temp_file)
-    @logger.debug("Queue for upload", :path => temp_file.path)
+    @logger.debug('Queue for upload', path: temp_file.path)
 
     # if the queue is full the calling thread will be used to upload
     temp_file.close # make sure the content is on disk
-    if temp_file.size > 0
+    unless temp_file.empty? # rubocop:disable GuardClause
       @uploader.upload_async(temp_file,
-                             :on_complete => method(:clean_temporary_file),
-                             :upload_options => upload_options )
+                             on_complete: method(:clean_temporary_file),
+                             upload_options: upload_options)
     end
   end
 
   # creates an instance for the rotation strategy
   def rotation_strategy
-    case @rotation_strategy
-    when "size"
+    case @rotation_strategy_val
+    when 'size'
       SizeRotationPolicy.new(size_file)
-    when "time"
+    when 'time'
       TimeRotationPolicy.new(time_file)
-    when "size_and_time"
+    when 'size_and_time'
       SizeAndTimeRotationPolicy.new(size_file, time_file)
     end
   end
 
+  # Cleans the temporary files after it is uploaded to azure blob
   def clean_temporary_file(file)
-    @logger.debug("Removing temporary file", :file => file.path)
+    @logger.debug('Removing temporary file', file: file.path)
     file.delete!
   end
 
+  # uploads files if there was a crash before
   def restore_from_crash
     @crash_uploader = Uploader.new(blob_container_resource, container_name, @logger, CRASH_RECOVERY_THREADPOOL)
 
     temp_folder_path = Pathname.new(@temporary_directory)
-    Dir.glob(::File.join(@temporary_directory, "**/*"))
-      .select { |file| ::File.file?(file) }
-      .each do |file|
+    Dir.glob(::File.join(@temporary_directory, '**/*'))
+       .select { |file| ::File.file?(file) }
+       .each do |file|
       temp_file = TemporaryFile.create_from_existing_file(file, temp_folder_path)
-      @logger.debug("Recovering from crash and uploading", :file => temp_file.path)
-      @crash_uploader.upload_async(temp_file, :on_complete => method(:clean_temporary_file), :upload_options => upload_options)
+      @logger.debug('Recovering from crash and uploading', file: temp_file.path)
+      @crash_uploader.upload_async(temp_file, on_complete: method(:clean_temporary_file), upload_options: upload_options)
     end
   end
-
-
-  public
-
-  def receive(event)
-    azure_login
-    azure_blob_service = Azure::Blob::BlobService.new
-    containers = azure_blob_service.list_containers
-    blob = azure_blob_service.create_block_blob(containers[0].name, event.timestamp.to_s, event.to_json)
-  end # def event
-
-  # inputs the credentials to the azure gem to log in and use azure API
-  def azure_login
-    Azure.config.storage_account_name ||= storage_account_name
-    Azure.config.storage_access_key ||= storage_access_key
-  end # def azure_login
-end # class LogStash::Outputs::LogstashAzureBlobOutput
+end

data/lib/logstash/outputs/blob/file_repository.rb

@@ -1,8 +1,8 @@
-# encoding: utf-8
-require "java"
-require "concurrent"
-require "concurrent/timer_task"
-require "logstash/util"
+
+require 'java'
+require 'concurrent'
+require 'concurrent/timer_task'
+require 'logstash/util'
 
 ConcurrentHashMap = java.util.concurrent.ConcurrentHashMap
 
@@ -10,40 +10,46 @@ module LogStash
   module Outputs
     class LogstashAzureBlobOutput
       # sub class for +LogstashAzureBlobOutput+
-      # this class manages the tmeporary directory for the temporary files
+      # this class manages the temporary directory for the temporary files
       class FileRepository
         DEFAULT_STATE_SWEEPER_INTERVAL_SECS = 60
         DEFAULT_STALE_TIME_SECS = 15 * 60
         # Ensure that all access or work done
         # on a factory is threadsafe
         class PrefixedValue
+          # initialize the factory
           def initialize(file_factory, stale_time)
             @file_factory = file_factory
             @lock = Mutex.new
             @stale_time = stale_time
           end
 
+          # activate the lock
           def with_lock
-            @lock.synchronize {
+            @lock.synchronize do
               yield @file_factory
-            }
+            end
           end
 
+          # boolean method
           def stale?
-            with_lock { |factory| factory.current.size == 0 && (Time.now - factory.current.ctime > @stale_time) }
+            with_lock { |factory| factory.current.size.zero? && (Time.now - factory.current.ctime > @stale_time) }
           end
 
-          def apply(prefix)
-            return self
+          # return this class
+          def apply(_prefix)
+            self
           end
 
+          # delete the current factory
           def delete!
-            with_lock{ |factory| factory.current.delete! }
+            with_lock { |factory| factory.current.delete! }
           end
         end
 
         # class for initializing the repo manager
         class FactoryInitializer
+          # initializes the class
           def initialize(tags, encoding, temporary_directory, stale_time)
             @tags = tags
             @encoding = encoding
@@ -51,17 +57,18 @@ module LogStash
             @stale_time = stale_time
           end
 
+          # applies the prefix key
           def apply(prefix_key)
             PrefixedValue.new(TemporaryFileFactory.new(prefix_key, @tags, @encoding, @temporary_directory), @stale_time)
           end
         end
-
+        # initializes the class with more variables
         def initialize(tags, encoding, temporary_directory,
                        stale_time = DEFAULT_STALE_TIME_SECS,
                        sweeper_interval = DEFAULT_STATE_SWEEPER_INTERVAL_SECS)
           # The path need to contains the prefix so when we start
           # logtash after a crash we keep the remote structure
-          @prefixed_factories =  ConcurrentHashMap.new
+          @prefixed_factories = ConcurrentHashMap.new
 
           @sweeper_interval = sweeper_interval
 
@@ -70,10 +77,12 @@ module LogStash
           start_stale_sweeper
         end
 
+        # gets the key set
         def keys
           @prefixed_factories.keySet
         end
 
+        # with lock for each file
         def each_files
           @prefixed_factories.elements.each do |prefixed_file|
             prefixed_file.with_lock { |factory| yield factory.current }
@@ -85,35 +94,41 @@ module LogStash
           @prefixed_factories.computeIfAbsent(prefix_key, @factory_initializer).with_lock { |factory| yield factory }
         end
 
+        # gets file from prefix_key
         def get_file(prefix_key)
           get_factory(prefix_key) { |factory| yield factory.current }
         end
 
+        # stops. shutdown
         def shutdown
           stop_stale_sweeper
         end
 
+        # gets factory's size
         def size
           @prefixed_factories.size
         end
 
+        # remove the stale given key and value
         def remove_stale(k, v)
-          if v.stale?
+          if v.stale? # rubocop:disable Style/GuardClause
             @prefixed_factories.remove(k, v)
             v.delete!
           end
         end
 
+        # starts the stale sweeper
         def start_stale_sweeper
-          @stale_sweeper = Concurrent::TimerTask.new(:execution_interval => @sweeper_interval) do
-            LogStash::Util.set_thread_name("LogstashAzureBlobOutput, Stale factory sweeper")
+          @stale_sweeper = Concurrent::TimerTask.new(execution_interval: @sweeper_interval) do
+            LogStash::Util.set_thread_name('LogstashAzureBlobOutput, Stale factory sweeper')
 
-            @prefixed_factories.forEach{|k,v| remove_stale(k,v)}
+            @prefixed_factories.forEach { |k, v| remove_stale(k, v) }
           end
 
           @stale_sweeper.execute
         end
 
+        # stops the stale sweeper
         def stop_stale_sweeper
           @stale_sweeper.shutdown
         end