logstash-output-azure 0.3.0 → 1.0.0

data/lib/logstash/outputs/blob/path_validator.rb

@@ -1,16 +1,16 @@
-# encoding: utf-8
 module LogStash
   module Outputs
     class LogstashAzureBlobOutput
       # a sub class of +LogstashAzureBlobOutput+
       # valdiates the path for the temporary directory
       class PathValidator
-        INVALID_CHARACTERS = "\^`><"
-
+        INVALID_CHARACTERS = "\^`><".freeze
+        # boolean method to check if a name is valid
         def self.valid?(name)
           name.match(matches_re).nil?
         end
 
+        # define the invalid characters that shouldn't be in the path name
         def self.matches_re
           /[#{Regexp.escape(INVALID_CHARACTERS)}]/
         end

data/lib/logstash/outputs/blob/size_and_time_rotation_policy.rb

@@ -1,22 +1,24 @@
-# encoding: utf-8
-require "logstash/outputs/blob/size_rotation_policy"
-require "logstash/outputs/blob/time_rotation_policy"
+require 'logstash/outputs/blob/size_rotation_policy'
+require 'logstash/outputs/blob/time_rotation_policy'
 
 module LogStash
   module Outputs
     class LogstashAzureBlobOutput
       # a sub class of +LogstashAzureBlobOutput+
-      # sets the rotation policy 
+      # sets the rotation policy
       class SizeAndTimeRotationPolicy
+        # initialize the class
         def initialize(file_size, time_file)
           @size_strategy = SizeRotationPolicy.new(file_size)
           @time_strategy = TimeRotationPolicy.new(time_file)
         end
 
+        # check if it is time to rotate
         def rotate?(file)
           @size_strategy.rotate?(file) || @time_strategy.rotate?(file)
         end
 
+        # boolean method
         def needs_periodic?
           true
         end

data/lib/logstash/outputs/blob/size_rotation_policy.rb

@@ -1,24 +1,25 @@
-# encoding: utf-8
 module LogStash
   module Outputs
     class LogstashAzureBlobOutput
       # a sub class of +LogstashAzureBlobOutput+
-      # sets the rotation policy by size 
+      # sets the rotation policy by size
       class SizeRotationPolicy
         attr_reader :size_file
-
+        # initialize the class
         def initialize(size_file)
           if size_file <= 0
-            raise LogStash::ConfigurationError, "`size_file` need to be greather than 0"
+            raise LogStash::ConfigurationError.new('`size_file` need to be greather than 0')
           end
 
           @size_file = size_file
         end
 
+        # boolean method to check if it is time to rotate
         def rotate?(file)
           file.size >= size_file
         end
 
+        # boolean method
         def needs_periodic?
           false
         end

data/lib/logstash/outputs/blob/temporary_file.rb

@@ -1,7 +1,6 @@
-# encoding: utf-8
-require "thread"
-require "forwardable"
-require "fileutils"
+require 'thread'
+require 'forwardable'
+require 'fileutils'
 
 module LogStash
   module Outputs
@@ -16,6 +15,7 @@ module LogStash
 
         attr_reader :fd
 
+        # initialize the class
         def initialize(key, fd, temp_path)
           @fd = fd
           @key = key
@@ -23,27 +23,28 @@ module LogStash
           @created_at = Time.now
         end
 
+        # gets the created at time
         def ctime
           @created_at
         end
 
-        def temp_path
-          @temp_path
-        end
+        # gets path to temporary directory
+        attr_reader :temp_path
 
+        # gets the size of file
         def size
           # Use the fd size to get the accurate result,
           # so we dont have to deal with fsync
           # if the file is close we will use the File::size
-          begin
-            @fd.size
-          rescue IOError
-            ::File.size(path)
-          end
+
+          @fd.size
+        rescue IOError
+          ::File.size(path)
         end
 
+        # gets the key
         def key
-          @key.gsub(/^\//, "")
+          @key.gsub(/^\//, '')
         end
 
         # Each temporary file is made inside a directory named with an UUID,
@@ -51,20 +52,28 @@ module LogStash
         # we delete the root of the UUID, using a UUID also remove the risk of deleting unwanted file, it acts as
         # a sandbox.
         def delete!
-          @fd.close rescue IOError # force close anyway
-          FileUtils.rm_r(@temp_path, :secure => true)
+          begin
+            @fd.close
+          rescue
+            IOError
+          end
+          FileUtils.rm_r(@temp_path, secure: true)
         end
 
+        # boolean method to determine if the file is empty
         def empty?
-          size == 0
+          size.zero?
         end
 
+        # creates the temporary file in an existing temporary directory from existing file
+        # @param file_path [String] path to the file
+        # @param temporary_folder [String] path to the temporary folder
         def self.create_from_existing_file(file_path, temporary_folder)
           key_parts = Pathname.new(file_path).relative_path_from(temporary_folder).to_s.split(::File::SEPARATOR)
 
-          TemporaryFile.new(key_parts.slice(1, key_parts.size).join("/"),
-                         ::File.open(file_path, "r"),
-                         ::File.join(temporary_folder, key_parts.slice(0, 1)))
+          TemporaryFile.new(key_parts.slice(1, key_parts.size).join('/'),
+                            ::File.open(file_path, 'r'),
+                            ::File.join(temporary_folder, key_parts.slice(0, 1)))
         end
       end
     end

data/lib/logstash/outputs/blob/temporary_file_factory.rb

@@ -1,9 +1,8 @@
-# encoding: utf-8
-require "socket"
-require "securerandom"
-require "fileutils"
-require "zlib"
-require "forwardable"
+require 'socket'
+require 'securerandom'
+require 'fileutils'
+require 'zlib'
+require 'forwardable'
 
 module LogStash
   module Outputs
@@ -11,14 +10,15 @@ module LogStash
       # a sub class of +LogstashAzureBlobOutput+
       # creates the temporary files to write and later upload
       class TemporaryFileFactory
-        FILE_MODE = "a"
-        GZIP_ENCODING = "gzip"
-        GZIP_EXTENSION = "txt.gz"
-        TXT_EXTENSION = "txt"
-        STRFTIME = "%Y-%m-%dT%H.%M"
+        FILE_MODE = 'a'.freeze
+        GZIP_ENCODING = 'gzip'.freeze
+        GZIP_EXTENSION = 'txt.gz'.freeze
+        TXT_EXTENSION = 'txt'.freeze
+        STRFTIME = '%Y-%m-%dT%H.%M'.freeze
 
         attr_accessor :counter, :tags, :prefix, :encoding, :temporary_directory, :current
 
+        # initialize the class
         def initialize(prefix, tags, encoding, temporary_directory)
           @counter = 0
           @prefix = prefix
@@ -31,41 +31,49 @@ module LogStash
           rotate!
         end
 
+        # do the rotation
         def rotate!
-          @lock.synchronize {
+          @lock.synchronize do
             @current = new_file
             increment_counter
             @current
-          }
+          end
         end
 
         private
+
+        # if it is not gzip ecoding, then it is txt extension
         def extension
           gzip? ? GZIP_EXTENSION : TXT_EXTENSION
         end
 
+        # boolean method to check if its gzip encoding
         def gzip?
           encoding == GZIP_ENCODING
         end
 
+        # increment the counter in 1 unit
         def increment_counter
           @counter += 1
         end
 
+        # gets the current time
         def current_time
           Time.now.strftime(STRFTIME)
         end
 
+        # method that generate the name of the file to be saved in blob storage
         def generate_name
           filename = "#{current_time}.#{SecureRandom.uuid}"
 
-          if tags.size > 0
+          if !tags.empty?
             "#{filename}.tag_#{tags.join('.')}.part#{counter}.#{extension}"
           else
             "#{filename}.part#{counter}.#{extension}"
           end
         end
 
+        # create the file to be saved in blob storage
         def new_file
           uuid = SecureRandom.uuid
           name = generate_name
@@ -85,25 +93,28 @@ module LogStash
           TemporaryFile.new(key, io, path)
         end
 
-        # clas for the necoding
+        # clas for the encoding
         class IOWrappedGzip
           extend Forwardable
 
           def_delegators :@gzip_writer, :write, :close
           attr_reader :file_io, :gzip_writer
 
+          # initialize the class for encoding
           def initialize(file_io)
             @file_io = file_io
             @gzip_writer = Zlib::GzipWriter.open(file_io)
           end
 
+          # gets the path
           def path
             @gzip_writer.to_io.path
           end
 
+          # gets the file size
           def size
             # to get the current file size
-            if @gzip_writer.pos == 0
+            if @gzip_writer.pos.zero?
               # Ensure a zero file size is returned when nothing has
               # yet been written to the gzip file.
               0
@@ -113,6 +124,7 @@ module LogStash
             end
           end
 
+          # gets the fsync
           def fsync
             @gzip_writer.to_io.fsync
           end

data/lib/logstash/outputs/blob/time_rotation_policy.rb

@@ -1,4 +1,3 @@
-# encoding: utf-8
 module LogStash
   module Outputs
     class LogstashAzureBlobOutput
@@ -6,19 +5,21 @@ module LogStash
       # sets the policy for time rotation
       class TimeRotationPolicy
         attr_reader :time_file
-
+        # initialize the class and validate the time file
         def initialize(time_file)
           if time_file <= 0
-            raise LogStash::ConfigurationError, "`time_file` need to be greather than 0"
+            raise LogStash::ConfigurationError.new('`time_file` need to be greather than 0')
           end
 
           @time_file = time_file * 60
         end
 
+        # rotates based on time policy
         def rotate?(file)
-          file.size > 0 && (Time.now - file.ctime) >= time_file
+          !file.empty? && (Time.now - file.ctime) >= time_file
         end
 
+        # boolean method
         def needs_periodic?
           true
         end

data/lib/logstash/outputs/blob/uploader.rb

@@ -1,6 +1,5 @@
-# encoding: utf-8
-require "logstash/util"
-require "azure"
+require 'logstash/util'
+require 'azure'
 
 module LogStash
   module Outputs
@@ -9,22 +8,24 @@ module LogStash
       # this class uploads the files to Azure cloud
       class Uploader
         TIME_BEFORE_RETRYING_SECONDS = 1
-        DEFAULT_THREADPOOL = Concurrent::ThreadPoolExecutor.new({
-                                                                  :min_threads => 1,
-                                                                  :max_threads => 8,
-                                                                  :max_queue => 1,
-                                                                  :fallback_policy => :caller_runs
-                                                                })
-	
-	attr_accessor :upload_options, :logger, :container_name, :blob_account
-	
-        def initialize(blob_account, container_name , logger, threadpool = DEFAULT_THREADPOOL)
+        DEFAULT_THREADPOOL = Concurrent::ThreadPoolExecutor.new(min_threads: 1,
+                                                                max_threads: 8,
+                                                                max_queue: 1,
+                                                                fallback_policy: :caller_runs)
+
+        attr_accessor :upload_options, :logger, :container_name, :blob_account
+
+        # Initializes the class
+        # @param blob_account [Object] endpoint to azure gem
+        # @param container_name [String] name of the container in azure blob, at this point, if it doesn't exist, it was already created
+        def initialize(blob_account, container_name, logger, threadpool = DEFAULT_THREADPOOL)
           @blob_account = blob_account
           @workers_pool = threadpool
           @logger = logger
-          @container_name = container_name	
-	end
+          @container_name = container_name
+        end
 
+        # Create threads to upload the file to the container
         def upload_async(file, options = {})
           @workers_pool.post do
             LogStash::Util.set_thread_name("LogstashAzureBlobOutput output uploader, file: #{file.path}")
@@ -32,32 +33,38 @@ module LogStash
           end
         end
 
+        # Uploads the file to the container
         def upload(file, options = {})
           upload_options = options.fetch(:upload_options, {})
 
           begin
-            content = Object::File.open(file.path, "rb").read
-	    filename = Object::File.basename file.path
-	    puts filename
-	    blob = blob_account.create_block_blob(container_name, filename, content)
-	    puts blob.name
+            content = Object::File.open(file.path, 'rb').read
+            filename = Object::File.basename file.path
+            puts filename
+            blob = blob_account.create_block_blob(container_name, filename, content)
+            puts blob.name
           rescue => e
             # When we get here it usually mean that LogstashAzureBlobOutput tried to do some retry by himself (default is 3)
             # When the retry limit is reached or another error happen we will wait and retry.
             #
             # Thread might be stuck here, but I think its better than losing anything
             # its either a transient errors or something bad really happened.
-            logger.error("Uploading failed, retrying", :exception => e.class, :message => e.message, :path => file.path, :backtrace => e.backtrace)
+            logger.error('Uploading failed, retrying', exception: e.class, message: e.message, path: file.path, backtrace: e.backtrace)
             retry
           end
 
           options[:on_complete].call(file) unless options[:on_complete].nil?
           blob
         rescue => e
-          logger.error("An error occured in the `on_complete` uploader", :exception => e.class, :message => e.message, :path => file.path, :backtrace => e.backtrace)
+          logger.error('An error occured in the `on_complete` uploader',
+                       exception: e.class,
+                       message: e.message,
+                       path: file.path,
+                       backtrace: e.backtrace)
           raise e # reraise it since we don't deal with it now
         end
 
+        # stop threads
         def stop
           @workers_pool.shutdown
           @workers_pool.wait_for_termination(nil) # block until its done

data/lib/logstash/outputs/blob/writable_directory_validator.rb

@@ -1,4 +1,3 @@
-# encoding: utf-8
 module LogStash
   module Outputs
     class LogstashAzureBlobOutput
@@ -6,13 +5,13 @@ module LogStash
       # validates that the specified tmeporary directory can be accesed with
       # write permission
       class WritableDirectoryValidator
+        # Checks if a path is valid
+        # @param path [String] String that represents the path
         def self.valid?(path)
-          begin
-            FileUtils.mkdir_p(path) unless Dir.exist?(path)
-            ::File.writable?(path)
-          rescue
-            false
-          end
+          FileUtils.mkdir_p(path) unless Dir.exist?(path)
+          ::File.writable?(path)
+        rescue
+          false
         end
       end
     end

data/logstash-output-azure.gemspec

@@ -1,25 +1,25 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-output-azure'
-  s.version       = '0.3.0'
+  s.version       = '1.0.0'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'Plugin for logstash to send output to Microsoft Azure Blob'
-  #s.description   = 'TODO: Write a longer description or delete this line.'
-  #s.homepage      = 'TODO: Put your plugin''s website or public repo URL here.'
+  # s.description   = 'TODO: Write a longer description or delete this line.'
+  # s.homepage      = 'TODO: Put your plugin''s website or public repo URL here.'
   s.authors       = ['Tuffk']
   s.email         = 'tuffkmulhall@gmail.com'
   s.require_paths = ['lib']
 
   # Files
-  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
-   # Tests
+  s.files = Dir['lib/**/*', 'spec/**/*', 'vendor/**/*', '*.gemspec', '*.md', 'CONTRIBUTORS', 'Gemfile', 'LICENSE', 'NOTICE.TXT']
+  # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
 
   # Special flag to let us know this is actually a logstash plugin
-  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+  s.metadata = { 'logstash_plugin' => 'true', 'logstash_group' => 'output' }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
-  s.add_runtime_dependency "logstash-codec-plain"
-  s.add_runtime_dependency "azure", "~> 0.7"
-  s.add_development_dependency "logstash-devutils"
+  s.add_runtime_dependency 'azure', '~> 0.7'
+  s.add_runtime_dependency 'logstash-codec-plain'
+  s.add_runtime_dependency 'logstash-core-plugin-api', '~> 2.0'
+  s.add_development_dependency 'logstash-devutils'
 end