logstash-output-azure 0.1.0

data/lib/logstash/outputs/blob/path_validator.rb

@@ -0,0 +1,18 @@
+# encoding: utf-8
+module LogStash
+  module Outputs
+    class LogstashAzureBlobOutput
+      class PathValidator
+        INVALID_CHARACTERS = "\^`><"
+
+        def self.valid?(name)
+          name.match(matches_re).nil?
+        end
+
+        def self.matches_re
+          /[#{Regexp.escape(INVALID_CHARACTERS)}]/
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/blob/size_and_time_rotation_policy.rb

@@ -0,0 +1,24 @@
+# encoding: utf-8
+require "logstash/outputs/blob/size_rotation_policy"
+require "logstash/outputs/blob/time_rotation_policy"
+
+module LogStash
+  module Outputs
+    class LogstashAzureBlobOutput
+      class SizeAndTimeRotationPolicy
+        def initialize(file_size, time_file)
+          @size_strategy = SizeRotationPolicy.new(file_size)
+          @time_strategy = TimeRotationPolicy.new(time_file)
+        end
+
+        def rotate?(file)
+          @size_strategy.rotate?(file) || @time_strategy.rotate?(file)
+        end
+
+        def needs_periodic?
+          true
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/blob/size_rotation_policy.rb

@@ -0,0 +1,26 @@
+# encoding: utf-8
+module LogStash
+  module Outputs
+    class LogstashAzureBlobOutput
+      class SizeRotationPolicy
+        attr_reader :size_file
+
+        def initialize(size_file)
+          if size_file <= 0
+            raise LogStash::ConfigurationError, "`size_file` need to be greather than 0"
+          end
+
+          @size_file = size_file
+        end
+
+        def rotate?(file)
+          file.size >= size_file
+        end
+
+        def needs_periodic?
+          false
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/blob/temporary_file.rb

@@ -0,0 +1,71 @@
+# encoding: utf-8
+require "thread"
+require "forwardable"
+require "fileutils"
+
+module LogStash
+  module Outputs
+    class LogstashAzureBlobOutput
+      # Wrap the actual file descriptor into an utility classe
+      # It make it more OOP and easier to reason with the paths.
+      class TemporaryFile
+        extend Forwardable
+
+        def_delegators :@fd, :path, :write, :close, :fsync
+
+        attr_reader :fd
+
+        def initialize(key, fd, temp_path)
+          @fd = fd
+          @key = key
+          @temp_path = temp_path
+          @created_at = Time.now
+        end
+
+        def ctime
+          @created_at
+        end
+
+        def temp_path
+          @temp_path
+        end
+
+        def size
+          # Use the fd size to get the accurate result,
+          # so we dont have to deal with fsync
+          # if the file is close we will use the File::size
+          begin
+            @fd.size
+          rescue IOError
+            ::File.size(path)
+          end
+        end
+
+        def key
+          @key.gsub(/^\//, "")
+        end
+
+        # Each temporary file is made inside a directory named with an UUID,
+        # instead of deleting the file directly and having the risk of deleting other files
+        # we delete the root of the UUID, using a UUID also remove the risk of deleting unwanted file, it acts as
+        # a sandbox.
+        def delete!
+          @fd.close rescue IOError # force close anyway
+          FileUtils.rm_r(@temp_path, :secure => true)
+        end
+
+        def empty?
+          size == 0
+        end
+
+        def self.create_from_existing_file(file_path, temporary_folder)
+          key_parts = Pathname.new(file_path).relative_path_from(temporary_folder).to_s.split(::File::SEPARATOR)
+
+          TemporaryFile.new(key_parts.slice(1, key_parts.size).join("/"),
+                         ::File.open(file_path, "r"),
+                         ::File.join(temporary_folder, key_parts.slice(0, 1)))
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/blob/temporary_file_factory.rb

@@ -0,0 +1,129 @@
+# encoding: utf-8
+require "socket"
+require "securerandom"
+require "fileutils"
+require "zlib"
+require "forwardable"
+
+module LogStash
+  module Outputs
+    class LogstashAzureBlobOutput
+      # Since the file can contains dynamic part, we have to handle a more local structure to
+      # allow a nice recovery from a crash.
+      #
+      # The local structure will look like this.
+      #
+      # <TEMPORARY_PATH>/<UUID>/<prefix>/ls.s3.localhost.%Y-%m-%dT%H.%m.tag_es_fb.part1.txt.gz
+      #
+      # Since the UUID should be fairly unique I can destroy the whole path when an upload is complete.
+      # I do not have to mess around to check if the other directory have file in it before destroying them.
+      class TemporaryFileFactory
+        FILE_MODE = "a"
+        GZIP_ENCODING = "gzip"
+        GZIP_EXTENSION = "txt.gz"
+        TXT_EXTENSION = "txt"
+        STRFTIME = "%Y-%m-%dT%H.%M"
+
+        attr_accessor :counter, :tags, :prefix, :encoding, :temporary_directory, :current
+
+        def initialize(prefix, tags, encoding, temporary_directory)
+          @counter = 0
+          @prefix = prefix
+
+          @tags = tags
+          @encoding = encoding
+          @temporary_directory = temporary_directory
+          @lock = Mutex.new
+
+          rotate!
+        end
+
+        def rotate!
+          @lock.synchronize {
+            @current = new_file
+            increment_counter
+            @current
+          }
+        end
+
+        private
+        def extension
+          gzip? ? GZIP_EXTENSION : TXT_EXTENSION
+        end
+
+        def gzip?
+          encoding == GZIP_ENCODING
+        end
+
+        def increment_counter
+          @counter += 1
+        end
+
+        def current_time
+          Time.now.strftime(STRFTIME)
+        end
+
+        def generate_name
+          filename = "ls.s3.#{SecureRandom.uuid}.#{current_time}"
+
+          if tags.size > 0
+            "#{filename}.tag_#{tags.join('.')}.part#{counter}.#{extension}"
+          else
+            "#{filename}.part#{counter}.#{extension}"
+          end
+        end
+
+        def new_file
+          uuid = SecureRandom.uuid
+          name = generate_name
+          path = ::File.join(temporary_directory, uuid)
+          key = ::File.join(prefix, name)
+
+          FileUtils.mkdir_p(::File.join(path, prefix))
+
+          io = if gzip?
+                 # We have to use this wrapper because we cannot access the size of the
+                 # file directly on the gzip writer.
+                 IOWrappedGzip.new(::File.open(::File.join(path, key), FILE_MODE))
+               else
+                 ::File.open(::File.join(path, key), FILE_MODE)
+               end
+
+          TemporaryFile.new(key, io, path)
+        end
+
+        class IOWrappedGzip
+          extend Forwardable
+
+          def_delegators :@gzip_writer, :write, :close
+          attr_reader :file_io, :gzip_writer
+
+          def initialize(file_io)
+            @file_io = file_io
+            @gzip_writer = Zlib::GzipWriter.open(file_io)
+          end
+
+          def path
+            @gzip_writer.to_io.path
+          end
+
+          def size
+            # to get the current file size
+            if @gzip_writer.pos == 0
+              # Ensure a zero file size is returned when nothing has
+              # yet been written to the gzip file.
+              0
+            else
+              @gzip_writer.flush
+              @gzip_writer.to_io.size
+            end
+          end
+
+          def fsync
+            @gzip_writer.to_io.fsync
+          end
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/blob/time_rotation_policy.rb

@@ -0,0 +1,26 @@
+# encoding: utf-8
+module LogStash
+  module Outputs
+    class LogstashAzureBlobOutput
+      class TimeRotationPolicy
+        attr_reader :time_file
+
+        def initialize(time_file)
+          if time_file <= 0
+            raise LogStash::ConfigurationError, "`time_file` need to be greather than 0"
+          end
+
+          @time_file = time_file * 60
+        end
+
+        def rotate?(file)
+          file.size > 0 && (Time.now - file.ctime) >= time_file
+        end
+
+        def needs_periodic?
+          true
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/blob/uploader.rb

@@ -0,0 +1,67 @@
+# encoding: utf-8
+require "logstash/util"
+require "azure"
+
+module LogStash
+  module Outputs
+    class LogstashAzureBlobOutput
+      class Uploader
+        TIME_BEFORE_RETRYING_SECONDS = 1
+        DEFAULT_THREADPOOL = Concurrent::ThreadPoolExecutor.new({
+                                                                  :min_threads => 1,
+                                                                  :max_threads => 8,
+                                                                  :max_queue => 1,
+                                                                  :fallback_policy => :caller_runs
+                                                                })
+
+
+        attr_reader :storage_account_name, :upload_options, :logger
+
+        def initialize(blob_account, logger, threadpool = DEFAULT_THREADPOOL)
+          @blob_account = blob_account
+          @workers_pool = threadpool
+          @logger = logger
+        end
+
+        def upload_async(file, options = {})
+          @workers_pool.post do
+            LogStash::Util.set_thread_name("LogstashAzureBlobOutput output uploader, file: #{file.path}")
+            upload(file, options)
+          end
+        end
+
+        def upload(file, options = {})
+          upload_options = options.fetch(:upload_options, {})
+
+          begin
+            Azure.config.storage_account_name = ENV['AZURE_STORAGE_ACCOUNT']
+            Azure.config.storage_access_key = ENV['AZURE_STORAGE_ACCESS_KEY']
+            azure_blob_service = Azure::Blob::BlobService.new
+            containers = azure_blob_service.list_containers
+            content = Object::File.open(file.path, "rb").read
+            blob = azure_blob_service.create_block_blob(containers[0].name, "#{file.ctime.iso8601}", content)
+          rescue => e
+            # When we get here it usually mean that LogstashAzureBlobOutput tried to do some retry by himself (default is 3)
+            # When the retry limit is reached or another error happen we will wait and retry.
+            #
+            # Thread might be stuck here, but I think its better than losing anything
+            # its either a transient errors or something bad really happened.
+            logger.error("Uploading failed, retrying", :exception => e.class, :message => e.message, :path => file.path, :backtrace => e.backtrace)
+            retry
+          end
+
+          options[:on_complete].call(file) unless options[:on_complete].nil?
+          blob
+        rescue => e
+          logger.error("An error occured in the `on_complete` uploader", :exception => e.class, :message => e.message, :path => file.path, :backtrace => e.backtrace)
+          raise e # reraise it since we don't deal with it now
+        end
+
+        def stop
+          @workers_pool.shutdown
+          @workers_pool.wait_for_termination(nil) # block until its done
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/blob/writable_directory_validator.rb

@@ -0,0 +1,17 @@
+# encoding: utf-8
+module LogStash
+  module Outputs
+    class LogstashAzureBlobOutput
+      class WritableDirectoryValidator
+        def self.valid?(path)
+          begin
+            FileUtils.mkdir_p(path) unless Dir.exist?(path)
+            ::File.writable?(path)
+          rescue
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/logstash-output-Logstash_Azure_Blob_Output.gemspec

@@ -0,0 +1,25 @@
+Gem::Specification.new do |s|
+  s.name          = 'logstash-output-azure'
+  s.version       = '0.1.0'
+  s.licenses      = ['Apache-2.0']
+  s.summary       = 'Plugin for logstash to send output to Microsoft Azure Blob'
+  #s.description   = 'TODO: Write a longer description or delete this line.'
+  #s.homepage      = 'TODO: Put your plugin''s website or public repo URL here.'
+  s.authors       = ['Tuffk']
+  s.email         = 'tuffkmulhall@gmail.com'
+  s.require_paths = ['lib']
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
+  s.add_runtime_dependency "logstash-codec-plain"
+  s.add_runtime_dependency "azure", "~> 0.7"
+  s.add_development_dependency "logstash-devutils"
+end

data/spec/outputs/Logstash_Azure_Blob_Output_spec.rb

@@ -0,0 +1,42 @@
+# encoding: utf-8
+
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/outputs/Logstash_Azure_Blob_Output'
+require 'logstash/codecs/plain'
+require 'logstash/event'
+require 'tmpdir'
+require 'pry'
+
+describe LogStash::Outputs::LogstashAzureBlobOutput do
+  let(:config_options) do
+    {
+      storage_account_name: ENV['AZURE_STORAGE_ACCOUNT'],
+      storage_access_key: ENV['AZURE_STORAGE_ACCESS_KEY'],
+      size_file: 5242880,
+      time_file: 15,
+      restore: true,
+      temporary_directory: File.join(Dir.tmpdir, 'logstash'),
+      prefix: '',
+      upload_queue_size: 2 * (Concurrent.processor_count * 0.25).ceil,
+      upload_workers_count: (Concurrent.processor_count * 0.5).ceil,
+      rotation_strategy: 'size_and_time',
+      tags: [],
+      encoding: "none"
+    }
+  end
+  let(:sample_event) { LogStash::Event.new(source: 'alguna', tags: %w[tag1 tag2], fields: { field1: 1, field2: true }) }
+
+  let(:output) { described_class.new() }
+
+  before do
+    output.register
+  end
+
+  describe 'receive message' do
+    subject { output.receive(sample_event) }
+    xit 'should return the blob sent to Azure' do
+      md5 = Digest::MD5.base64digest(sample_event.to_json)
+      expect(subject.properties[:content_md5]).to eq(md5)
+    end
+  end
+end