logstash-output-azureblob 0.9.0

data/lib/logstash/outputs/blob/temporary_file_factory.rb

@@ -0,0 +1,135 @@
+require 'socket'
+require 'securerandom'
+require 'fileutils'
+require 'zlib'
+require 'forwardable'
+
+module LogStash
+  module Outputs
+    class LogstashAzureBlobOutput
+      # a sub class of +LogstashAzureBlobOutput+
+      # creates the temporary files to write and later upload
+      class TemporaryFileFactory
+        FILE_MODE = 'a'.freeze
+        GZIP_ENCODING = 'gzip'.freeze
+        GZIP_EXTENSION = 'txt.gz'.freeze
+        TXT_EXTENSION = 'txt'.freeze
+        STRFTIME = '%Y-%m-%dT%H.%M'.freeze
+
+        attr_accessor :counter, :tags, :prefix, :encoding, :temporary_directory, :current
+
+        # initialize the class
+        def initialize(prefix, tags, encoding, temporary_directory)
+          @counter = 0
+          @prefix = prefix
+
+          @tags = tags
+          @encoding = encoding
+          @temporary_directory = temporary_directory
+          @lock = Mutex.new
+
+          rotate!
+        end
+
+        # do the rotation
+        def rotate!
+          @lock.synchronize do
+            @current = new_file
+            increment_counter
+            @current
+          end
+        end
+
+        private
+
+        # if it is not gzip ecoding, then it is txt extension
+        def extension
+          gzip? ? GZIP_EXTENSION : TXT_EXTENSION
+        end
+
+        # boolean method to check if its gzip encoding
+        def gzip?
+          encoding == GZIP_ENCODING
+        end
+
+        # increment the counter in 1 unit
+        def increment_counter
+          @counter += 1
+        end
+
+        # gets the current time
+        def current_time
+          Time.now.strftime(STRFTIME)
+        end
+
+        # method that generate the name of the file to be saved in blob storage
+        def generate_name
+          filename = "#{current_time}.#{SecureRandom.uuid}"
+
+          if !tags.empty?
+            "#{filename}.tag_#{tags.join('.')}.part#{counter}.#{extension}"
+          else
+            "#{filename}.part#{counter}.#{extension}"
+          end
+        end
+
+        # create the file to be saved in blob storage
+        def new_file
+          uuid = SecureRandom.uuid
+          name = generate_name
+          path = ::File.join(temporary_directory, uuid)
+          key = ::File.join(prefix, name)
+
+          FileUtils.mkdir_p(::File.join(path, prefix))
+
+          io = if gzip?
+                 # We have to use this wrapper because we cannot access the size of the
+                 # file directly on the gzip writer.
+                 IOWrappedGzip.new(::File.open(::File.join(path, key), FILE_MODE))
+               else
+                 ::File.open(::File.join(path, key), FILE_MODE)
+               end
+
+          TemporaryFile.new(key, io, path)
+        end
+
+        # clas for the encoding
+        class IOWrappedGzip
+          extend Forwardable
+
+          def_delegators :@gzip_writer, :write, :close
+          attr_reader :file_io, :gzip_writer
+
+          # initialize the class for encoding
+          def initialize(file_io)
+            @file_io = file_io
+            @gzip_writer = Zlib::GzipWriter.open(file_io)
+          end
+
+          # gets the path
+          def path
+            @gzip_writer.to_io.path
+          end
+
+          # gets the file size
+          def size
+            # to get the current file size
+            if @gzip_writer.pos.zero?
+              # Ensure a zero file size is returned when nothing has
+              # yet been written to the gzip file.
+              0
+            else
+              @gzip_writer.flush
+              @gzip_writer.to_io.size
+            end
+          end
+
+          # gets the fsync
+          def fsync
+            @gzip_writer.to_io.fsync
+          end
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/blob/time_rotation_policy.rb

@@ -0,0 +1,29 @@
+module LogStash
+  module Outputs
+    class LogstashAzureBlobOutput
+      # a sub class of +LogstashAzureBlobOutput+
+      # sets the policy for time rotation
+      class TimeRotationPolicy
+        attr_reader :time_file
+        # initialize the class and validate the time file
+        def initialize(time_file)
+          if time_file <= 0
+            raise LogStash::ConfigurationError.new('`time_file` need to be greather than 0')
+          end
+
+          @time_file = time_file * 60
+        end
+
+        # rotates based on time policy
+        def rotate?(file)
+          !file.empty? && (Time.now - file.ctime) >= time_file
+        end
+
+        # boolean method
+        def needs_periodic?
+          true
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/blob/uploader.rb

@@ -0,0 +1,75 @@
+require 'logstash/util'
+require 'azure/storage/blob'
+require 'azure/storage/common'
+
+module LogStash
+  module Outputs
+    class LogstashAzureBlobOutput
+      # a sub class of +LogstashAzureBlobOutput+
+      # this class uploads the files to Azure cloud
+      class Uploader
+        TIME_BEFORE_RETRYING_SECONDS = 1
+        DEFAULT_THREADPOOL = Concurrent::ThreadPoolExecutor.new(min_threads: 1,
+                                                                max_threads: 8,
+                                                                max_queue: 1,
+                                                                fallback_policy: :caller_runs)
+
+        attr_accessor :logger, :container_name, :blob_account
+
+        # Initializes the class
+        # @param blob_account [Object] endpoint to azure gem
+        # @param container_name [String] name of the container in azure blob, at this point, if it doesn't exist, it was already created
+        def initialize(blob_account, container_name, logger, threadpool = DEFAULT_THREADPOOL)
+          @blob_account = blob_account
+          @workers_pool = threadpool
+          @logger = logger
+          @container_name = container_name
+        end
+
+        # Create threads to upload the file to the container
+        def upload_async(file, options = {})
+          @workers_pool.post do
+            LogStash::Util.set_thread_name("LogstashAzureBlobOutput output uploader, file: #{file.path}")
+            upload(file, options)
+          end
+        end
+
+        # Uploads the file to the container
+        def upload(file, options = {})
+
+          begin
+            content = Object::File.open(file.path, 'rb').read
+            filename = Object::File.basename file.path
+            puts filename
+            blob = blob_account.create_block_blob(container_name, filename, content)
+            puts blob.name
+          rescue => e
+            # When we get here it usually mean that LogstashAzureBlobOutput tried to do some retry by himself (default is 3)
+            # When the retry limit is reached or another error happen we will wait and retry.
+            #
+            # Thread might be stuck here, but I think its better than losing anything
+            # its either a transient errors or something bad really happened.
+            logger.error('Uploading failed, retrying', exception: e.class, message: e.message, filename: filename, path: file.path, container: container_name, blobAccount: blob_account, backtrace: e.backtrace)
+            retry
+          end
+
+          options[:on_complete].call(file) unless options[:on_complete].nil?
+          blob
+        rescue => e
+          logger.error('An error occured in the `on_complete` uploader',
+                       exception: e.class,
+                       message: e.message,
+                       path: file.path,
+                       backtrace: e.backtrace)
+          raise e # reraise it since we don't deal with it now
+        end
+
+        # stop threads
+        def stop
+          @workers_pool.shutdown
+          @workers_pool.wait_for_termination(nil) # block until its done
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/blob/writable_directory_validator.rb

@@ -0,0 +1,19 @@
+module LogStash
+  module Outputs
+    class LogstashAzureBlobOutput
+      # a sub class of +LogstashAzureBlobOutput+
+      # validates that the specified tmeporary directory can be accesed with
+      # write permission
+      class WritableDirectoryValidator
+        # Checks if a path is valid
+        # @param path [String] String that represents the path
+        def self.valid?(path)
+          FileUtils.mkdir_p(path) unless Dir.exist?(path)
+          ::File.writable?(path)
+        rescue
+          false
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/template.rb

@@ -0,0 +1,16 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+
+# An azureblob output that does nothing.
+class LogStash::Outputs::Azureblob < LogStash::Outputs::Base
+  config_name "azureblob"
+
+  public
+  def register
+  end # def register
+
+  public
+  def receive(event)
+    return "Event received"
+  end # def event
+end # class LogStash::Outputs::Azureblob

data/logstash-output-azureblob.gemspec

@@ -0,0 +1,25 @@
+Gem::Specification.new do |s|
+  s.name          = 'logstash-output-azureblob'
+  s.version       = '0.9.0'
+  s.licenses      = ['Apache License (2.0)']
+  s.summary       = 'Output plugin for logstash to send output to Azure Blob Storage'
+  s.description   = 'This output plugin will send logs to azure blob storage. This does not support ADLS2 endpoints'
+  s.homepage      = 'https://github.com/seanstark/logstash-output-azureblob'
+  s.authors       = ['Sean Stark']
+  s.email         = 'sean.stark@microsoft.com'
+  s.require_paths = ['lib']
+
+  # Files
+  s.files = Dir['lib/**/*', 'spec/**/*', 'vendor/**/*', '*.gemspec', '*.md', 'CONTRIBUTORS', 'Gemfile', 'LICENSE', 'NOTICE.TXT']
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { 'logstash_plugin' => 'true', 'logstash_group' => 'output' }
+
+  # Gem dependencies
+  s.add_runtime_dependency 'azure-storage-blob', '~> 2.0'
+  s.add_runtime_dependency 'logstash-codec-plain'
+  s.add_runtime_dependency 'logstash-core-plugin-api', '~> 2.1'
+  s.add_development_dependency 'logstash-devutils'
+end

data/spec/outputs/azureblob_spec.rb

@@ -0,0 +1,49 @@
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/outputs/azure_blob'
+require 'logstash/codecs/plain'
+require 'logstash/event'
+require 'tmpdir'
+require 'pry'
+
+describe LogStash::Outputs::LogstashAzureBlobOutput do
+  let(:config_options) do
+    {
+      storage_account_name: ENV['AZURE_STORAGE_ACCOUNT'],
+      storage_access_key: ENV['AZURE_STORAGE_ACCESS_KEY'],
+      container_name: 'test',
+      size_file: 5242880,
+      time_file: 15,
+      restore: true,
+      temporary_directory: File.join(Dir.tmpdir, 'logstash'),
+      prefix: '',
+      upload_queue_size: 2 * (Concurrent.processor_count * 0.25).ceil,
+      upload_workers_count: (Concurrent.processor_count * 0.5).ceil,
+      rotation_strategy: 'size_and_time',
+      tags: [],
+      encoding: 'none'
+    }
+  end
+  let(:sample_event) { LogStash::Event.new(source: 'alguna', tags: %w[tag1 tag2], fields: { field1: 1, field2: true }) }
+
+  # let(:output) { described_class.new() }
+
+  #  before do
+  #    output.register
+  #  end
+
+  # it 'should create' do
+  #   blober = described_class.new
+  #   blober.register
+  #   expect(blober.storage_account_name).not_to be_nil
+  #   expect(blober.storage_access_key).not_to be_nil
+  #   expect(blober.container_name).not_to be_nil
+  # end
+
+  describe 'receive message' do
+    subject { output.receive(sample_event) }
+    # xit 'should return the blob sent to Azure' do
+    #  md5 = Digest::MD5.base64digest(sample_event.to_json)
+    #  expect(subject.properties[:content_md5]).to eq(md5)
+    # end
+  end
+end

data/spec/outputs/blob/file_repository_spec.rb

@@ -0,0 +1,140 @@
+require 'logstash/outputs/azure_blob'
+require 'stud/temporary'
+require 'fileutils'
+require_relative '../../spec_helper'
+
+describe LogStash::Outputs::LogstashAzureBlobOutput::FileRepository do
+  let(:tags) { [] }
+  let(:encoding) { 'none' }
+  let(:temporary_directory) { Stud::Temporary.pathname }
+  let(:prefix_key) { 'a-key' }
+
+  before do
+    FileUtils.mkdir_p(temporary_directory)
+  end
+
+  subject { described_class.new(tags, encoding, temporary_directory) }
+
+  it 'returns a temporary file' do
+    subject.get_file(prefix_key) do |file|
+      expect(file).to be_kind_of(LogStash::Outputs::LogstashAzureBlobOutput::TemporaryFile)
+    end
+  end
+
+  it 'returns the same file for the same prefix key' do
+    file_path = nil
+
+    subject.get_file(prefix_key) do |file|
+      file_path = file.path
+    end
+
+    subject.get_file(prefix_key) do |file|
+      expect(file.path).to eq(file_path)
+    end
+  end
+
+  it 'returns the same file for the same dynamic prefix key' do
+    prefix = '%{type}/%{+YYYY}/%{+MM}/%{+dd}/'
+    event = LogStash::Event.new('type' => 'syslog')
+    key = event.sprintf(prefix)
+    file_path = nil
+
+    subject.get_file(key) do |file|
+      file_path = file.path
+    end
+
+    subject.get_file(key) do |file|
+      expect(file.path).to eq(file_path)
+    end
+  end
+
+  it 'returns different file for different prefix keys' do
+    file_path = nil
+
+    subject.get_file(prefix_key) do |file|
+      file_path = file.path
+    end
+
+    subject.get_file('another_prefix_key') do |file|
+      expect(file.path).not_to eq(file_path)
+    end
+  end
+
+  it 'allows to get the file factory for a specific prefix' do
+    subject.get_factory(prefix_key) do |factory|
+      expect(factory).to be_kind_of(LogStash::Outputs::LogstashAzureBlobOutput::TemporaryFileFactory)
+    end
+  end
+
+  it 'returns a different file factory for a different prefix keys' do
+    factory = nil
+
+    subject.get_factory(prefix_key) do |f|
+      factory = f
+    end
+
+    subject.get_factory('another_prefix_key') do |f|
+      expect(factory).not_to eq(f)
+    end
+  end
+
+  it 'returns the number of prefix keys' do
+    expect(subject.size).to eq(0)
+    subject.get_file(prefix_key) { |file| file.write('something') }
+    expect(subject.size).to eq(1)
+  end
+
+  it 'returns all available keys' do
+    subject.get_file(prefix_key) { |file| file.write('something') }
+    expect(subject.keys.toArray).to include(prefix_key)
+    expect(subject.keys.toArray.size).to eq(1)
+  end
+
+  it 'clean stale factories' do
+    @file_repository = described_class.new(tags, encoding, temporary_directory, 1, 1)
+    expect(@file_repository.size).to eq(0)
+    path = ''
+    @file_repository.get_factory(prefix_key) do |factory|
+      factory.current.write('hello')
+      # force a rotation so we get an empty file that will get stale.
+      factory.rotate!
+      path = factory.current.temp_path
+    end
+
+    @file_repository.get_file('another-prefix') { |file| file.write('hello') }
+    expect(@file_repository.size).to eq(2)
+    try(10) { expect(@file_repository.size).to eq(1) }
+    expect(File.directory?(path)).to be_falsey
+  end
+end
+
+describe LogStash::Outputs::LogstashAzureBlobOutput::FileRepository::PrefixedValue do
+  let(:factory) { spy('factory', current: file) }
+  subject { described_class.new(factory, 1) }
+
+  context '#stale?' do
+    context 'the file is empty and older than stale time' do
+      let(:file) { double('file', size: 0, ctime: Time.now - 5) }
+
+      it 'returns true' do
+        expect(subject.stale?).to be_truthy
+      end
+    end
+
+    context 'when the file has data in it' do
+      let(:file) { double('file', size: 200, ctime: Time.now - 5) }
+
+      it 'returns false' do
+        expect(subject.stale?).to be_falsey
+      end
+    end
+
+    context 'when the file is not old enough' do
+      let(:file) { double('file', size: 0, ctime: Time.now + 100) }
+
+      it 'returns false' do
+        expect(subject.stale?).to be_falsey
+      end
+    end
+  end
+end

data/spec/outputs/blob/size_and_time_rotation_policy_spec.rb

@@ -0,0 +1,76 @@
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/outputs/blob/size_and_time_rotation_policy'
+require 'logstash/outputs/blob/temporary_file'
+
+describe LogStash::Outputs::LogstashAzureBlobOutput::SizeAndTimeRotationPolicy do
+  let(:file_size) { 10 }
+  let(:time_file) { 1 }
+  subject { described_class.new(file_size, time_file) }
+
+  let(:temporary_directory) { Stud::Temporary.pathname }
+  let(:temporary_file) { Stud::Temporary.file }
+  let(:name) { 'foobar' }
+  let(:content) { 'hello' * 1000 }
+  let(:file) { LogStash::Outputs::LogstashAzureBlobOutput::TemporaryFile.new(name, temporary_file, temporary_directory) }
+
+  it 'raises an exception if the `time_file` is set to 0' do
+    expect { described_class.new(100, 0) }.to raise_error(LogStash::ConfigurationError, /time_file/)
+  end
+
+  it 'raises an exception if the `time_file` is < 0' do
+    expect { described_class.new(100, -100) }.to raise_error(LogStash::ConfigurationError, /time_file/)
+  end
+
+  it 'raises an exception if the `size_file` is 0' do
+    expect { described_class.new(0, 100) }.to raise_error(LogStash::ConfigurationError, /size_file/)
+  end
+
+  it 'raises an exception if the `size_file` is < 0' do
+    expect { described_class.new(-100, 100) }.to raise_error(LogStash::ConfigurationError, /size_file/)
+  end
+
+  it 'returns true if the size on disk is higher than the `file_size`' do
+    file.write(content)
+    file.fsync
+    expect(subject.rotate?(file)).to be_truthy
+  end
+
+  it 'returns false if the size is inferior than the `file_size`' do
+    expect(subject.rotate?(file)).to be_falsey
+  end
+
+  context 'when the size of the file is superior to 0' do
+    let(:file_size) { 10000 }
+
+    before :each do
+      file.write(content)
+      file.fsync
+    end
+
+    it 'returns true if the file old enough' do
+      allow(file).to receive(:ctime).and_return(Time.now - (time_file * 2 * 60))
+      expect(subject.rotate?(file)).to be_truthy
+    end
+
+    it 'returns false is not old enough' do
+      allow(file).to receive(:ctime).and_return(Time.now + time_file * 10)
+      expect(subject.rotate?(file)).to be_falsey
+    end
+  end
+
+  context 'When the size of the file is 0' do
+    it 'returns false if the file old enough' do
+      expect(subject.rotate?(file)).to be_falsey
+    end
+
+    it 'returns false is not old enough' do
+      expect(subject.rotate?(file)).to be_falsey
+    end
+  end
+
+  context '#needs_periodic?' do
+    it 'return true' do
+      expect(subject.needs_periodic?).to be_truthy
+    end
+  end
+end

data/spec/outputs/blob/size_rotation_policy_spec.rb

@@ -0,0 +1,39 @@
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/outputs/blob/size_rotation_policy'
+require 'logstash/outputs/blob/temporary_file'
+require 'fileutils'
+
+describe LogStash::Outputs::LogstashAzureBlobOutput::SizeRotationPolicy do
+  subject { described_class.new(size_file) }
+
+  let(:temporary_directory) { Stud::Temporary.directory }
+  let(:temporary_file) { Stud::Temporary.file }
+  let(:name) { 'foobar' }
+  let(:content) { 'hello' * 1000 }
+  let(:size_file) { 10 } # in bytes
+  let(:file) { LogStash::Outputs::LogstashAzureBlobOutput::TemporaryFile.new(name, temporary_file, temporary_directory) }
+
+  it 'returns true if the size on disk is higher than the `size_file`' do
+    file.write(content)
+    file.fsync
+    expect(subject.rotate?(file)).to be_truthy
+  end
+
+  it 'returns false if the size is inferior than the `size_file`' do
+    expect(subject.rotate?(file)).to be_falsey
+  end
+
+  it 'raises an exception if the `size_file` is 0' do
+    expect { described_class.new(0) }.to raise_error(LogStash::ConfigurationError, /need to be greather than 0/)
+  end
+
+  it 'raises an exception if the `size_file` is < 0' do
+    expect { described_class.new(-100) }.to raise_error(LogStash::ConfigurationError, /need to be greather than 0/)
+  end
+
+  context '#needs_periodic?' do
+    it 'return false' do
+      expect(subject.needs_periodic?).to be_falsey
+    end
+  end
+end

data/spec/outputs/blob/temporary_file_factory_spec.rb

@@ -0,0 +1,88 @@
+require 'logstash/outputs/blob/temporary_file_factory'
+require 'logstash/outputs/blob/temporary_file'
+require 'stud/temporary'
+require 'fileutils'
+
+describe LogStash::Outputs::LogstashAzureBlobOutput::TemporaryFileFactory do
+  let(:prefix) { 'foobar' }
+  let(:tags) { [] }
+  let(:temporary_directory) { Stud::Temporary.pathname }
+
+  before do
+    FileUtils.mkdir_p(temporary_directory)
+  end
+
+  subject { described_class.new(prefix, tags, encoding, temporary_directory) }
+
+  shared_examples 'file factory' do
+    it 'creates the file on disk' do
+      expect(File.exist?(subject.current.path)).to be_truthy
+    end
+
+    it 'returns a size equal to zero after file creation' do
+      expect(subject.current.size).to eq(0)
+    end
+
+    it 'create a temporary file when initialized' do
+      expect(subject.current).to be_kind_of(LogStash::Outputs::LogstashAzureBlobOutput::TemporaryFile)
+    end
+
+    it 'create a file in the right format' do
+      expect(subject.current.path).to match(extension)
+    end
+
+    it 'allow to rotate the file' do
+      file_path = subject.current.path
+      expect(subject.rotate!.path).not_to eq(file_path)
+    end
+
+    it 'increments the part name on rotation' do
+      expect(subject.current.path).to match(/part0/)
+      expect(subject.rotate!.path).to match(/part1/)
+    end
+
+    it 'includes the date' do
+      n = Time.now
+      expect(subject.current.path).to include(n.strftime('%Y-%m-%dT'))
+    end
+
+    it 'include the file key in the path' do
+      file = subject.current
+      expect(file.path).to match(/#{file.key}/)
+    end
+
+    it 'create a unique directory in the temporary directory for each file' do
+      uuid = 'hola'
+      expect(SecureRandom).to receive(:uuid).and_return(uuid).twice
+      expect(subject.current.path).to include(uuid)
+    end
+
+    context 'with tags supplied' do
+      let(:tags) { %w[secret service] }
+
+      it 'adds tags to the filename' do
+        expect(subject.current.path).to match(/tag_#{tags.join('.')}.part/)
+      end
+    end
+
+    context 'without tags' do
+      it "doesn't add tags to the filename" do
+        expect(subject.current.path).not_to match(/tag_/)
+      end
+    end
+  end
+
+  context 'when gzip' do
+    let(:encoding) { 'gzip' }
+    let(:extension) { /\.txt.gz$/ }
+
+    include_examples 'file factory'
+  end
+
+  context 'when encoding set to `none`' do
+    let(:encoding) { 'none' }
+    let(:extension) { /\.txt$/ }
+
+    include_examples 'file factory'
+  end
+end