fluent-plugin-azurestorage-gen2 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 857dcdb31e03f6564e547e0183a3bf24f06c8a0e
+  data.tar.gz: f1f1d048e69afba0f469a1999b6786473e282092
+SHA512:
+  metadata.gz: 6c87884fa508bc01f7470653b0a3b67d4a4cb02bddab03eb9071c1360f25abcf13558d1023a11be7d1277dcebf0794d4fb01ff19982a2000c2879ff335421773
+  data.tar.gz: 1491be525cbf66dbd7240ddea5a2ba33ded91e4c4c5816d238fc4ac49a76057e05b7bc7dea881220dc32fff836239d2d485175699285041bb1330c955b103e0b

data/.gitignore

@@ -0,0 +1,58 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+GEMSPEC_ARGS
+Gemfile.lock
+
+# Used by dotenv library to load environment variables.
+# .env
+
+# Ignore Byebug command history file.
+.byebug_history
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+*.bridgesupport
+build-iPhoneOS/
+build-iPhoneSimulator/
+
+## Specific to RubyMotion (use of CocoaPods):
+#
+# We recommend against adding the Pods directory to your .gitignore. However
+# you should judge for yourself, the pros and cons are mentioned at:
+# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
+#
+# vendor/Pods/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
+
+# Used by RuboCop. Remote config files pulled in from inherit_from directive.
+# .rubocop-https?--*

data/.travis.yml

@@ -0,0 +1,14 @@
+language: ruby
+
+rvm:
+  - 2.4
+
+gemfile:
+ - Gemfile
+
+branches:
+  only:
+    - master
+
+# bundle exec rake test
+script: make gem

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Olivér Sz,
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/Makefile

@@ -0,0 +1,15 @@
+gem:
+	rm -f fluent-plugin-azurestorage-gen2*.gem
+	gem build fluent-plugin-azurestorage-gen2.gemspec
+
+install: gem
+	gem install fluent-plugin-azurestorage-gen2*.gem
+
+push: gem
+	gem push fluent-plugin-azurestorage-gen2*.gem
+
+tag-and-branch:
+	git tag "v$$(cat VERSION)" $(RELEASE_COMMIT)
+	git checkout -b "release/$$(cat VERSION)" $(RELEASE_COMMIT)
+	git push origin "v$$(cat VERSION)"
+	git push -u origin "release/$$(cat VERSION)"

data/README.md

@@ -0,0 +1,37 @@
+# Azure Datalake Storage Gen2 Fluentd Output Plugin (IN PROGRESS)
+
+[![Build Status](https://travis-ci.org/oleewere/fluent-plugin-azurestorage-gen2.svg?branch=master)](https://travis-ci.org/oleewere/fluent-plugin-azurestorage-gen2)
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
+
+## Requirements
+
+| fluent-plugin-azurestorage-gen2 | fluentd | ruby |
+|------------------------|---------|------|
+| >= 0.1.0 | >= v0.14.0 | >= 2.4 |
+
+## Overview
+
+Fluent output plugin that can use ABFS api and append blobs with MSI support
+
+## Installation
+
+Install from RubyGems:
+```
+$ gem install fluent-plugin-azurestorage-gen2
+```
+
+## Configuration
+
+TODO
+
+### Configuration options
+
+TODO
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,14 @@
+
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/plugin/test_*.rb']
+  test.verbose = true
+end
+
+task :default => [:build]
+

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/fluent-plugin-azurestorage-gen2.gemspec

@@ -0,0 +1,29 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name = "fluent-plugin-azurestorage-gen2"
+  gem.description = "Azure Storage output plugin for Fluentd event collector"
+  gem.license = "Apache-2.0"
+  gem.homepage = "https://github.com/oleewere/fluent-plugin-azurestorage-gen2"
+  gem.summary = gem.description
+  gem.version = File.read("VERSION").strip
+  gem.authors = ["Oliver Szabo"]
+  gem.email = ["oleewere@gmail.com"]
+  #gem.platform    = Gem::Platform::RUBY
+  gem.files = `git ls-files`.split("\n")
+  gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map {|f| File.basename(f)}
+  gem.require_paths = ['lib']
+
+  gem.add_runtime_dependency 'fluentd', ['>= 1.0', '< 2']
+  gem.add_runtime_dependency 'azure-storage-common', '~> 1.1', '>= 1.1.0'
+  gem.add_runtime_dependency 'azure-storage-blob', '~> 1.1', '>= 1.1.0'
+  gem.add_runtime_dependency 'uuidtools', '~> 2.1', '>= 2.1.5'
+  gem.add_runtime_dependency 'typhoeus', '~> 1.0', '>= 1.0.1'
+  gem.add_runtime_dependency 'json', '~> 2.1', '>= 2.1.0'
+  gem.add_runtime_dependency "yajl-ruby", '~> 1.4'
+  gem.add_development_dependency 'rake', '~> 12.3', '>= 12.3.1'
+  gem.add_development_dependency 'test-unit', '~> 3.3', '>= 3.3.3'
+  gem.add_development_dependency 'test-unit-rr', '~> 1.0', '>= 1.0.5'
+end

data/lib/fluent/plugin/out_azurestorage_gen2.rb

@@ -0,0 +1,366 @@
+require 'net/http'
+require 'json'
+require 'tempfile'
+require 'time'
+require 'typhoeus'
+require 'fluent/plugin/output'
+require 'zlib'
+
+module Fluent::Plugin
+    class AzureStorageGen2Output < Fluent::Plugin::Output
+      Fluent::Plugin.register_output('azurestorage_gen2', self)
+
+        helpers :compat_parameters, :formatter, :inject
+
+        def initialize
+            super
+            @compressor = nil
+        end
+      
+        config_param :path, :string, :default => ""
+        config_param :azure_storage_account, :string, :default => nil
+        config_param :azure_storage_access_key, :string, :default => nil, :secret => true
+        config_param :azure_instance_msi, :string, :default => nil
+        config_param :azure_oauth_refresh_interval, :integer, :default => 60 * 60 # one hour
+        config_param :azure_container, :string, :default => nil
+        config_param :azure_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
+        config_param :file_extension, :string, :default => "log"
+        config_param :store_as, :string, :default => "gzip"
+        config_param :auto_create_container, :bool, :default => false
+        config_param :format, :string, :default => "out_file"
+        config_param :time_slice_format, :string, :default => '%Y%m%d'
+
+        DEFAULT_FORMAT_TYPE = "out_file"
+        URL_DOMAIN_SUFFIX = '.dfs.core.windows.net'
+        ACCESS_TOKEN_API_VERSION = "2018-02-01"
+        ABFS_API_VERSION = "2018-11-09"
+        AZURE_BLOCK_SIZE_LIMIT = 4 * 1024 * 1024 - 1
+
+        config_section :format do
+            config_set_default :@type, DEFAULT_FORMAT_TYPE
+        end
+  
+        config_section :buffer do
+            config_set_default :chunk_keys, ['time']
+            config_set_default :timekey, (60 * 60 * 24)
+        end
+
+        def configure(conf)
+            compat_parameters_convert(conf, :buffer, :formatter, :inject)
+            super
+
+            @formatter = formatter_create
+      
+            if @localtime
+              @path_slicer = Proc.new {|path|
+                Time.now.strftime(path)
+              }
+            else
+              @path_slicer = Proc.new {|path|
+                Time.now.utc.strftime(path)
+              }
+            end
+      
+            if @azure_container.nil?
+              raise Fluent::ConfigError, "azure_container is needed"
+            end
+
+            @azure_storage_path = ''
+            @last_azure_storage_path = ''
+            @current_index = 0
+        end
+
+        def multi_workers_ready?
+            true
+        end
+
+        def start
+            setup_access_token
+            ensure_container
+            super
+        end
+
+        def write(chunk)
+            metadata = chunk.metadata
+            
+            #tmp = Tempfile.new("azure-")
+            #begin
+            #    tmp.close
+            #    generate_log_name(metadata, @current_index)
+            #    if @last_azure_storage_path != @azure_storage_path
+            #        @current_index = 0
+            #        generate_log_name(metadata, @current_index)
+            #    end
+            #    content = File.open(tmp.path, 'rb') { |file| file.read }
+            #    raw_data = raw_data.chomp
+            #    log.debug "Content: #{content}"
+            #    upload_blob(content, metadata)
+            #    @last_azure_storage_path = @azure_storage_path
+            #ensure
+            #    tmp.close(true) rescue nil
+            #    tmp.unlink
+            #end
+            raw_data=''
+            generate_log_name(metadata, @current_index)
+            if @last_azure_storage_path != @azure_storage_path
+                @current_index = 0
+                generate_log_name(metadata, @current_index)
+            end
+            chunk.each do |emit_time, record|
+                line = record["message"].chomp
+                raw_data << "#{line}\n"
+            end
+            raw_data = raw_data.chomp
+            unless raw_data.empty?
+                upload_blob(raw_data, metadata)
+            end
+            @last_azure_storage_path = @azure_storage_path
+
+        end
+
+        private
+        def upload_blob(content, metadata)
+            log.debug "azurestorage_gen2:  Uploading blob: #{@azure_storage_path}"
+            existing_content_length = get_blob_properties(@azure_storage_path)
+            if existing_content_length == 0
+                create_blob(@azure_storage_path)
+            end
+            append_blob(content, metadata, existing_content_length)
+        end
+
+        def format(tag, time, record)
+            r = inject_values_to_record(tag, time, record)
+            @formatter.format(tag, time, r)
+        end
+
+        private
+        def generate_log_name(metadata, index)
+            time_slice = if metadata.timekey.nil?
+                       ''.freeze
+                     else
+                       Time.at(metadata.timekey).utc.strftime(@time_slice_format)
+                     end
+            path = @path_slicer.call(@path)
+            values_for_object_key = {
+                "%{path}" => path,
+                "%{time_slice}" => time_slice,
+                "%{index}" => index,
+                "%{file_extension}" => @file_extension
+            }
+            storage_path = @azure_object_key_format.gsub(%r(%{[^}]+}), values_for_object_key)
+            extracted_path = extract_placeholders(storage_path, metadata)
+            extracted_path = "/" + extracted_path unless extracted_path.start_with?("/")
+            @azure_storage_path = extracted_path
+        end
+
+        def setup_access_token
+            @get_token_lock = Concurrent::ReadWriteLock.new
+            acquire_access_token
+            if @azure_oauth_refresh_interval > 0
+                log.info("azurestorage_gen2: Start getting access token every #{@azure_oauth_refresh_interval} seconds.")
+                @get_token_task = Concurrent::TimerTask.new(
+                    execution_interval: @azure_oauth_refresh_interval) {
+                    begin
+                        acquire_access_token
+                    rescue Exception => e
+                        log.warn("#{e.message}, continue with previous credentials.")
+                    end
+                }
+                @get_token_task.execute
+            end
+        end
+
+        # Referenced from azure doc.
+        # https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/tutorial-linux-vm-access-storage#get-an-access-token-and-use-it-to-call-azure-storage
+        def acquire_access_token
+            params = { :"api-version" => ACCESS_TOKEN_API_VERSION, :resource => "https://storage.azure.com/" }
+            unless @azure_instance_msi.nil?
+                params[:msi_res_id] = @azure_instance_msi
+            end
+            request = Typhoeus::Request.new("http://169.254.169.254/metadata/identity/oauth2/token", params: params, headers: { Metadata: "true"})
+            request.on_complete do |response|
+                if response.success?
+                  data = JSON.parse(response.body)
+                  log.debug "azurestorage_gen2: Token response: #{data}"
+                  @azure_access_token = data["access_token"]
+                else
+                    raise Fluent::UnrecoverableError, "Failed to acquire access token. #{response.code}: #{response.body}"
+                end
+            end
+            request.run
+        end
+
+        private
+        def ensure_container
+            headers = {:"x-ms-version" =>  ABFS_API_VERSION, :"Authorization" => "Bearer #{@azure_access_token}",:"Content-Length" => "0"}
+            params = {:resource => "filesystem" }
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}", :method => :head, :params => params, :headers=> headers)
+            request.on_complete do |response|
+                if response.success?
+                  log.info "azurestorage_gen2: Container '#{@azure_container}' exists."
+                elsif response.timed_out?
+                    raise Fluent::UnrecoverableError,  "Get container '#{@azure_container}' request timed out."
+                elsif response.code == 404
+                    log.info "azurestorage_gen2: Container '#{@azure_container}' does not exist. Creating it if needed..."
+                    if @auto_create_container
+                        create_container
+                    else
+                        raise Fluent::ConfigError, "The specified container does not exist: container = #{@azure_container}"
+                    end
+                else
+                    raise Fluent::UnrecoverableError, "Get container request failed - code: #{response.code}, body: #{response.body}"
+                end
+            end
+            request.run
+        end
+
+        private
+        def create_container
+            headers = {:"x-ms-version" =>  ABFS_API_VERSION, :"Authorization" => "Bearer #{@azure_access_token}",:"Content-Length" => "0"}
+            params = {:resource => "filesystem" }
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}", :method => :put, :params => params, :headers=> headers)
+            request.on_complete do |response|
+                if response.success?
+                    log.debug "azurestorage_gen2: Container '#{@azure_container}' created, response code: #{response.code}"
+                elsif response.timed_out?
+                    raise Fluent::UnrecoverableError,  "Creating container '#{@azure_container}' request timed out."
+                else
+                    raise Fluent::UnrecoverableError, "Creating container request failed - code: #{response.code}, body: #{response.body}"
+                end
+            end
+            request.run
+        end
+
+        private
+        def create_blob(blob_path)
+            headers = {:"x-ms-version" =>  ABFS_API_VERSION, :"Authorization" => "Bearer #{@azure_access_token}",:"Content-Length" => "0", :"Content-Type" => "application/json"}
+            params = {:resource => "file", :recursive => "false"}
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}/testprefix#{blob_path}", :method => :put, :params => params, :headers=> headers)
+            request.on_complete do |response|
+                if response.success?
+                    log.debug "azurestorage_gen2: Blob '#{blob_path}' has been created, response code: #{response.code}"
+                elsif response.timed_out?
+                    raise Fluent::UnrecoverableError,  "Creating blob '#{@blob_path}' request timed out."
+                elsif response.code == 409
+                    log.debug "azurestorage_gen2: Blob already exists: #{blob_path}"
+                else
+                    raise Fluent::UnrecoverableError, "Creating blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}"
+                end
+            end
+            request.run
+        end
+
+        private
+        def append_blob_block(blob_path, content, position)
+            log.debug "azurestorage_gen2: append_blob_block.start: Append blob ('#{blob_path}') called with position #{position}"
+            headers = {:"x-ms-version" =>  ABFS_API_VERSION, :"x-ms-content-type" => "text/plain", :"Authorization" => "Bearer #{@azure_access_token}"}
+            params = {:action => "append", :position => "#{position}"}
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :patch, :body => content, :params => params, :headers=> headers)
+            request.on_complete do |response|
+                if response.success?
+                    log.debug "azurestorage_gen2: Blob '#{blob_path}' has been appended, response code: #{response.code}"
+                elsif response.timed_out?
+                    raise Fluent::UnrecoverableError,  "Appending blob #{@blob_path}' request timed out."
+                elsif response.code == 404
+                    raise AppendBlobResponseError.new("Blob '#{blob_path}' has not found. Error code: #{response.code}", 404)
+                elsif response.code == 409
+                    raise AppendBlobResponseError.new("Blob '#{blob_path}' has conflict. Error code: #{response.code}", 409)
+                else
+                    raise Fluent::UnrecoverableError, "Appending blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}"
+                end
+            end
+            request.run
+        end
+
+        private
+        def flush(blob_path, position)
+            log.debug "azurestorage_gen2: flush_blob.start: Flush blob ('#{blob_path}') called with position #{position}"
+            headers = {:"x-ms-version" =>  ABFS_API_VERSION, :"Authorization" => "Bearer #{@azure_access_token}",:"Content-Length" => "0"}
+            params = {:action => "flush", :position => "#{position}"}
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :patch, :params => params, :headers=> headers)
+            request.on_complete do |response|
+                if response.success?
+                    log.debug "azurestorage_gen2: Blob '#{blob_path}' flush was successful, response code: #{response.code}"
+                elsif response.timed_out?
+                    raise Fluent::UnrecoverableError,  "Bloub '#{@blob_path}' flush request timed out."
+                else
+                    raise Fluent::UnrecoverableError, "Blob flush request failed - code: #{response.code}, body: #{response.body}"
+                end
+            end
+            request.run
+        end
+
+        private
+        def get_blob_properties(blob_path)
+            headers = {:"x-ms-version" =>  ABFS_API_VERSION, :"Authorization" => "Bearer #{@azure_access_token}",:"Content-Length" => "0"}
+            params = {}
+            content_length = -1
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :head, :params => params, :headers=> headers)
+            request.on_complete do |response|
+                if response.success?
+                  log.debug "azurestorage_gen2: Get blob properties for '#{blob_path}', response headers: #{response.headers}"
+                  content_length = response.headers['Content-Length'].to_i
+                elsif response.timed_out?
+                    raise Fluent::UnrecoverableError,  "Get blob properties '#{@blob_path}' request timed out."
+                elsif response.code == 404
+                    log.debug "azurestorage_gen2: Blob '#{@blob_path}' does not exist. Creating it if needed..."
+                    content_length = 0
+                else
+                    raise Fluent::UnrecoverableError, "Get blob properties '#{@blob_path}' request failed - code: #{response.code}, body: #{response.body}"
+                end
+            end
+            request.run
+            content_length
+        end
+
+        private
+        def append_blob(content, metadata, existing_content_length)
+          position = 0
+          log.debug "azurestorage_gen2: append_blob.start: Content size: #{content.length}"
+          loop do
+            begin
+              size = [content.length - position, AZURE_BLOCK_SIZE_LIMIT].min
+              log.debug "azurestorage_gen2: append_blob.chunk: content[#{position}..#{position + size}]"
+              append_blob_block(@azure_storage_path, content[position..position + size], existing_content_length)
+
+              position += size
+              existing_content_length += size
+              break if position >= content.length
+            rescue AppendBlobResponseError => ex
+              status_code = ex.status_code
+  
+              if status_code == 409 # exceeds azure block limit
+                @current_index += 1
+                old_azure_storage_path = @azure_storage_path
+                generate_log_name(metadata, @current_index)
+  
+                # If index is not a part of format, rethrow exception.
+                if old_azure_storage_path == @azure_storage_path
+                  log.warn "azurestorage_gen2: append_blob: blocks limit reached, you need to use %{index} for the format."
+                  raise
+                end
+                flush(old_azure_storage_path, existing_content_length)
+  
+                log.info "azurestorage_gen2: append_blob: blocks limit reached, creating new blob #{@azure_storage_path}."
+                create_blob(@azure_storage_path)
+              elsif status_code == 404 # blob not found
+                log.debug "azurestorage_gen2: append_blob: #{@azure_storage_path} blob doesn't exist, creating new blob."
+                create_blob(@azure_storage_path)
+              else
+                raise
+              end
+            end
+          end
+          flush(@azure_storage_path, existing_content_length)
+          log.debug "azurestorage_gen2: append_blob.complete"
+        end
+    end
+
+    class AppendBlobResponseError < StandardError
+        attr_reader :status_code
+        def initialize(message="Default message", status_code=0)
+          @status_code = status_code
+          super(message)
+        end
+    end
+end

data/test/helper.rb

@@ -0,0 +1,8 @@
+$LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
+require "test-unit"
+require "fluent/test"
+require "fluent/test/driver/output"
+require "fluent/test/helpers"
+
+Test::Unit::TestCase.include(Fluent::Test::Helpers)
+Test::Unit::TestCase.extend(Fluent::Test::Helpers)

data/test/plugin/test_out_azurestorage_gen2.rb

@@ -0,0 +1,54 @@
+require 'fluent/test'
+require 'fluent/test/driver/output'
+require 'fluent/test/helpers'
+require 'fluent/plugin/out_azurestorage_gen2'
+
+require 'test/unit/rr'
+require 'zlib'
+require 'fileutils'
+
+include Fluent::Test::Helpers
+
+class AzureStorageGen2OutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    azure_storage_account test_storage_account
+    azure_storage_access_key dGVzdF9zdG9yYWdlX2FjY2Vzc19rZXk=
+    azure_container test_container
+    path log
+    utc
+    buffer_type memory
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::AzureStorageGen2Output) do
+      # for testing.
+      def contents
+        @emit_streams
+      end
+
+      def write(chunk)
+        @emit_streams = []
+        event = chunk.read
+        @emit_streams << event
+      end
+
+      private
+      def ensure_container
+      end
+
+    end.configure(conf)
+  end
+
+  def test_configure
+    # TODO write tests
+    d = create_driver
+    assert_equal 'test_storage_account', d.instance.azure_storage_account
+    assert_equal 'dGVzdF9zdG9yYWdlX2FjY2Vzc19rZXk=', d.instance.azure_storage_access_key
+    assert_equal 'test_container', d.instance.azure_container
+    assert_equal 'log', d.instance.path
+  end
+end

metadata

@@ -0,0 +1,252 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-azurestorage-gen2
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Oliver Szabo
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-12-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: azure-storage-common
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+- !ruby/object:Gem::Dependency
+  name: azure-storage-blob
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+- !ruby/object:Gem::Dependency
+  name: uuidtools
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.5
+- !ruby/object:Gem::Dependency
+  name: typhoeus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+- !ruby/object:Gem::Dependency
+  name: yajl-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 12.3.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 12.3.1
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.3.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.3.3
+- !ruby/object:Gem::Dependency
+  name: test-unit-rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.5
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.5
+description: Azure Storage output plugin for Fluentd event collector
+email:
+- oleewere@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE
+- Makefile
+- README.md
+- Rakefile
+- VERSION
+- fluent-plugin-azurestorage-gen2.gemspec
+- lib/fluent/plugin/out_azurestorage_gen2.rb
+- test/helper.rb
+- test/plugin/test_out_azurestorage_gen2.rb
+homepage: https://github.com/oleewere/fluent-plugin-azurestorage-gen2
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Azure Storage output plugin for Fluentd event collector
+test_files:
+- test/helper.rb
+- test/plugin/test_out_azurestorage_gen2.rb