fluent-plugin-azurestorage-gen2 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6f15931cc13f12c4aeb6723ba2a44c604998e819
-  data.tar.gz: 557964c5e84e0274a840af3e18500d9197ea8cd4
+  metadata.gz: 511b8f199ba0f1c6b7937711b0e3d146baf3a856
+  data.tar.gz: ff6cf330c0ecacd65a48c8f312a87bb8bdb402a5
 SHA512:
-  metadata.gz: 7d0e97ee7d6b1429e98f91aa72135c4f84bf4cfd8b573cc8ec747cb36fe0e65fb98bce5d02d5416db83a4946c406c93f27ede48a1ba989d25bf675771311e6cd
-  data.tar.gz: 6ed3f231452a94eb0e66244648be66500af2fd82577b3796df11d0cda363238b95b50cc478ae01b54d1e8deb4cdb12101151ccfecb8177f53923db97b395248b
+  metadata.gz: 368340200f930c8f4984ab5b630e779f375061d82faab35cf55f1ae83f7b92e58f63f7a4a152880a51d62840a1fe9e4bc56ffacdb3a6c1930c5a617d11378eae
+  data.tar.gz: e0a54bb0bfd963134cf7e1afeb80c8ea936e3421002d444d9cf6abbaaafc8d2e1bcedcbb118afbe0a2a44a76c9d9ebfe207495af11cb0c19a2f63621cc2c83ab

data/README.md

@@ -1,4 +1,4 @@
-# Azure Datalake Storage Gen2 Fluentd Output Plugin (IN PROGRESS)
+# Azure Datalake Storage Gen2 Fluentd Output Plugin
 
 [![Build Status](https://travis-ci.org/oleewere/fluent-plugin-azurestorage-gen2.svg?branch=master)](https://travis-ci.org/oleewere/fluent-plugin-azurestorage-gen2)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
@@ -9,11 +9,11 @@
 
 | fluent-plugin-azurestorage-gen2 | fluentd | ruby |
 |------------------------|---------|------|
-| >= 0.1.1 | >= v0.14.0 | >= 2.4 |
+| >= 0.1.2 | >= v0.14.0 | >= 2.4 |
 
 ## Overview
 
-Fluent output plugin that can use ABFS api and append blobs with MSI support
+Fluent output plugin that can use ABFS api and append blobs with MSI and OAuth support.
 
 ## Installation
 
@@ -24,6 +24,7 @@ $ gem install fluent-plugin-azurestorage-gen2
 
 ## Configuration
 
+- Configuration on VMs by using MSI:
 ```
 <match **>
   @type azurestorage_gen2
@@ -47,6 +48,32 @@ $ gem install fluent-plugin-azurestorage-gen2
 </match>
 ```
 
+- Configuration outside of VMs with OAuth credentials:
+```
+<match **>
+  @type azurestorage_gen2
+  azure_storage_account            mystorageabfs
+  azure_container                  mycontainer
+  azure_object_key_format          %{path}-%{index}.%{file_extension}
+  azure_oauth_tenant_id            <my tenant id>
+  azure_oauth_app_id               <my app client id>
+  azure_oauth_secret               <my client secret>
+  azure_oauth_refresh_interval     3600
+  time_slice_format                %Y%m%d-%H
+  file_extension                   log
+  path                             "/cluster-logs/myfolder/${tag[1]}-#{Socket.gethostname}-%M"
+  auto_create_container            true
+  <buffer tag,time>
+    @type file
+    path /var/log/fluent/azurestorage-buffer
+    timekey 5m
+    timekey_wait 0s
+    timekey_use_utc true
+    chunk_limit_size 64m
+  </buffer>
+</match>
+```
+
 ### Configuration options
 
 ### azure_storage_account
@@ -62,9 +89,21 @@ Your Azure Storage Access Key(Primary or Secondary). This also can be got from A
 
 Your Azure Managed Service Identity ID. When storage key authentication is not used, the plugin uses OAuth2 to authenticate as given MSI. This authentication method only works on Azure VM. If the VM has only one MSI assigned, this parameter becomes optional and the only MSI will be used. Otherwise this parameter is required.
 
+### azure_oauth_tenant_id
+
+Azure account tenant id from your Azure Directory. Required if OAuth based credential mechanism is used.
+
+### azure_oauth_app_id
+
+OAuth client id that is used for OAuth based authentication. Required if OAuth based credential mechanism is used.
+
+### azure_oauth_secret
+
+OAuth client secret that is used for OAuth based authentication. Required if OAuth based credential mechanism is used.
+
 ### azure_oauth_refresh_interval
 
-OAuth2 access token refreshment interval in second. Only applies when MSI authentication is used.
+OAuth2 access token refreshment interval in second. Only applies when MSI / OAuth authentication is used.
 
 ### azure_container (Required)
 
@@ -205,7 +244,6 @@ Use UTC instead of local time.
 ## TODOs
 
 - add storage key support
-- add compression (if append is not used)
 
 ## Contributing
 

data/VERSION

@@ -1 +1 @@
-0.1.1
+0.1.2

data/lib/fluent/plugin/azurestorage_gen2_compressor_gzip_command.rb

@@ -0,0 +1,51 @@
+module Fluent
+  class AzureStorageGen2Output
+    class GzipCommandCompressor < Compressor
+      AzureStorageGen2Output.register_compressor('gzip_command', self)
+
+      config_param :command_parameter, :string, :default => ''
+
+      def configure(conf)
+        super
+        check_command('gzip')
+      end
+
+      def ext
+        'gz'.freeze
+      end
+
+      def content_type
+        'application/x-gzip'.freeze
+      end
+
+      def compress(chunk, tmp)
+        chunk_is_file = @buffer_type == 'file'
+        path = if chunk_is_file
+                 chunk.path
+               else
+                 w = Tempfile.new("chunk-gzip-tmp")
+                 chunk.write_to(w)
+                 w.close
+                 w.path
+               end
+
+        res = system "gzip #{@command_parameter} -c #{path} > #{tmp.path}"
+        unless res
+          log.warn "failed to execute gzip command. Fallback to GzipWriter. status = #{$?}"
+          begin
+            tmp.truncate(0)
+            gw = Zlib::GzipWriter.new(tmp)
+            chunk.write_to(gw)
+            gw.close
+          ensure
+            gw.close rescue nil
+          end
+        end
+      ensure
+        unless chunk_is_file
+          w.close(true) rescue nil
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/azurestorage_gen2_compressor_lzma2.rb

@@ -0,0 +1,34 @@
+module Fluent
+  class AzureStorageGen2Output
+    class LZMA2Compressor < Compressor
+      AzureStorageGen2Output.register_compressor('lzma2', self)
+
+      config_param :command_parameter, :string, :default => '-qf0'
+
+      def configure(conf)
+        super
+        check_command('xz', 'LZMA2')
+      end
+
+      def ext
+        'xz'.freeze
+      end
+
+      def content_type
+        'application/x-xz'.freeze
+      end
+
+      def compress(chunk, tmp)
+        w = Tempfile.new("chunk-xz-tmp")
+        chunk.write_to(w)
+        w.close
+
+        # We don't check the return code because we can't recover lzop failure.
+        system "xz #{@command_parameter} -c #{w.path} > #{tmp.path}"
+      ensure
+        w.close rescue nil
+        w.unlink rescue nil
+      end
+    end
+  end
+end

data/lib/fluent/plugin/azurestorage_gen2_compressor_lzo.rb

@@ -0,0 +1,34 @@
+module Fluent
+  class AzureStorageGen2Output
+    class LZOCompressor < Compressor
+      AzureStorageGen2Output.register_compressor('lzo', self)
+
+      config_param :command_parameter, :string, :default => '-qf1'
+
+      def configure(conf)
+        super
+        check_command('lzop', 'LZO')
+      end
+
+      def ext
+        'lzo'.freeze
+      end
+
+      def content_type
+        'application/x-lzop'.freeze
+      end
+
+      def compress(chunk, tmp)
+        w = Tempfile.new("chunk-tmp")
+        chunk.write_to(w)
+        w.close
+
+        # We don't check the return code because we can't recover lzop failure.
+        system "lzop #{@command_parameter} -o #{tmp.path} #{w.path}"
+      ensure
+        w.close rescue nil
+        w.unlink rescue nil
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_azurestorage_gen2.rb

@@ -1,4 +1,6 @@
 require 'net/http'
+require 'base64'
+require 'openssl'
 require 'json'
 require 'tempfile'
 require 'time'
@@ -21,14 +23,19 @@ module Fluent::Plugin
         config_param :azure_storage_account, :string, :default => nil
         config_param :azure_storage_access_key, :string, :default => nil, :secret => true
         config_param :azure_instance_msi, :string, :default => nil
+        config_param :azure_oauth_app_id, :string, :default => nil, :secret => true
+        config_param :azure_oauth_secret, :string, :default => nil, :secret => true
+        config_param :azure_oauth_tenant_id, :string, :default => nil
         config_param :azure_oauth_refresh_interval, :integer, :default => 60 * 60 # one hour
         config_param :azure_container, :string, :default => nil
         config_param :azure_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
         config_param :file_extension, :string, :default => "log"
-        config_param :store_as, :string, :default => "gzip"
+        config_param :store_as, :string, :default => "none"
         config_param :auto_create_container, :bool, :default => false
         config_param :format, :string, :default => "out_file"
         config_param :time_slice_format, :string, :default => '%Y%m%d'
+        config_param :command_parameter, :string, :default => nil
+        config_param :message_field, :string, :default => nil
 
         DEFAULT_FORMAT_TYPE = "out_file"
         URL_DOMAIN_SUFFIX = '.dfs.core.windows.net'
@@ -49,6 +56,14 @@ module Fluent::Plugin
             compat_parameters_convert(conf, :buffer, :formatter, :inject)
             super
 
+            begin
+                @compressor = COMPRESSOR_REGISTRY.lookup(@store_as).new(:buffer_type => @buffer_type, :log => log)
+            rescue => e
+                log.warn "#{@store_as} not found. Use 'text' instead"
+                @compressor = TextCompressor.new
+            end
+            @compressor.configure(conf)
+
             @formatter = formatter_create
       
             if @localtime
@@ -68,6 +83,14 @@ module Fluent::Plugin
             @azure_storage_path = ''
             @last_azure_storage_path = ''
             @current_index = 0
+            if @store_as.nil? || @store_as == "none"
+                @blob_content_type = "text/plain"
+                @final_file_extension = @file_extension
+            else
+                @blob_content_type = @compressor.content_type
+                @final_file_extension = @compressor.ext
+            end
+
         end
 
         def multi_workers_ready?
@@ -82,39 +105,44 @@ module Fluent::Plugin
 
         def write(chunk)
             metadata = chunk.metadata
-            
-            #tmp = Tempfile.new("azure-")
-            #begin
-            #    tmp.close
-            #    generate_log_name(metadata, @current_index)
-            #    if @last_azure_storage_path != @azure_storage_path
-            #        @current_index = 0
-            #        generate_log_name(metadata, @current_index)
-            #    end
-            #    content = File.open(tmp.path, 'rb') { |file| file.read }
-            #    raw_data = raw_data.chomp
-            #    log.debug "Content: #{content}"
-            #    upload_blob(content, metadata)
-            #    @last_azure_storage_path = @azure_storage_path
-            #ensure
-            #    tmp.close(true) rescue nil
-            #    tmp.unlink
-            #end
-            raw_data=''
-            generate_log_name(metadata, @current_index)
-            if @last_azure_storage_path != @azure_storage_path
-                @current_index = 0
+            if @store_as.nil? || @store_as == "none"
+                raw_data=''
                 generate_log_name(metadata, @current_index)
+                if @last_azure_storage_path != @azure_storage_path
+                    @current_index = 0
+                    generate_log_name(metadata, @current_index)
+                end
+                chunk.each do |emit_time, record|
+                    if @message_field.nil? || @message_field.empty?
+                        raw_data << "#{Yajl.dump(record)}\n"
+                    elsif record.key?(@message_field)
+                        line = record[@message_field].chomp
+                        raw_data << "#{line}\n"
+                    end
+                end
+                raw_data = raw_data.chomp
+                unless raw_data.empty?
+                    upload_blob(raw_data, metadata)
+                end
+                @last_azure_storage_path = @azure_storage_path
+            else
+                tmp = Tempfile.new("azure-")
+                begin
+                    @compressor.compress(chunk, tmp)
+                    tmp.close
+                    generate_log_name(metadata, @current_index)
+                    if @last_azure_storage_path != @azure_storage_path
+                        @current_index = 0
+                        generate_log_name(metadata, @current_index)
+                    end
+                    log.debug "Start uploading temp file: #{tmp.path}"
+                    content = File.open(tmp.path, 'rb') { |file| file.read }
+                    upload_blob(content, metadata)
+                    @last_azure_storage_path = @azure_storage_path
+                ensure
+                    tmp.unlink
+                end
             end
-            chunk.each do |emit_time, record|
-                line = record["message"].chomp
-                raw_data << "#{line}\n"
-            end
-            raw_data = raw_data.chomp
-            unless raw_data.empty?
-                upload_blob(raw_data, metadata)
-            end
-            @last_azure_storage_path = @azure_storage_path
 
         end
 
@@ -145,7 +173,8 @@ module Fluent::Plugin
                 "%{path}" => path,
                 "%{time_slice}" => time_slice,
                 "%{index}" => index,
-                "%{file_extension}" => @file_extension
+                "%{uuid_flush}" => uuid_random,
+                "%{file_extension}" => @final_file_extension
             }
             storage_path = @azure_object_key_format.gsub(%r(%{[^}]+}), values_for_object_key)
             extracted_path = extract_placeholders(storage_path, metadata)
@@ -154,25 +183,40 @@ module Fluent::Plugin
         end
 
         def setup_access_token
-            @get_token_lock = Concurrent::ReadWriteLock.new
-            acquire_access_token
-            if @azure_oauth_refresh_interval > 0
-                log.info("azurestorage_gen2: Start getting access token every #{@azure_oauth_refresh_interval} seconds.")
-                @get_token_task = Concurrent::TimerTask.new(
-                    execution_interval: @azure_oauth_refresh_interval) {
-                    begin
-                        acquire_access_token
-                    rescue Exception => e
-                        log.warn("#{e.message}, continue with previous credentials.")
-                    end
-                }
-                @get_token_task.execute
+            if @azure_storage_access_key.nil?
+                @get_token_lock = Concurrent::ReadWriteLock.new
+                acquire_access_token
+                if @azure_oauth_refresh_interval > 0
+                    log.info("azurestorage_gen2: Start getting access token every #{@azure_oauth_refresh_interval} seconds.")
+                    @get_token_task = Concurrent::TimerTask.new(
+                        execution_interval: @azure_oauth_refresh_interval) {
+                        begin
+                            acquire_access_token
+                        rescue Exception => e
+                            log.warn("#{e.message}, continue with previous credentials.")
+                        end
+                    }
+                    @get_token_task.execute
+                end
+            else
+                log.info "azurestorage_gen2: Access storage key is configured, MSI support is disabled."
+            end
+        end
+
+        def acquire_access_token
+            if !@azure_instance_msi.nil?
+                acquire_access_token_msi
+            elsif !@azure_oauth_app_id.nil? and !@azure_oauth_secret.nil? and !@azure_oauth_tenant_id.nil?
+                acquire_access_token_oauth_app
+            else
+                raise Fluent::UnrecoverableError, "Using MSI or simple OAuth 2.0 based authentication parameters (azure_oauth_tenant_id, azure_oauth_app_id, azure_oauth_secret) are required."
             end
         end
 
         # Referenced from azure doc.
         # https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/tutorial-linux-vm-access-storage#get-an-access-token-and-use-it-to-call-azure-storage
-        def acquire_access_token
+        private
+        def acquire_access_token_msi
             params = { :"api-version" => ACCESS_TOKEN_API_VERSION, :resource => "https://storage.azure.com/" }
             unless @azure_instance_msi.nil?
                 params[:msi_res_id] = @azure_instance_msi
@@ -190,10 +234,30 @@ module Fluent::Plugin
             request.run
         end
 
+        private
+        def acquire_access_token_oauth_app
+            params = { :"api-version" => ACCESS_TOKEN_API_VERSION, :resource => "https://storage.azure.com/"}
+            content = "grant_type=client_credentials&client_id=#{@azure_oauth_app_id}&client_secret=#{@azure_oauth_secret}&resource=https://storage.azure.com/"
+            request = Typhoeus::Request.new("https://login.microsoftonline.com/#{@azure_oauth_tenant_id}/oauth2/token", params: params, :body => content)
+            request.on_complete do |response|
+                if response.success?
+                  data = JSON.parse(response.body)
+                  log.debug "azurestorage_gen2: Token response: #{data}"
+                  @azure_access_token = data["access_token"]
+                else
+                    raise Fluent::UnrecoverableError, "Failed to acquire access token. #{response.code}: #{response.body}"
+                end
+            end
+            request.run
+        end
+
         private
         def ensure_container
-            headers = {:"x-ms-version" =>  ABFS_API_VERSION, :"Authorization" => "Bearer #{@azure_access_token}",:"Content-Length" => "0"}
+            datestamp = create_request_date
+            headers = {:"x-ms-version" =>  ABFS_API_VERSION, :"x-ms-date" => datestamp,:"Content-Length" => "0"}
             params = {:resource => "filesystem" }
+            auth_header = create_auth_header("head", datestamp, "#{@azure_container}", headers, params)
+            headers[:Authorization] = auth_header
             request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}", :method => :head, :params => params, :headers=> headers)
             request.on_complete do |response|
                 if response.success?
@@ -208,7 +272,7 @@ module Fluent::Plugin
                         raise Fluent::ConfigError, "The specified container does not exist: container = #{@azure_container}"
                     end
                 else
-                    raise Fluent::UnrecoverableError, "Get container request failed - code: #{response.code}, body: #{response.body}"
+                    raise Fluent::UnrecoverableError, "Get container request failed - code: #{response.code}, headers: #{response.headers}"
                 end
             end
             request.run
@@ -216,8 +280,11 @@ module Fluent::Plugin
 
         private
         def create_container
-            headers = {:"x-ms-version" =>  ABFS_API_VERSION, :"Authorization" => "Bearer #{@azure_access_token}",:"Content-Length" => "0"}
+            datestamp = create_request_date
+            headers = {:"x-ms-version" =>  ABFS_API_VERSION, :"x-ms-date" => datestamp, :"Content-Length" => "0"}
             params = {:resource => "filesystem" }
+            auth_header = create_auth_header("put", datestamp, "#{@azure_container}", headers, params)
+            headers[:Authorization] = auth_header
             request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}", :method => :put, :params => params, :headers=> headers)
             request.on_complete do |response|
                 if response.success?
@@ -225,7 +292,7 @@ module Fluent::Plugin
                 elsif response.timed_out?
                     raise Fluent::UnrecoverableError,  "Creating container '#{@azure_container}' request timed out."
                 else
-                    raise Fluent::UnrecoverableError, "Creating container request failed - code: #{response.code}, body: #{response.body}"
+                    raise Fluent::UnrecoverableError, "Creating container request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
                 end
             end
             request.run
@@ -233,8 +300,11 @@ module Fluent::Plugin
 
         private
         def create_blob(blob_path)
-            headers = {:"x-ms-version" =>  ABFS_API_VERSION, :"Authorization" => "Bearer #{@azure_access_token}",:"Content-Length" => "0", :"Content-Type" => "application/json"}
+            datestamp = create_request_date
+            headers = {:"x-ms-version" =>  ABFS_API_VERSION, :"x-ms-date" => datestamp,:"Content-Length" => "0", :"Content-Type" => "application/json"}
             params = {:resource => "file", :recursive => "false"}
+            auth_header = create_auth_header("put", datestamp, "#{@azure_container}#{blob_path}", headers, params)
+            headers[:Authorization] = auth_header
             request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :put, :params => params, :headers=> headers)
             request.on_complete do |response|
                 if response.success?
@@ -244,7 +314,7 @@ module Fluent::Plugin
                 elsif response.code == 409
                     log.debug "azurestorage_gen2: Blob already exists: #{blob_path}"
                 else
-                    raise Fluent::UnrecoverableError, "Creating blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}"
+                    raise Fluent::UnrecoverableError, "Creating blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
                 end
             end
             request.run
@@ -253,8 +323,11 @@ module Fluent::Plugin
         private
         def append_blob_block(blob_path, content, position)
             log.debug "azurestorage_gen2: append_blob_block.start: Append blob ('#{blob_path}') called with position #{position}"
-            headers = {:"x-ms-version" =>  ABFS_API_VERSION, :"x-ms-content-type" => "text/plain", :"Authorization" => "Bearer #{@azure_access_token}"}
+            datestamp = create_request_date
+            headers = {:"x-ms-version" =>  ABFS_API_VERSION,  :"x-ms-date" => datestamp, :"x-ms-content-type" => "#{@blob_content_type}"}
             params = {:action => "append", :position => "#{position}"}
+            auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}", headers, params)
+            headers[:Authorization] = auth_header
             request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :patch, :body => content, :params => params, :headers=> headers)
             request.on_complete do |response|
                 if response.success?
@@ -266,7 +339,7 @@ module Fluent::Plugin
                 elsif response.code == 409
                     raise AppendBlobResponseError.new("Blob '#{blob_path}' has conflict. Error code: #{response.code}", 409)
                 else
-                    raise Fluent::UnrecoverableError, "Appending blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}"
+                    raise Fluent::UnrecoverableError, "Appending blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
                 end
             end
             request.run
@@ -275,8 +348,11 @@ module Fluent::Plugin
         private
         def flush(blob_path, position)
             log.debug "azurestorage_gen2: flush_blob.start: Flush blob ('#{blob_path}') called with position #{position}"
-            headers = {:"x-ms-version" =>  ABFS_API_VERSION, :"Authorization" => "Bearer #{@azure_access_token}",:"Content-Length" => "0"}
+            datestamp = create_request_date
+            headers = {:"x-ms-version" =>  ABFS_API_VERSION, :"x-ms-date" => datestamp,:"Content-Length" => "0"}
             params = {:action => "flush", :position => "#{position}"}
+            auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}",headers, params)
+            headers[:Authorization] = auth_header
             request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :patch, :params => params, :headers=> headers)
             request.on_complete do |response|
                 if response.success?
@@ -284,7 +360,7 @@ module Fluent::Plugin
                 elsif response.timed_out?
                     raise Fluent::UnrecoverableError,  "Bloub '#{blob_path}' flush request timed out."
                 else
-                    raise Fluent::UnrecoverableError, "Blob flush request failed - code: #{response.code}, body: #{response.body}"
+                    raise Fluent::UnrecoverableError, "Blob flush request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
                 end
             end
             request.run
@@ -292,9 +368,12 @@ module Fluent::Plugin
 
         private
         def get_blob_properties(blob_path)
-            headers = {:"x-ms-version" =>  ABFS_API_VERSION, :"Authorization" => "Bearer #{@azure_access_token}",:"Content-Length" => "0"}
+            datestamp = create_request_date
+            headers = {:"x-ms-version" =>  ABFS_API_VERSION, :"x-ms-date" => datestamp, :"Content-Length" => "0"}
             params = {}
             content_length = -1
+            auth_header = create_auth_header("head", datestamp, "#{@azure_container}#{blob_path}", headers, params)
+            headers[:Authorization] = auth_header
             request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :head, :params => params, :headers=> headers)
             request.on_complete do |response|
                 if response.success?
@@ -306,7 +385,7 @@ module Fluent::Plugin
                     log.debug "azurestorage_gen2: Blob '#{blob_path}' does not exist. Creating it if needed..."
                     content_length = 0
                 else
-                    raise Fluent::UnrecoverableError, "Get blob properties '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}"
+                    raise Fluent::UnrecoverableError, "Get blob properties '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
                 end
             end
             request.run
@@ -354,6 +433,178 @@ module Fluent::Plugin
           flush(@azure_storage_path, existing_content_length)
           log.debug "azurestorage_gen2: append_blob.complete"
         end
+
+        private
+        def create_auth_header(method, datestamp, resource, headers, params)
+            if @azure_storage_access_key.nil?
+                "Bearer #{@azure_access_token}"
+            else
+                "SharedKey #{@azure_storage_account}:#{signed(method, datestamp, resource, headers, params)}"
+            end
+        end
+
+        private
+        def signed(method, datestamp, resource, headers, params)
+            decoded_access_key=Base64.strict_decode64(@azure_storage_access_key).unpack("H*").first
+            sign_request(decoded_access_key, signable_string(method, resource, params, headers, datestamp))
+        end
+
+        private
+        def sign_request(key, signable_string)
+            signed = OpenSSL::HMAC.digest('sha256', key, signable_string)
+            Base64.strict_encode64(signed)
+        end
+
+        private
+        def signable_string(method, resource, params, headers, datestamp)
+            [
+              method.to_s.upcase,
+              headers.fetch("Content-Encoding", ""),
+              headers.fetch("Content-Language", ""),
+              headers.fetch("Content-Length", "").sub(/^0+/, ""),
+              headers.fetch("Content-MD5", ""),
+              headers.fetch("Content-Type", ""),
+              headers.fetch("Date", ""),
+              headers.fetch("If-Modified-Since", ""),
+              headers.fetch("If-Match", ""),
+              headers.fetch("If-None-Match", ""),
+              headers.fetch("If-Unmodified-Since", ""),
+              headers.fetch("Range", ""),
+              "x-ms-date:#{datestamp}\nx-ms-version:#{ABFS_API_VERSION}",
+              get_canonicalized_resource(resource, params)
+            ].join("\n")
+        end
+
+        private
+        def get_canonicalized_resource(resource, params)
+            if params.empty?
+                canonicalized_resource="/#{@azure_storage_account}"
+            else
+                canonicalized_params = params
+                .map{|paramKey, paramValue| "#{paramKey.to_s.downcase}:#{paramValue}"}
+                .join("\n")
+                canonicalized_resource="/#{@azure_storage_account}/#{resource}\n#{canonicalized_params}"
+            end
+        end
+
+        private
+        def hex_to_bin(hex)
+            hex = '0' << hex unless (hex.length % 2) == 0
+            hex.scan(/[A-Fa-f0-9]{2}/).inject('') { |encoded, byte| encoded << [byte].pack('H*') }
+        end
+
+        private
+        def create_request_date
+            Time.now.strftime('%a, %e %b %y %H:%M:%S %Z')
+        end
+
+        def uuid_random
+            require 'uuidtools'
+            ::UUIDTools::UUID.random_create.to_s
+        end
+        
+        def timekey_to_timeformat(timekey)
+            case timekey
+            when nil          then ''
+            when 0...60       then '%Y%m%d%H%M%S' # 60 exclusive
+            when 60...3600    then '%Y%m%d%H%M'
+            when 3600...86400 then '%Y%m%d%H'
+            else                   '%Y%m%d'
+            end
+        end
+
+        class Compressor
+            include Fluent::Configurable
+      
+            def initialize(opts = {})
+              super()
+              @buffer_type = opts[:buffer_type]
+              @log = opts[:log]
+            end
+      
+            attr_reader :buffer_type, :log
+      
+            def configure(conf)
+              super
+            end
+      
+            def ext
+            end
+      
+            def content_type
+            end
+      
+            def compress(chunk, tmp)
+            end
+      
+            private
+            def check_command(command, algo = nil)
+              require 'open3'
+      
+              algo = command if algo.nil?
+              begin
+                Open3.capture3("#{command} -V")
+              rescue Errno::ENOENT
+                raise Fluent::ConfigError, "'#{command}' utility must be in PATH for #{algo} compression"
+              end
+            end
+          end
+      
+        class GzipCompressor < Compressor
+            def ext
+              'gz'.freeze
+            end
+      
+            def content_type
+              'application/x-gzip'.freeze
+            end
+      
+            def compress(chunk, tmp)
+              w = Zlib::GzipWriter.new(tmp)
+              chunk.write_to(w)
+              w.finish
+            ensure
+              w.finish rescue nil
+            end
+        end
+      
+        class TextCompressor < Compressor
+            def ext
+              'txt'.freeze
+            end
+      
+            def content_type
+              'text/plain'.freeze
+            end
+      
+            def compress(chunk, tmp)
+              chunk.write_to(tmp)
+            end
+        end
+      
+        class JsonCompressor < TextCompressor
+            def ext
+              'json'.freeze
+            end
+      
+            def content_type
+              'application/json'.freeze
+            end
+        end
+      
+        COMPRESSOR_REGISTRY = Fluent::Registry.new(:azurestorage_compressor_type, 'fluent/plugin/azurestorage_gen2_compressor_')
+        {
+              'gzip' => GzipCompressor,
+              'json' => JsonCompressor,
+              'text' => TextCompressor
+        }.each { |name, compressor|
+            COMPRESSOR_REGISTRY.register(name, compressor)
+        }
+      
+        def self.register_compressor(name, compressor)
+            COMPRESSOR_REGISTRY.register(name, compressor)
+        end
+
     end
 
     class AppendBlobResponseError < StandardError

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-azurestorage-gen2
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Oliver Szabo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-12-16 00:00:00.000000000 Z
+date: 2019-12-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -180,6 +180,9 @@ files:
 - Rakefile
 - VERSION
 - fluent-plugin-azurestorage-gen2.gemspec
+- lib/fluent/plugin/azurestorage_gen2_compressor_gzip_command.rb
+- lib/fluent/plugin/azurestorage_gen2_compressor_lzma2.rb
+- lib/fluent/plugin/azurestorage_gen2_compressor_lzo.rb
 - lib/fluent/plugin/out_azurestorage_gen2.rb
 - test/helper.rb
 - test/plugin/test_out_azurestorage_gen2.rb