fluent-plugin-azurestorage-gen2 0.1.6 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2f30aa7c60fa869b86eb0d921aab12831c6355b6
-  data.tar.gz: 522010167e94cb9500e89b13aa68c4ec0bba1053
+  metadata.gz: 04eea9a39f7cc1d5533d895d6efd3aa725cd9c44
+  data.tar.gz: 0c3779b86a040e07924c0e7d4f78a01cee9ac274
 SHA512:
-  metadata.gz: 5d40bc5c3964bd9187e6710dbda66f7e3556f310425120cf2c46da19a34c8905676278f4428d744a3255ce83e55bf8bdd2791dae755b25aefcf32d39e771c8b6
-  data.tar.gz: 149ee0c0ffc588e048fd0fa87cc343af153d6dc6de4132e9ace1d06f13b80e9d4f6ca60dba1892972693e6dfb37fc0dc649e233e538ccdcf982bbbc6540d1d1f
+  metadata.gz: 0ec050faad375da7b4b13b5686de44b9c9c7f24cef7bedc1f647392620a3dff379f2f409b475d072947215e50ff6ae6d3a976320467a0d41eb579883dc0ae79f
+  data.tar.gz: c71a5e917920770e4339bab5e208c51b53c9cfcd9a4afe7255996149da0b70db539d6ca12ba983c1147b9611ed8cb342831ce217c7a4adfde93ac3232a2225fb

data/Makefile

@@ -8,8 +8,6 @@ install: gem
 push: gem
 	gem push fluent-plugin-azurestorage-gen2*.gem
 
-tag-and-branch:
+tag:
 	git tag "v$$(cat VERSION)" $(RELEASE_COMMIT)
-	git checkout -b "release/$$(cat VERSION)" $(RELEASE_COMMIT)
-	git push origin "v$$(cat VERSION)"
-	git push -u origin "release/$$(cat VERSION)"
+	git push origin "v$$(cat VERSION)"

data/README.md

@@ -3,13 +3,13 @@
 [![Build Status](https://travis-ci.org/oleewere/fluent-plugin-azurestorage-gen2.svg?branch=master)](https://travis-ci.org/oleewere/fluent-plugin-azurestorage-gen2)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 [![Gem Version](https://badge.fury.io/rb/fluent-plugin-azurestorage-gen2.svg)](http://badge.fury.io/rb/fluent-plugin-azurestorage-gen2)
-![](https://ruby-gem-downloads-badge.herokuapp.com/fluent-plugin-azurestorage-gen2?type=total)
+![](https://ruby-gem-downloads-badge.herokuapp.com/fluent-plugin-azurestorage-gen2?type=total&metric=true)
 
 ## Requirements
 
 | fluent-plugin-azurestorage-gen2 | fluentd | ruby |
 |------------------------|---------|------|
-| >= 0.1.6 | >= v0.14.0 | >= 2.4 |
+| >= 0.1.0 | >= v0.14.0 | >= 2.4 |
 
 ## Overview
 
@@ -34,9 +34,11 @@ $ gem install fluent-plugin-azurestorage-gen2
   azure_object_key_format          %{path}-%{index}.%{file_extension}
   azure_oauth_refresh_interval     3600
   time_slice_format                %Y%m%d-%H
-  file_extension                   log
+  file_extension                   log # only used with store_as none
   path                             "/cluster-logs/myfolder/${tag[1]}-#{Socket.gethostname}-%M"
   auto_create_container            true
+  store_as                         gzip
+  format                           single_value
   <buffer tag,time>
     @type file
     path /var/log/fluent/azurestorage-buffer
@@ -60,9 +62,10 @@ $ gem install fluent-plugin-azurestorage-gen2
   azure_oauth_secret               <my client secret>
   azure_oauth_refresh_interval     3600
   time_slice_format                %Y%m%d-%H
-  file_extension                   log
   path                             "/cluster-logs/myfolder/${tag[1]}-#{Socket.gethostname}-%M"
+  store_as                         gzip
   auto_create_container            true
+  format                           single_value
   <buffer tag,time>
     @type file
     path /var/log/fluent/azurestorage-buffer
@@ -119,7 +122,15 @@ This plugin create container if not exist when you set 'auto_create_container' t
 
 ### skip_container_check
 
-You can skip the initial container listing (and container creation) operations at startup. That can be useful if the user is not allowed to perform these operations.
+You can skip the initial container listing (and container creation) operations at startup. That can be useful if the user is not allowed to perform this operation.
+
+### enable_retry
+
+If you set this option, operations can be retried in the buffer. Default value is false. (Used for create/update/flush Blob operations)
+
+### url_domain_suffix
+
+The defaultt `url_domain_suffix` is `.dfs.core.windows.net`, you can override this in case of private endpoints.
 
 ### azure_object_key_format
 
@@ -173,13 +184,13 @@ azure_object_key_format %{path}/events/ts=%{time_slice}/events_%{index}-%{hostna
 
 ### file_extension
 
-File extension for the uploaded files. Only uses if `store_as` is not set, or set as `none`
+File extension for the uploaded files. Only used if `store_as` is not set, or set as `none`
 
 ### store_as
 
 Archive format on Azure Storage. You can use following types:
 
-- none (default - no tmp file creation for log processing)
+- none (default - no tmp file creation for log processing, use with json or single value format)
 - gzip
 - json
 - text

data/VERSION

@@ -1 +1 @@
-0.1.6
+0.2.4

data/lib/fluent/plugin/azurestorage_gen2_compressor_gzip_command.rb

@@ -24,6 +24,7 @@ module Fluent
                  chunk.path
                else
                  w = Tempfile.new("chunk-gzip-tmp")
+                 w.binmode
                  chunk.write_to(w)
                  w.close
                  w.path

data/lib/fluent/plugin/azurestorage_gen2_compressor_lzma2.rb

@@ -20,6 +20,7 @@ module Fluent
 
       def compress(chunk, tmp)
         w = Tempfile.new("chunk-xz-tmp")
+        w.binmode
         chunk.write_to(w)
         w.close
 

data/lib/fluent/plugin/azurestorage_gen2_compressor_lzo.rb

@@ -20,6 +20,7 @@ module Fluent
 
       def compress(chunk, tmp)
         w = Tempfile.new("chunk-tmp")
+        w.binmode
         chunk.write_to(w)
         w.close
 

data/lib/fluent/plugin/out_azurestorage_gen2.rb

@@ -27,20 +27,20 @@ module Fluent::Plugin
         config_param :azure_oauth_secret, :string, :default => nil, :secret => true
         config_param :azure_oauth_tenant_id, :string, :default => nil
         config_param :azure_oauth_use_azure_cli, :bool, :default => false
-        config_param :azure_oauth_refresh_interval, :integer, :default => 60 * 60 # one hour
+        config_param :azure_oauth_refresh_interval, :integer, :default => 60 * 60
         config_param :azure_container, :string, :default => nil
         config_param :azure_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
         config_param :file_extension, :string, :default => "log"
         config_param :store_as, :string, :default => "none"
         config_param :auto_create_container, :bool, :default => false
         config_param :skip_container_check, :bool, :default => false
+        config_param :enable_retry, :bool, :default => false
+        config_param :url_domain_suffix, :string, :default => '.dfs.core.windows.net'
         config_param :format, :string, :default => "out_file"
         config_param :time_slice_format, :string, :default => '%Y%m%d'
         config_param :command_parameter, :string, :default => nil
-        config_param :message_field, :string, :default => nil
 
         DEFAULT_FORMAT_TYPE = "out_file"
-        URL_DOMAIN_SUFFIX = '.dfs.core.windows.net'
         ACCESS_TOKEN_API_VERSION = "2018-02-01"
         ABFS_API_VERSION = "2018-11-09"
         AZURE_BLOCK_SIZE_LIMIT = 4 * 1024 * 1024 - 1
@@ -118,40 +118,35 @@ module Fluent::Plugin
         def write(chunk)
             metadata = chunk.metadata
             if @store_as.nil? || @store_as == "none"
-                raw_data=''
                 generate_log_name(metadata, @current_index)
                 if @last_azure_storage_path != @azure_storage_path
                     @current_index = 0
                     generate_log_name(metadata, @current_index)
                 end
-                chunk.each do |emit_time, record|
-                    if @message_field.nil? || @message_field.empty?
-                        raw_data << "#{Yajl.dump(record)}\n"
-                    elsif record.key?(@message_field)
-                        line = record[@message_field].chomp
-                        raw_data << "#{line}\n"
-                    end
-                end
+                raw_data = chunk.read
                 unless raw_data.empty?
+                    log.debug "azurestorage_gen2: processing raw data", chunk_id: dump_unique_id_hex(chunk.unique_id)
                     upload_blob(raw_data, metadata)
                 end
+                chunk.close rescue nil
                 @last_azure_storage_path = @azure_storage_path
             else
                 tmp = Tempfile.new("azure-")
+                tmp.binmode
                 begin
                     @compressor.compress(chunk, tmp)
-                    tmp.close
+                    tmp.rewind
                     generate_log_name(metadata, @current_index)
                     if @last_azure_storage_path != @azure_storage_path
                         @current_index = 0
                         generate_log_name(metadata, @current_index)
                     end
-                    log.debug "Start uploading temp file: #{tmp.path}"
+                    log.debug "azurestorage_gen2: Start uploading temp file: #{tmp.path}"
                     content = File.open(tmp.path, 'rb') { |file| file.read }
                     upload_blob(content, metadata)
                     @last_azure_storage_path = @azure_storage_path
                 ensure
-                    tmp.unlink
+                    tmp.close(true) rescue nil
                 end
             end
 
@@ -159,7 +154,7 @@ module Fluent::Plugin
 
         private
         def upload_blob(content, metadata)
-            log.debug "azurestorage_gen2:  Uploading blob: #{@azure_storage_path}"
+            log.debug "azurestorage_gen2: Uploading blob: #{@azure_storage_path}"
             existing_content_length = get_blob_properties(@azure_storage_path)
             if existing_content_length == 0
                 create_blob(@azure_storage_path)
@@ -274,7 +269,7 @@ module Fluent::Plugin
             params = {:resource => "filesystem" }
             auth_header = create_auth_header("head", datestamp, "#{@azure_container}", headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}", :method => :head, :params => params, :headers=> headers)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}", :method => :head, :params => params, :headers=> headers)
             request.on_complete do |response|
                 if response.success?
                   log.info "azurestorage_gen2: Container '#{@azure_container}' exists."
@@ -301,12 +296,12 @@ module Fluent::Plugin
             params = {:resource => "filesystem" }
             auth_header = create_auth_header("put", datestamp, "#{@azure_container}", headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}", :method => :put, :params => params, :headers=> headers)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}", :method => :put, :params => params, :headers=> headers)
             request.on_complete do |response|
                 if response.success?
                     log.debug "azurestorage_gen2: Container '#{@azure_container}' created, response code: #{response.code}"
                 elsif response.timed_out?
-                    raise Fluent::UnrecoverableError,  "Creating container '#{@azure_container}' request timed out."
+                    raise Fluent::UnrecoverableError, "Creating container '#{@azure_container}' request timed out."
                 else
                     raise Fluent::UnrecoverableError, "Creating container request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
                 end
@@ -321,16 +316,16 @@ module Fluent::Plugin
             params = {:resource => "file", :recursive => "false"}
             auth_header = create_auth_header("put", datestamp, "#{@azure_container}#{blob_path}", headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :put, :params => params, :headers=> headers)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :put, :params => params, :headers=> headers)
             request.on_complete do |response|
                 if response.success?
                     log.debug "azurestorage_gen2: Blob '#{blob_path}' has been created, response code: #{response.code}"
                 elsif response.timed_out?
-                    raise Fluent::UnrecoverableError,  "Creating blob '#{blob_path}' request timed out."
+                    raise_error  "Creating blob '#{blob_path}' request timed out."
                 elsif response.code == 409
                     log.debug "azurestorage_gen2: Blob already exists: #{blob_path}"
                 else
-                    raise Fluent::UnrecoverableError, "Creating blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
+                    raise_error "Creating blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
                 end
             end
             request.run
@@ -338,24 +333,24 @@ module Fluent::Plugin
 
         private
         def append_blob_block(blob_path, content, position)
-            log.debug "azurestorage_gen2: append_blob_block.start: Append blob ('#{blob_path}') called with position #{position}"
+            log.debug "azurestorage_gen2: append_blob_block.start: Append blob ('#{blob_path}') called with position #{position} (content length: #{content.length}, end position: #{position + content.length})"
             datestamp = create_request_date
             headers = {:"x-ms-version" =>  ABFS_API_VERSION,  :"x-ms-date" => datestamp, :"Content-Length" => content.length}
             params = {:action => "append", :position => "#{position}"}
             auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}", headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :patch, :headers=> headers, :params => params, :body => content)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :patch, :headers=> headers, :params => params, :body => content)
             request.on_complete do |response|
                 if response.success?
                     log.debug "azurestorage_gen2: Blob '#{blob_path}' has been appended, response code: #{response.code}"
                 elsif response.timed_out?
-                    raise Fluent::UnrecoverableError,  "Appending blob #{blob_path}' request timed out."
+                    raise_error  "Appending blob #{blob_path}' request timed out."
                 elsif response.code == 404
                     raise AppendBlobResponseError.new("Blob '#{blob_path}' has not found. Error code: #{response.code}", 404)
                 elsif response.code == 409
                     raise AppendBlobResponseError.new("Blob '#{blob_path}' has conflict. Error code: #{response.code}", 409)
                 else
-                    raise Fluent::UnrecoverableError, "Appending blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
+                    raise_error "Appending blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
                 end
             end
             request.run
@@ -369,14 +364,14 @@ module Fluent::Plugin
             params = {:action => "flush", :position => "#{position}"}
             auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}",headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :patch, :params => params, :headers=> headers)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :patch, :params => params, :headers=> headers)
             request.on_complete do |response|
                 if response.success?
                     log.debug "azurestorage_gen2: Blob '#{blob_path}' flush was successful, response code: #{response.code}"
                 elsif response.timed_out?
-                    raise Fluent::UnrecoverableError,  "Bloub '#{blob_path}' flush request timed out."
+                    raise_error  "Bloub '#{blob_path}' flush request timed out."
                 else
-                    raise Fluent::UnrecoverableError, "Blob flush request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
+                    raise_error "Blob flush request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
                 end
             end
             request.run
@@ -390,18 +385,18 @@ module Fluent::Plugin
             content_length = -1
             auth_header = create_auth_header("head", datestamp, "#{@azure_container}#{blob_path}", headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :head, :params => params, :headers=> headers)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :head, :params => params, :headers=> headers)
             request.on_complete do |response|
                 if response.success?
                   log.debug "azurestorage_gen2: Get blob properties for '#{blob_path}', response headers: #{response.headers}"
                   content_length = response.headers['Content-Length'].to_i
                 elsif response.timed_out?
-                    raise Fluent::UnrecoverableError,  "Get blob properties '#{blob_path}' request timed out."
+                    raise_error  "Get blob properties '#{blob_path}' request timed out."
                 elsif response.code == 404
                     log.debug "azurestorage_gen2: Blob '#{blob_path}' does not exist. Creating it if needed..."
                     content_length = 0
                 else
-                    raise Fluent::UnrecoverableError, "Get blob properties '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
+                    raise_error "Get blob properties '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
                 end
             end
             request.run
@@ -514,6 +509,15 @@ module Fluent::Plugin
             Time.now.strftime('%a, %e %b %y %H:%M:%S %Z')
         end
 
+        private
+        def raise_error(error_message)
+            if @enable_retry
+                raise BlobOperationError, error_message
+            else
+                raise Fluent::UnrecoverableError,  error_message
+            end
+        end
+
         def uuid_random
             require 'uuidtools'
             ::UUIDTools::UUID.random_create.to_s
@@ -630,4 +634,10 @@ module Fluent::Plugin
           super(message)
         end
     end
+
+    class BlobOperationError < StandardError
+        def initialize(message="Default message")
+            super(message)
+        end
+    end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-azurestorage-gen2
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.2.4
 platform: ruby
 authors:
 - Oliver Szabo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-03 00:00:00.000000000 Z
+date: 2020-01-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd