fluent-plugin-azurestorage-gen2 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 13b891a2c06d9a6a18d088c8cf02eb781d3538eb
-  data.tar.gz: cc89a73ec41eed32b5a7f3a4d104c3ac80422642
+  metadata.gz: c1f459cce5a616b607bcd07b2a0e60acb2b7f0e2
+  data.tar.gz: 6c162b6991996a2fb61f7f3f8af87a3e74454b6f
 SHA512:
-  metadata.gz: da9ade031001af9fea91952af3da3fea6fa32dac029b4b2ae84cb9ac92c2b68e2fd7f9232380ec38fd96990ad53b07d9a16ec11699c12ed4b3e2978a72c03364
-  data.tar.gz: 76d5f0f943d2647b388852d4c4d274c0af6c8cfae9b3440afaed6b5cb4db240106efe106e974cb8ad744fe7f3ecb4deee143271e42ef67f2f9db184427b96143
+  metadata.gz: 75e25084709b908aa16411d04262e828c52bbe367cc11d250ab8c9b7575cf819134b69aa0dc15c6c9ba8d902212c40bb79336ce9661378d4fc2968d87020f112
+  data.tar.gz: 460b141c22880446e7a6a15990f5c7d753646c372aefb41a7b13ecb6fb270fe9fe44528cb985c54fde282adba39179ca42a86bbd6713b530d66703c6ba3e425f

data/README.md

@@ -9,7 +9,7 @@
 
 | fluent-plugin-azurestorage-gen2 | fluentd | ruby |
 |------------------------|---------|------|
-| >= 0.1.3 | >= v0.14.0 | >= 2.4 |
+| >= 0.1.5 | >= v0.14.0 | >= 2.4 |
 
 ## Overview
 
@@ -117,6 +117,10 @@ Azure Storage Container name
 
 This plugin create container if not exist when you set 'auto_create_container' to true.
 
+### skip_container_check
+
+You can skip the initial container listing (and container creation) operations at startup. That can be useful if the user is not allowed to perform these operations.
+
 ### azure_object_key_format
 
 The format of Azure Storage object keys. You can use several built-in variables:

data/VERSION

@@ -1 +1 @@
-0.1.4
+0.1.5

data/lib/fluent/plugin/out_azurestorage_gen2.rb

@@ -33,6 +33,7 @@ module Fluent::Plugin
         config_param :file_extension, :string, :default => "log"
         config_param :store_as, :string, :default => "none"
         config_param :auto_create_container, :bool, :default => false
+        config_param :skip_container_check, :bool, :default => false
         config_param :format, :string, :default => "out_file"
         config_param :time_slice_format, :string, :default => '%Y%m%d'
         config_param :command_parameter, :string, :default => nil
@@ -57,13 +58,17 @@ module Fluent::Plugin
             compat_parameters_convert(conf, :buffer, :formatter, :inject)
             super
 
-            begin
-                @compressor = COMPRESSOR_REGISTRY.lookup(@store_as).new(:buffer_type => @buffer_type, :log => log)
-            rescue => e
-                log.warn "#{@store_as} not found. Use 'text' instead"
-                @compressor = TextCompressor.new
+            if @store_as.nil? || @store_as == "none"
+                log.info "azurestorage_gen2: Compression is disabled (store_as: #{@store_as})"
+            else
+                begin
+                    @compressor = COMPRESSOR_REGISTRY.lookup(@store_as).new(:buffer_type => @buffer_type, :log => log)
+                rescue => e
+                    log.warn "#{@store_as} not found. Use 'text' instead"
+                    @compressor = TextCompressor.new
+                end
+                @compressor.configure(conf)
             end
-            @compressor.configure(conf)
 
             @formatter = formatter_create
       
@@ -84,11 +89,15 @@ module Fluent::Plugin
             @azure_storage_path = ''
             @last_azure_storage_path = ''
             @current_index = 0
-            if @store_as.nil? || @store_as == "none"
+            if @store_as.nil? || @store_as == "none" || @store_as == "text"
                 @blob_content_type = "text/plain"
+            else
+                @blob_content_type = "application/octet-stream"
+            end
+
+            if @store_as.nil? || @store_as == "none"
                 @final_file_extension = @file_extension
             else
-                @blob_content_type = @compressor.content_type
                 @final_file_extension = @compressor.ext
             end
 
@@ -100,10 +109,17 @@ module Fluent::Plugin
 
         def start
             setup_access_token
-            ensure_container
+            if !@skip_container_check
+                ensure_container
+            end
             super
         end
 
+        def format(tag, time, record)
+            r = inject_values_to_record(tag, time, record)
+            @formatter.format(tag, time, r)
+        end
+
         def write(chunk)
             metadata = chunk.metadata
             if @store_as.nil? || @store_as == "none"
@@ -121,7 +137,6 @@ module Fluent::Plugin
                         raw_data << "#{line}\n"
                     end
                 end
-                raw_data = raw_data.chomp
                 unless raw_data.empty?
                     upload_blob(raw_data, metadata)
                 end
@@ -157,11 +172,6 @@ module Fluent::Plugin
             append_blob(content, metadata, existing_content_length)
         end
 
-        def format(tag, time, record)
-            r = inject_values_to_record(tag, time, record)
-            @formatter.format(tag, time, r)
-        end
-
         private
         def generate_log_name(metadata, index)
             time_slice = if metadata.timekey.nil?
@@ -242,7 +252,7 @@ module Fluent::Plugin
             params = { :"api-version" => ACCESS_TOKEN_API_VERSION, :resource => "https://storage.azure.com/"}
             headers = {:"Content-Type" => "application/x-www-form-urlencoded"}
             content = "grant_type=client_credentials&client_id=#{@azure_oauth_app_id}&client_secret=#{@azure_oauth_secret}&resource=https://storage.azure.com/"
-            request = Typhoeus::Request.new("https://login.microsoftonline.com/#{@azure_oauth_tenant_id}/oauth2/token", :body => content, :headers => headers, verbose: true)
+            request = Typhoeus::Request.new("https://login.microsoftonline.com/#{@azure_oauth_tenant_id}/oauth2/token", :body => content, :headers => headers)
             request.on_complete do |response|
                 if response.success?
                   data = JSON.parse(response.body)
@@ -312,7 +322,7 @@ module Fluent::Plugin
         private
         def create_blob(blob_path)
             datestamp = create_request_date
-            headers = {:"x-ms-version" =>  ABFS_API_VERSION, :"x-ms-date" => datestamp,:"Content-Length" => "0", :"Content-Type" => "application/json"}
+            headers = {:"x-ms-version" =>  ABFS_API_VERSION, :"x-ms-date" => datestamp,:"Content-Length" => "0", :"Content-Type" => "#{@blob_content_type}"}
             params = {:resource => "file", :recursive => "false"}
             auth_header = create_auth_header("put", datestamp, "#{@azure_container}#{blob_path}", headers, params)
             headers[:Authorization] = auth_header
@@ -335,7 +345,7 @@ module Fluent::Plugin
         def append_blob_block(blob_path, content, position)
             log.debug "azurestorage_gen2: append_blob_block.start: Append blob ('#{blob_path}') called with position #{position}"
             datestamp = create_request_date
-            headers = {:"x-ms-version" =>  ABFS_API_VERSION,  :"x-ms-date" => datestamp, :"x-ms-content-type" => "#{@blob_content_type}", :"Content-Length" => content.length}
+            headers = {:"x-ms-version" =>  ABFS_API_VERSION,  :"x-ms-date" => datestamp, :"Content-Type" => "#{@blob_content_type}", :"Content-Length" => content.length}
             params = {:action => "append", :position => "#{position}"}
             auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}", headers, params)
             headers[:Authorization] = auth_header

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-azurestorage-gen2
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - Oliver Szabo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-12-23 00:00:00.000000000 Z
+date: 2020-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd