fluent-plugin-azurestorage-gen2 0.2.8 → 0.2.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 46b6cf60479daf8f64e5196ad36332a11591220e
-  data.tar.gz: 09aec8214be51d7be9ce4b2e7350e145d4b0fb51
+  metadata.gz: 4bff097a151b5af0d596f82ab181d3e141abf0de
+  data.tar.gz: f453dabd8d7719502820e51ae870803d3845a80f
 SHA512:
-  metadata.gz: a3f4d5c6245c11e5f8b4b33d26825a3d9f93796d553a1ac1d55c931e891762467c22abc228f546d7c2efdf107c3cc471d47dc6bc0114bd1f5c1dced3a6b30bae
-  data.tar.gz: 67b751890b5aee2086d6d617b41d4da6883d8cfa9e77acc2ac7de54f58e457725cb718606a65ec385972fed1a24518cc9d981f15a454393b1e08b31b662735b9
+  metadata.gz: 361a92e5ab216018f95a1f0f0e331d0401bf894da0b65ea3a2bcd1ecfb62e3ce3fd599b8633e77ae385537e9590d2cffa51ec4df38917e5f42660932edc87350
+  data.tar.gz: da8cec0e4867005f2e794c1733c58bfcb33c5ef7e2a6f071f93436506e2d70c3b9a8ecd755ad305973af16bbc9ee163d62d50e632a7d7d26a80964de4d844cc0

data/VERSION

@@ -1 +1 @@
-0.2.8
+0.2.9

data/lib/fluent/plugin/out_azurestorage_gen2.rb

@@ -41,6 +41,7 @@ module Fluent::Plugin
         config_param :url_domain_suffix, :string, :default => '.dfs.core.windows.net'
         config_param :format, :string, :default => "out_file"
         config_param :time_slice_format, :string, :default => '%Y%m%d'
+        config_param :hex_random_length, :integer, default: 4
         config_param :command_parameter, :string, :default => nil
 
         DEFAULT_FORMAT_TYPE = "out_file"
@@ -75,16 +76,6 @@ module Fluent::Plugin
 
             @formatter = formatter_create
       
-            if @localtime
-              @path_slicer = Proc.new {|path|
-                Time.now.strftime(path)
-              }
-            else
-              @path_slicer = Proc.new {|path|
-                Time.now.utc.strftime(path)
-              }
-            end
-      
             if @azure_container.nil?
               raise Fluent::ConfigError, "azure_container is needed"
             end
@@ -98,7 +89,7 @@ module Fluent::Plugin
             else
                 @final_file_extension = @compressor.ext
             end
-
+            @values_for_object_chunk = {}
         end
 
         def multi_workers_ready?
@@ -127,17 +118,16 @@ module Fluent::Plugin
         end
 
         def write(chunk)
-            metadata = chunk.metadata
             if @store_as.nil? || @store_as == "none"
-                generate_log_name(metadata, @current_index)
+                generate_log_name(chunk, @current_index)
                 if @last_azure_storage_path != @azure_storage_path
                     @current_index = 0
-                    generate_log_name(metadata, @current_index)
+                    generate_log_name(chunk, @current_index)
                 end
                 raw_data = chunk.read
                 unless raw_data.empty?
                     log.debug "azurestorage_gen2: processing raw data", chunk_id: dump_unique_id_hex(chunk.unique_id)
-                    upload_blob(raw_data, metadata)
+                    upload_blob(raw_data, chunk)
                 end
                 chunk.close rescue nil
                 @last_azure_storage_path = @azure_storage_path
@@ -147,51 +137,68 @@ module Fluent::Plugin
                 begin
                     @compressor.compress(chunk, tmp)
                     tmp.rewind
-                    generate_log_name(metadata, @current_index)
+                    generate_log_name(chunk, @current_index)
                     if @last_azure_storage_path != @azure_storage_path
                         @current_index = 0
-                        generate_log_name(metadata, @current_index)
+                        generate_log_name(chunk, @current_index)
                     end
                     log.debug "azurestorage_gen2: Start uploading temp file: #{tmp.path}"
                     content = File.open(tmp.path, 'rb') { |file| file.read }
-                    upload_blob(content, metadata)
+                    upload_blob(content, chunk)
                     @last_azure_storage_path = @azure_storage_path
                 ensure
                     tmp.close(true) rescue nil
                 end
+                @values_for_object_chunk.delete(chunk.unique_id)
             end
 
         end
 
         private
-        def upload_blob(content, metadata)
+        def upload_blob(content, chunk)
             log.debug "azurestorage_gen2: Uploading blob: #{@azure_storage_path}"
             existing_content_length = get_blob_properties(@azure_storage_path)
             if existing_content_length == 0
                 create_blob(@azure_storage_path)
             end
-            append_blob(content, metadata, existing_content_length)
+            append_blob(content, chunk, existing_content_length)
         end
 
         private
-        def generate_log_name(metadata, index)
+        def generate_log_name(chunk, index)
+            metadata = chunk.metadata
             time_slice = if metadata.timekey.nil?
                        ''.freeze
                      else
                        Time.at(metadata.timekey).utc.strftime(@time_slice_format)
                      end
-            path = @path_slicer.call(@path)
-            values_for_object_key = {
-                "%{path}" => path,
-                "%{time_slice}" => time_slice,
+            if @localtime
+                hms_slicer = Time.now.strftime("%H%M%S")
+            else
+                hms_slicer = Time.now.utc.strftime("%H%M%S")
+            end
+
+            @values_for_object_chunk[chunk.unique_id] ||= {
+                "%{hex_random}" => hex_random(chunk),
+            }
+            values_for_object_key_pre = {
+                "%{path}" => @path,
                 "%{index}" => index,
                 "%{uuid_flush}" => uuid_random,
                 "%{file_extension}" => @final_file_extension
             }
-            storage_path = @azure_object_key_format.gsub(%r(%{[^}]+}), values_for_object_key)
-            extracted_path = extract_placeholders(storage_path, metadata)
-            extracted_path = "/" + extracted_path unless extracted_path.start_with?("/")
-            @azure_storage_path = extracted_path
+            values_for_object_key_post = {
+                "%{date_slice}" => time_slice,
+                "%{time_slice}" => time_slice,
+                "%{hms_slice}" => hms_slicer,
+            }.merge!(@values_for_object_chunk[chunk.unique_id])
+            storage_path = @azure_object_key_format.gsub(%r(%{[^}]+})) do |matched_key|
+                values_for_object_key_pre.fetch(matched_key, matched_key)
+            end
+            storage_path = extract_placeholders(storage_path, chunk)
+            storage_path = storage_path.gsub(%r(%{[^}]+}), values_for_object_key_post)
+            storage_path = "/" + storage_path unless storage_path.start_with?("/")
+            @azure_storage_path = storage_path
         end
 
         def setup_access_token
@@ -427,7 +434,7 @@ module Fluent::Plugin
         end
 
         private
-        def append_blob(content, metadata, existing_content_length)
+        def append_blob(content, chunk, existing_content_length)
           position = 0
           log.debug "azurestorage_gen2: append_blob.start: Content size: #{content.length}"
           loop do
@@ -545,6 +552,12 @@ module Fluent::Plugin
             require 'uuidtools'
             ::UUIDTools::UUID.random_create.to_s
         end
+
+        def hex_random(chunk)
+            unique_hex = Fluent::UniqueId.hex(chunk.unique_id)
+            unique_hex.reverse!
+            unique_hex[0...@hex_random_length]
+        end
         
         def timekey_to_timeformat(timekey)
             case timekey

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-azurestorage-gen2
 version: !ruby/object:Gem::Version
-  version: 0.2.8
+  version: 0.2.9
 platform: ruby
 authors:
 - Oliver Szabo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-21 00:00:00.000000000 Z
+date: 2020-02-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd