fluent-plugin-azurestorage-gen2 0.1.6 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2f30aa7c60fa869b86eb0d921aab12831c6355b6
-  data.tar.gz: 522010167e94cb9500e89b13aa68c4ec0bba1053
+  metadata.gz: c4b4636ae463e1d010ac8b6c069c1c9e47d226a9
+  data.tar.gz: b215a1aa8a4c8c9e768b38f1ee3a318b8d5f42a2
 SHA512:
-  metadata.gz: 5d40bc5c3964bd9187e6710dbda66f7e3556f310425120cf2c46da19a34c8905676278f4428d744a3255ce83e55bf8bdd2791dae755b25aefcf32d39e771c8b6
-  data.tar.gz: 149ee0c0ffc588e048fd0fa87cc343af153d6dc6de4132e9ace1d06f13b80e9d4f6ca60dba1892972693e6dfb37fc0dc649e233e538ccdcf982bbbc6540d1d1f
+  metadata.gz: 8c7eb7bd110a30963f4cc5b99303829d234b9285a7a29e4b9eb3115511e0c7870f06f94cf237b60ce8cd76e74a0c634a71341747b0991b5080f7dbab2f94ec3f
+  data.tar.gz: a02576ae303d449b2d2301c88d80e076792acf58d90b50fafa4822a3ab96581f1566ccb052cd153d42a323fb4563575b648271f89a9220d815ba1a3d5f953be2

data/README.md

@@ -9,7 +9,7 @@
 
 | fluent-plugin-azurestorage-gen2 | fluentd | ruby |
 |------------------------|---------|------|
-| >= 0.1.6 | >= v0.14.0 | >= 2.4 |
+| >= 0.2.0 | >= v0.14.0 | >= 2.4 |
 
 ## Overview
 
@@ -37,6 +37,7 @@ $ gem install fluent-plugin-azurestorage-gen2
   file_extension                   log
   path                             "/cluster-logs/myfolder/${tag[1]}-#{Socket.gethostname}-%M"
   auto_create_container            true
+  format                           single_value
   <buffer tag,time>
     @type file
     path /var/log/fluent/azurestorage-buffer
@@ -63,6 +64,7 @@ $ gem install fluent-plugin-azurestorage-gen2
   file_extension                   log
   path                             "/cluster-logs/myfolder/${tag[1]}-#{Socket.gethostname}-%M"
   auto_create_container            true
+  format                           single_value
   <buffer tag,time>
     @type file
     path /var/log/fluent/azurestorage-buffer
@@ -103,7 +105,7 @@ OAuth client secret that is used for OAuth based authentication. Required if OAu
 
 ### azure_oauth_refresh_interval
 
-OAuth2 access token refreshment interval in second. Only applies when MSI / OAuth authentication is used.
+OAuth2 access token refreshment interval in second. Only applies when MSI / OAuth authentication is used. (default value is 59 minutes)
 
 ### azure_oauth_use_azure_cli (Preview)
 
@@ -173,13 +175,13 @@ azure_object_key_format %{path}/events/ts=%{time_slice}/events_%{index}-%{hostna
 
 ### file_extension
 
-File extension for the uploaded files. Only uses if `store_as` is not set, or set as `none`
+File extension for the uploaded files. Only used if `store_as` is not set, or set as `none`
 
 ### store_as
 
 Archive format on Azure Storage. You can use following types:
 
-- none (default - no tmp file creation for log processing)
+- none (default - no tmp file creation for log processing, use with json or single value format)
 - gzip
 - json
 - text

data/VERSION

@@ -1 +1 @@
-0.1.6
+0.2.0

data/lib/fluent/plugin/out_azurestorage_gen2.rb

@@ -27,7 +27,7 @@ module Fluent::Plugin
         config_param :azure_oauth_secret, :string, :default => nil, :secret => true
         config_param :azure_oauth_tenant_id, :string, :default => nil
         config_param :azure_oauth_use_azure_cli, :bool, :default => false
-        config_param :azure_oauth_refresh_interval, :integer, :default => 60 * 60 # one hour
+        config_param :azure_oauth_refresh_interval, :integer, :default => 60 * 59 # 59 minutes, do not wait one hour
         config_param :azure_container, :string, :default => nil
         config_param :azure_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
         config_param :file_extension, :string, :default => "log"
@@ -37,7 +37,6 @@ module Fluent::Plugin
         config_param :format, :string, :default => "out_file"
         config_param :time_slice_format, :string, :default => '%Y%m%d'
         config_param :command_parameter, :string, :default => nil
-        config_param :message_field, :string, :default => nil
 
         DEFAULT_FORMAT_TYPE = "out_file"
         URL_DOMAIN_SUFFIX = '.dfs.core.windows.net'
@@ -118,21 +117,14 @@ module Fluent::Plugin
         def write(chunk)
             metadata = chunk.metadata
             if @store_as.nil? || @store_as == "none"
-                raw_data=''
                 generate_log_name(metadata, @current_index)
                 if @last_azure_storage_path != @azure_storage_path
                     @current_index = 0
                     generate_log_name(metadata, @current_index)
                 end
-                chunk.each do |emit_time, record|
-                    if @message_field.nil? || @message_field.empty?
-                        raw_data << "#{Yajl.dump(record)}\n"
-                    elsif record.key?(@message_field)
-                        line = record[@message_field].chomp
-                        raw_data << "#{line}\n"
-                    end
-                end
+                raw_data = chunk.read
                 unless raw_data.empty?
+                    log.debug "azurestorage_gen2: processing raw data", chunk_id: dump_unique_id_hex(chunk.unique_id)
                     upload_blob(raw_data, metadata)
                 end
                 @last_azure_storage_path = @azure_storage_path

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-azurestorage-gen2
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.2.0
 platform: ruby
 authors:
 - Oliver Szabo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-03 00:00:00.000000000 Z
+date: 2020-01-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd