fluent-plugin-azurestorage-gen2 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 511b8f199ba0f1c6b7937711b0e3d146baf3a856
-  data.tar.gz: ff6cf330c0ecacd65a48c8f312a87bb8bdb402a5
+  metadata.gz: b3bcdaa488c738cafda86bcae69dbc0ac6d8521e
+  data.tar.gz: d0f9bd8a6a2ae52dea8e57570e112d72cdd8e05a
 SHA512:
-  metadata.gz: 368340200f930c8f4984ab5b630e779f375061d82faab35cf55f1ae83f7b92e58f63f7a4a152880a51d62840a1fe9e4bc56ffacdb3a6c1930c5a617d11378eae
-  data.tar.gz: e0a54bb0bfd963134cf7e1afeb80c8ea936e3421002d444d9cf6abbaaafc8d2e1bcedcbb118afbe0a2a44a76c9d9ebfe207495af11cb0c19a2f63621cc2c83ab
+  metadata.gz: 1529a5bda90bedf0c4f6a03ce89d3372fd736c844f426d2d6919aef2a776765fb8ffa1305fda0304179e99eb13cc8746008cf2461a9d997d05a4519c03579a18
+  data.tar.gz: 48843accc54dca373c08efa02f72ec4ee1142d07003af82ed00548f5b1767568770b91fa1549633836feddad28a1733a915065cb81d157c1a680975189573075

data/README.md

@@ -3,17 +3,17 @@
 [![Build Status](https://travis-ci.org/oleewere/fluent-plugin-azurestorage-gen2.svg?branch=master)](https://travis-ci.org/oleewere/fluent-plugin-azurestorage-gen2)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 [![Gem Version](https://badge.fury.io/rb/fluent-plugin-azurestorage-gen2.svg)](http://badge.fury.io/rb/fluent-plugin-azurestorage-gen2)
-![](https://ruby-gem-downloads-badge.herokuapp.com/fluent-plugin-azurestorage-gen2?extension=png)
+![](https://ruby-gem-downloads-badge.herokuapp.com/fluent-plugin-azurestorage-gen2?type=total)
 
 ## Requirements
 
 | fluent-plugin-azurestorage-gen2 | fluentd | ruby |
 |------------------------|---------|------|
-| >= 0.1.2 | >= v0.14.0 | >= 2.4 |
+| >= 0.1.3 | >= v0.14.0 | >= 2.4 |
 
 ## Overview
 
-Fluent output plugin that can use ABFS api and append blobs with MSI and OAuth support.
+Fluent output plugin that can use ABFS api and append blobs with MSI and OAuth support. This plugin an extension of [fluent-plugin-azurestorage](https://github.com/htgc/fluent-plugin-azurestorage), but re-implemented against the Azure Storage Gen2 API with Typhoeus.
 
 ## Installation
 
@@ -93,18 +93,22 @@ Your Azure Managed Service Identity ID. When storage key authentication is not u
 
 Azure account tenant id from your Azure Directory. Required if OAuth based credential mechanism is used.
 
-### azure_oauth_app_id
+### azure_oauth_app_id (Preview)
 
 OAuth client id that is used for OAuth based authentication. Required if OAuth based credential mechanism is used.
 
-### azure_oauth_secret
+### azure_oauth_secret (Preview)
 
 OAuth client secret that is used for OAuth based authentication. Required if OAuth based credential mechanism is used.
 
-### azure_oauth_refresh_interval
+### azure_oauth_refresh_interval (Preview)
 
 OAuth2 access token refreshment interval in second. Only applies when MSI / OAuth authentication is used.
 
+### azure_oauth_use_azure_cli (Preview)
+
+Use az (azure cli) for get the access token, use only if the cli is installed on the host where fluentd runs. With this option you do not require to use `azure_oauth_app_id`, `azure_oauth_secret` or `azure_oauth_tenant_id`
+
 ### azure_container (Required)
 
 Azure Storage Container name
@@ -163,6 +167,21 @@ The [fluent-mixin-config-placeholders](https://github.com/tagomoris/fluent-mixin
 azure_object_key_format %{path}/events/ts=%{time_slice}/events_%{index}-%{hostname}.%{file_extension}
 ```
 
+### file_extension
+
+File extension for the uploaded files. Only uses if `store_as` is not set, or set as `none`
+
+### store_as
+
+Archive format on Azure Storage. You can use following types:
+
+- none (default - no tmp file creation for log processing)
+- gzip
+- json
+- text
+- lzo (Need lzop command)
+- lzma2 (Need xz command)
+
 ### format
 
 Change one line format in the Azure Storage object. Supported formats are 'out_file', 'json', 'ltsv' and 'single_value'.

data/VERSION

@@ -1 +1 @@
-0.1.2
+0.1.3

data/lib/fluent/plugin/out_azurestorage_gen2.rb

@@ -26,6 +26,7 @@ module Fluent::Plugin
         config_param :azure_oauth_app_id, :string, :default => nil, :secret => true
         config_param :azure_oauth_secret, :string, :default => nil, :secret => true
         config_param :azure_oauth_tenant_id, :string, :default => nil
+        config_param :azure_oauth_use_azure_cli, :bool, :default => false
         config_param :azure_oauth_refresh_interval, :integer, :default => 60 * 60 # one hour
         config_param :azure_container, :string, :default => nil
         config_param :azure_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
@@ -208,8 +209,10 @@ module Fluent::Plugin
                 acquire_access_token_msi
             elsif !@azure_oauth_app_id.nil? and !@azure_oauth_secret.nil? and !@azure_oauth_tenant_id.nil?
                 acquire_access_token_oauth_app
+            elsif @azure_oauth_use_azure_cli
+                acquire_access_token_by_az
             else
-                raise Fluent::UnrecoverableError, "Using MSI or simple OAuth 2.0 based authentication parameters (azure_oauth_tenant_id, azure_oauth_app_id, azure_oauth_secret) are required."
+                raise Fluent::UnrecoverableError, "Using MSI or 'az cli tool' or simple OAuth 2.0 based authentication parameters (azure_oauth_tenant_id, azure_oauth_app_id, azure_oauth_secret) are required."
             end
         end
 
@@ -237,8 +240,9 @@ module Fluent::Plugin
         private
         def acquire_access_token_oauth_app
             params = { :"api-version" => ACCESS_TOKEN_API_VERSION, :resource => "https://storage.azure.com/"}
+            headers = {:"Content-Type" => "application/x-www-form-urlencoded"}
             content = "grant_type=client_credentials&client_id=#{@azure_oauth_app_id}&client_secret=#{@azure_oauth_secret}&resource=https://storage.azure.com/"
-            request = Typhoeus::Request.new("https://login.microsoftonline.com/#{@azure_oauth_tenant_id}/oauth2/token", params: params, :body => content)
+            request = Typhoeus::Request.new("https://login.microsoftonline.com/#{@azure_oauth_tenant_id}/oauth2/token", :body => content, :headers => headers, verbose: true)
             request.on_complete do |response|
                 if response.success?
                   data = JSON.parse(response.body)
@@ -251,6 +255,12 @@ module Fluent::Plugin
             request.run
         end
 
+        private
+        def acquire_access_token_by_az
+            access_token=`az account get-access-token --resource https://storage.azure.com/ --query accessToken -o tsv`
+            log.debug "azurestorage_gen2: Token response: #{access_token}"
+            @azure_access_token = access_token
+
         private
         def ensure_container
             datestamp = create_request_date
@@ -324,7 +334,7 @@ module Fluent::Plugin
         def append_blob_block(blob_path, content, position)
             log.debug "azurestorage_gen2: append_blob_block.start: Append blob ('#{blob_path}') called with position #{position}"
             datestamp = create_request_date
-            headers = {:"x-ms-version" =>  ABFS_API_VERSION,  :"x-ms-date" => datestamp, :"x-ms-content-type" => "#{@blob_content_type}"}
+            headers = {:"x-ms-version" =>  ABFS_API_VERSION,  :"x-ms-date" => datestamp, :"x-ms-content-type" => "#{@blob_content_type}", :"Content-Length" => content.length}
             params = {:action => "append", :position => "#{position}"}
             auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}", headers, params)
             headers[:Authorization] = auth_header
@@ -349,7 +359,7 @@ module Fluent::Plugin
         def flush(blob_path, position)
             log.debug "azurestorage_gen2: flush_blob.start: Flush blob ('#{blob_path}') called with position #{position}"
             datestamp = create_request_date
-            headers = {:"x-ms-version" =>  ABFS_API_VERSION, :"x-ms-date" => datestamp,:"Content-Length" => "0"}
+            headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp, :"Content-Length" => "0"}
             params = {:action => "flush", :position => "#{position}"}
             auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}",headers, params)
             headers[:Authorization] = auth_header

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-azurestorage-gen2
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Oliver Szabo