fluent-plugin-azurestorage-gen2 0.1.2 → 0.1.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +25 -6
- data/VERSION +1 -1
- data/lib/fluent/plugin/out_azurestorage_gen2.rb +14 -4
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b3bcdaa488c738cafda86bcae69dbc0ac6d8521e
|
4
|
+
data.tar.gz: d0f9bd8a6a2ae52dea8e57570e112d72cdd8e05a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1529a5bda90bedf0c4f6a03ce89d3372fd736c844f426d2d6919aef2a776765fb8ffa1305fda0304179e99eb13cc8746008cf2461a9d997d05a4519c03579a18
|
7
|
+
data.tar.gz: 48843accc54dca373c08efa02f72ec4ee1142d07003af82ed00548f5b1767568770b91fa1549633836feddad28a1733a915065cb81d157c1a680975189573075
|
data/README.md
CHANGED
@@ -3,17 +3,17 @@
|
|
3
3
|
[![Build Status](https://travis-ci.org/oleewere/fluent-plugin-azurestorage-gen2.svg?branch=master)](https://travis-ci.org/oleewere/fluent-plugin-azurestorage-gen2)
|
4
4
|
[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
|
5
5
|
[![Gem Version](https://badge.fury.io/rb/fluent-plugin-azurestorage-gen2.svg)](http://badge.fury.io/rb/fluent-plugin-azurestorage-gen2)
|
6
|
-
![](https://ruby-gem-downloads-badge.herokuapp.com/fluent-plugin-azurestorage-gen2?
|
6
|
+
![](https://ruby-gem-downloads-badge.herokuapp.com/fluent-plugin-azurestorage-gen2?type=total)
|
7
7
|
|
8
8
|
## Requirements
|
9
9
|
|
10
10
|
| fluent-plugin-azurestorage-gen2 | fluentd | ruby |
|
11
11
|
|------------------------|---------|------|
|
12
|
-
| >= 0.1.
|
12
|
+
| >= 0.1.3 | >= v0.14.0 | >= 2.4 |
|
13
13
|
|
14
14
|
## Overview
|
15
15
|
|
16
|
-
Fluent output plugin that can use ABFS api and append blobs with MSI and OAuth support.
|
16
|
+
Fluent output plugin that can use ABFS api and append blobs with MSI and OAuth support. This plugin an extension of [fluent-plugin-azurestorage](https://github.com/htgc/fluent-plugin-azurestorage), but re-implemented against the Azure Storage Gen2 API with Typhoeus.
|
17
17
|
|
18
18
|
## Installation
|
19
19
|
|
@@ -93,18 +93,22 @@ Your Azure Managed Service Identity ID. When storage key authentication is not u
|
|
93
93
|
|
94
94
|
Azure account tenant id from your Azure Directory. Required if OAuth based credential mechanism is used.
|
95
95
|
|
96
|
-
### azure_oauth_app_id
|
96
|
+
### azure_oauth_app_id (Preview)
|
97
97
|
|
98
98
|
OAuth client id that is used for OAuth based authentication. Required if OAuth based credential mechanism is used.
|
99
99
|
|
100
|
-
### azure_oauth_secret
|
100
|
+
### azure_oauth_secret (Preview)
|
101
101
|
|
102
102
|
OAuth client secret that is used for OAuth based authentication. Required if OAuth based credential mechanism is used.
|
103
103
|
|
104
|
-
### azure_oauth_refresh_interval
|
104
|
+
### azure_oauth_refresh_interval (Preview)
|
105
105
|
|
106
106
|
OAuth2 access token refreshment interval in second. Only applies when MSI / OAuth authentication is used.
|
107
107
|
|
108
|
+
### azure_oauth_use_azure_cli (Preview)
|
109
|
+
|
110
|
+
Use az (azure cli) for get the access token, use only if the cli is installed on the host where fluentd runs. With this option you do not require to use `azure_oauth_app_id`, `azure_oauth_secret` or `azure_oauth_tenant_id`
|
111
|
+
|
108
112
|
### azure_container (Required)
|
109
113
|
|
110
114
|
Azure Storage Container name
|
@@ -163,6 +167,21 @@ The [fluent-mixin-config-placeholders](https://github.com/tagomoris/fluent-mixin
|
|
163
167
|
azure_object_key_format %{path}/events/ts=%{time_slice}/events_%{index}-%{hostname}.%{file_extension}
|
164
168
|
```
|
165
169
|
|
170
|
+
### file_extension
|
171
|
+
|
172
|
+
File extension for the uploaded files. Only uses if `store_as` is not set, or set as `none`
|
173
|
+
|
174
|
+
### store_as
|
175
|
+
|
176
|
+
Archive format on Azure Storage. You can use following types:
|
177
|
+
|
178
|
+
- none (default - no tmp file creation for log processing)
|
179
|
+
- gzip
|
180
|
+
- json
|
181
|
+
- text
|
182
|
+
- lzo (Need lzop command)
|
183
|
+
- lzma2 (Need xz command)
|
184
|
+
|
166
185
|
### format
|
167
186
|
|
168
187
|
Change one line format in the Azure Storage object. Supported formats are 'out_file', 'json', 'ltsv' and 'single_value'.
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.1.
|
1
|
+
0.1.3
|
@@ -26,6 +26,7 @@ module Fluent::Plugin
|
|
26
26
|
config_param :azure_oauth_app_id, :string, :default => nil, :secret => true
|
27
27
|
config_param :azure_oauth_secret, :string, :default => nil, :secret => true
|
28
28
|
config_param :azure_oauth_tenant_id, :string, :default => nil
|
29
|
+
config_param :azure_oauth_use_azure_cli, :bool, :default => false
|
29
30
|
config_param :azure_oauth_refresh_interval, :integer, :default => 60 * 60 # one hour
|
30
31
|
config_param :azure_container, :string, :default => nil
|
31
32
|
config_param :azure_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
|
@@ -208,8 +209,10 @@ module Fluent::Plugin
|
|
208
209
|
acquire_access_token_msi
|
209
210
|
elsif !@azure_oauth_app_id.nil? and !@azure_oauth_secret.nil? and !@azure_oauth_tenant_id.nil?
|
210
211
|
acquire_access_token_oauth_app
|
212
|
+
elsif @azure_oauth_use_azure_cli
|
213
|
+
acquire_access_token_by_az
|
211
214
|
else
|
212
|
-
raise Fluent::UnrecoverableError, "Using MSI or simple OAuth 2.0 based authentication parameters (azure_oauth_tenant_id, azure_oauth_app_id, azure_oauth_secret) are required."
|
215
|
+
raise Fluent::UnrecoverableError, "Using MSI or 'az cli tool' or simple OAuth 2.0 based authentication parameters (azure_oauth_tenant_id, azure_oauth_app_id, azure_oauth_secret) are required."
|
213
216
|
end
|
214
217
|
end
|
215
218
|
|
@@ -237,8 +240,9 @@ module Fluent::Plugin
|
|
237
240
|
private
|
238
241
|
def acquire_access_token_oauth_app
|
239
242
|
params = { :"api-version" => ACCESS_TOKEN_API_VERSION, :resource => "https://storage.azure.com/"}
|
243
|
+
headers = {:"Content-Type" => "application/x-www-form-urlencoded"}
|
240
244
|
content = "grant_type=client_credentials&client_id=#{@azure_oauth_app_id}&client_secret=#{@azure_oauth_secret}&resource=https://storage.azure.com/"
|
241
|
-
request = Typhoeus::Request.new("https://login.microsoftonline.com/#{@azure_oauth_tenant_id}/oauth2/token",
|
245
|
+
request = Typhoeus::Request.new("https://login.microsoftonline.com/#{@azure_oauth_tenant_id}/oauth2/token", :body => content, :headers => headers, verbose: true)
|
242
246
|
request.on_complete do |response|
|
243
247
|
if response.success?
|
244
248
|
data = JSON.parse(response.body)
|
@@ -251,6 +255,12 @@ module Fluent::Plugin
|
|
251
255
|
request.run
|
252
256
|
end
|
253
257
|
|
258
|
+
private
|
259
|
+
def acquire_access_token_by_az
|
260
|
+
access_token=`az account get-access-token --resource https://storage.azure.com/ --query accessToken -o tsv`
|
261
|
+
log.debug "azurestorage_gen2: Token response: #{access_token}"
|
262
|
+
@azure_access_token = access_token
|
263
|
+
|
254
264
|
private
|
255
265
|
def ensure_container
|
256
266
|
datestamp = create_request_date
|
@@ -324,7 +334,7 @@ module Fluent::Plugin
|
|
324
334
|
def append_blob_block(blob_path, content, position)
|
325
335
|
log.debug "azurestorage_gen2: append_blob_block.start: Append blob ('#{blob_path}') called with position #{position}"
|
326
336
|
datestamp = create_request_date
|
327
|
-
headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp, :"x-ms-content-type" => "#{@blob_content_type}"}
|
337
|
+
headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp, :"x-ms-content-type" => "#{@blob_content_type}", :"Content-Length" => content.length}
|
328
338
|
params = {:action => "append", :position => "#{position}"}
|
329
339
|
auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}", headers, params)
|
330
340
|
headers[:Authorization] = auth_header
|
@@ -349,7 +359,7 @@ module Fluent::Plugin
|
|
349
359
|
def flush(blob_path, position)
|
350
360
|
log.debug "azurestorage_gen2: flush_blob.start: Flush blob ('#{blob_path}') called with position #{position}"
|
351
361
|
datestamp = create_request_date
|
352
|
-
headers = {:"x-ms-version" =>
|
362
|
+
headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp, :"Content-Length" => "0"}
|
353
363
|
params = {:action => "flush", :position => "#{position}"}
|
354
364
|
auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}",headers, params)
|
355
365
|
headers[:Authorization] = auth_header
|