fluent-plugin-azurestorage-gen2 0.1.4 → 0.1.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +5 -1
- data/VERSION +1 -1
- data/lib/fluent/plugin/out_azurestorage_gen2.rb +28 -18
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c1f459cce5a616b607bcd07b2a0e60acb2b7f0e2
|
4
|
+
data.tar.gz: 6c162b6991996a2fb61f7f3f8af87a3e74454b6f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 75e25084709b908aa16411d04262e828c52bbe367cc11d250ab8c9b7575cf819134b69aa0dc15c6c9ba8d902212c40bb79336ce9661378d4fc2968d87020f112
|
7
|
+
data.tar.gz: 460b141c22880446e7a6a15990f5c7d753646c372aefb41a7b13ecb6fb270fe9fe44528cb985c54fde282adba39179ca42a86bbd6713b530d66703c6ba3e425f
|
data/README.md
CHANGED
@@ -9,7 +9,7 @@
|
|
9
9
|
|
10
10
|
| fluent-plugin-azurestorage-gen2 | fluentd | ruby |
|
11
11
|
|------------------------|---------|------|
|
12
|
-
| >= 0.1.
|
12
|
+
| >= 0.1.5 | >= v0.14.0 | >= 2.4 |
|
13
13
|
|
14
14
|
## Overview
|
15
15
|
|
@@ -117,6 +117,10 @@ Azure Storage Container name
|
|
117
117
|
|
118
118
|
This plugin create container if not exist when you set 'auto_create_container' to true.
|
119
119
|
|
120
|
+
### skip_container_check
|
121
|
+
|
122
|
+
You can skip the initial container listing (and container creation) operations at startup. That can be useful if the user is not allowed to perform these operations.
|
123
|
+
|
120
124
|
### azure_object_key_format
|
121
125
|
|
122
126
|
The format of Azure Storage object keys. You can use several built-in variables:
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.1.
|
1
|
+
0.1.5
|
@@ -33,6 +33,7 @@ module Fluent::Plugin
|
|
33
33
|
config_param :file_extension, :string, :default => "log"
|
34
34
|
config_param :store_as, :string, :default => "none"
|
35
35
|
config_param :auto_create_container, :bool, :default => false
|
36
|
+
config_param :skip_container_check, :bool, :default => false
|
36
37
|
config_param :format, :string, :default => "out_file"
|
37
38
|
config_param :time_slice_format, :string, :default => '%Y%m%d'
|
38
39
|
config_param :command_parameter, :string, :default => nil
|
@@ -57,13 +58,17 @@ module Fluent::Plugin
|
|
57
58
|
compat_parameters_convert(conf, :buffer, :formatter, :inject)
|
58
59
|
super
|
59
60
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
61
|
+
if @store_as.nil? || @store_as == "none"
|
62
|
+
log.info "azurestorage_gen2: Compression is disabled (store_as: #{@store_as})"
|
63
|
+
else
|
64
|
+
begin
|
65
|
+
@compressor = COMPRESSOR_REGISTRY.lookup(@store_as).new(:buffer_type => @buffer_type, :log => log)
|
66
|
+
rescue => e
|
67
|
+
log.warn "#{@store_as} not found. Use 'text' instead"
|
68
|
+
@compressor = TextCompressor.new
|
69
|
+
end
|
70
|
+
@compressor.configure(conf)
|
65
71
|
end
|
66
|
-
@compressor.configure(conf)
|
67
72
|
|
68
73
|
@formatter = formatter_create
|
69
74
|
|
@@ -84,11 +89,15 @@ module Fluent::Plugin
|
|
84
89
|
@azure_storage_path = ''
|
85
90
|
@last_azure_storage_path = ''
|
86
91
|
@current_index = 0
|
87
|
-
if @store_as.nil? || @store_as == "none"
|
92
|
+
if @store_as.nil? || @store_as == "none" || @store_as == "text"
|
88
93
|
@blob_content_type = "text/plain"
|
94
|
+
else
|
95
|
+
@blob_content_type = "application/octet-stream"
|
96
|
+
end
|
97
|
+
|
98
|
+
if @store_as.nil? || @store_as == "none"
|
89
99
|
@final_file_extension = @file_extension
|
90
100
|
else
|
91
|
-
@blob_content_type = @compressor.content_type
|
92
101
|
@final_file_extension = @compressor.ext
|
93
102
|
end
|
94
103
|
|
@@ -100,10 +109,17 @@ module Fluent::Plugin
|
|
100
109
|
|
101
110
|
def start
|
102
111
|
setup_access_token
|
103
|
-
|
112
|
+
if !@skip_container_check
|
113
|
+
ensure_container
|
114
|
+
end
|
104
115
|
super
|
105
116
|
end
|
106
117
|
|
118
|
+
def format(tag, time, record)
|
119
|
+
r = inject_values_to_record(tag, time, record)
|
120
|
+
@formatter.format(tag, time, r)
|
121
|
+
end
|
122
|
+
|
107
123
|
def write(chunk)
|
108
124
|
metadata = chunk.metadata
|
109
125
|
if @store_as.nil? || @store_as == "none"
|
@@ -121,7 +137,6 @@ module Fluent::Plugin
|
|
121
137
|
raw_data << "#{line}\n"
|
122
138
|
end
|
123
139
|
end
|
124
|
-
raw_data = raw_data.chomp
|
125
140
|
unless raw_data.empty?
|
126
141
|
upload_blob(raw_data, metadata)
|
127
142
|
end
|
@@ -157,11 +172,6 @@ module Fluent::Plugin
|
|
157
172
|
append_blob(content, metadata, existing_content_length)
|
158
173
|
end
|
159
174
|
|
160
|
-
def format(tag, time, record)
|
161
|
-
r = inject_values_to_record(tag, time, record)
|
162
|
-
@formatter.format(tag, time, r)
|
163
|
-
end
|
164
|
-
|
165
175
|
private
|
166
176
|
def generate_log_name(metadata, index)
|
167
177
|
time_slice = if metadata.timekey.nil?
|
@@ -242,7 +252,7 @@ module Fluent::Plugin
|
|
242
252
|
params = { :"api-version" => ACCESS_TOKEN_API_VERSION, :resource => "https://storage.azure.com/"}
|
243
253
|
headers = {:"Content-Type" => "application/x-www-form-urlencoded"}
|
244
254
|
content = "grant_type=client_credentials&client_id=#{@azure_oauth_app_id}&client_secret=#{@azure_oauth_secret}&resource=https://storage.azure.com/"
|
245
|
-
request = Typhoeus::Request.new("https://login.microsoftonline.com/#{@azure_oauth_tenant_id}/oauth2/token", :body => content, :headers => headers
|
255
|
+
request = Typhoeus::Request.new("https://login.microsoftonline.com/#{@azure_oauth_tenant_id}/oauth2/token", :body => content, :headers => headers)
|
246
256
|
request.on_complete do |response|
|
247
257
|
if response.success?
|
248
258
|
data = JSON.parse(response.body)
|
@@ -312,7 +322,7 @@ module Fluent::Plugin
|
|
312
322
|
private
|
313
323
|
def create_blob(blob_path)
|
314
324
|
datestamp = create_request_date
|
315
|
-
headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp,:"Content-Length" => "0", :"Content-Type" => "
|
325
|
+
headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp,:"Content-Length" => "0", :"Content-Type" => "#{@blob_content_type}"}
|
316
326
|
params = {:resource => "file", :recursive => "false"}
|
317
327
|
auth_header = create_auth_header("put", datestamp, "#{@azure_container}#{blob_path}", headers, params)
|
318
328
|
headers[:Authorization] = auth_header
|
@@ -335,7 +345,7 @@ module Fluent::Plugin
|
|
335
345
|
def append_blob_block(blob_path, content, position)
|
336
346
|
log.debug "azurestorage_gen2: append_blob_block.start: Append blob ('#{blob_path}') called with position #{position}"
|
337
347
|
datestamp = create_request_date
|
338
|
-
headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp, :"
|
348
|
+
headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp, :"Content-Type" => "#{@blob_content_type}", :"Content-Length" => content.length}
|
339
349
|
params = {:action => "append", :position => "#{position}"}
|
340
350
|
auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}", headers, params)
|
341
351
|
headers[:Authorization] = auth_header
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-azurestorage-gen2
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Oliver Szabo
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-01-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|