fluent-plugin-azurestorage-gen2 0.1.4 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +5 -1
- data/VERSION +1 -1
- data/lib/fluent/plugin/out_azurestorage_gen2.rb +28 -18
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c1f459cce5a616b607bcd07b2a0e60acb2b7f0e2
|
4
|
+
data.tar.gz: 6c162b6991996a2fb61f7f3f8af87a3e74454b6f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 75e25084709b908aa16411d04262e828c52bbe367cc11d250ab8c9b7575cf819134b69aa0dc15c6c9ba8d902212c40bb79336ce9661378d4fc2968d87020f112
|
7
|
+
data.tar.gz: 460b141c22880446e7a6a15990f5c7d753646c372aefb41a7b13ecb6fb270fe9fe44528cb985c54fde282adba39179ca42a86bbd6713b530d66703c6ba3e425f
|
data/README.md
CHANGED
@@ -9,7 +9,7 @@
|
|
9
9
|
|
10
10
|
| fluent-plugin-azurestorage-gen2 | fluentd | ruby |
|
11
11
|
|------------------------|---------|------|
|
12
|
-
| >= 0.1.
|
12
|
+
| >= 0.1.5 | >= v0.14.0 | >= 2.4 |
|
13
13
|
|
14
14
|
## Overview
|
15
15
|
|
@@ -117,6 +117,10 @@ Azure Storage Container name
|
|
117
117
|
|
118
118
|
This plugin create container if not exist when you set 'auto_create_container' to true.
|
119
119
|
|
120
|
+
### skip_container_check
|
121
|
+
|
122
|
+
You can skip the initial container listing (and container creation) operations at startup. That can be useful if the user is not allowed to perform these operations.
|
123
|
+
|
120
124
|
### azure_object_key_format
|
121
125
|
|
122
126
|
The format of Azure Storage object keys. You can use several built-in variables:
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.1.
|
1
|
+
0.1.5
|
@@ -33,6 +33,7 @@ module Fluent::Plugin
|
|
33
33
|
config_param :file_extension, :string, :default => "log"
|
34
34
|
config_param :store_as, :string, :default => "none"
|
35
35
|
config_param :auto_create_container, :bool, :default => false
|
36
|
+
config_param :skip_container_check, :bool, :default => false
|
36
37
|
config_param :format, :string, :default => "out_file"
|
37
38
|
config_param :time_slice_format, :string, :default => '%Y%m%d'
|
38
39
|
config_param :command_parameter, :string, :default => nil
|
@@ -57,13 +58,17 @@ module Fluent::Plugin
|
|
57
58
|
compat_parameters_convert(conf, :buffer, :formatter, :inject)
|
58
59
|
super
|
59
60
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
61
|
+
if @store_as.nil? || @store_as == "none"
|
62
|
+
log.info "azurestorage_gen2: Compression is disabled (store_as: #{@store_as})"
|
63
|
+
else
|
64
|
+
begin
|
65
|
+
@compressor = COMPRESSOR_REGISTRY.lookup(@store_as).new(:buffer_type => @buffer_type, :log => log)
|
66
|
+
rescue => e
|
67
|
+
log.warn "#{@store_as} not found. Use 'text' instead"
|
68
|
+
@compressor = TextCompressor.new
|
69
|
+
end
|
70
|
+
@compressor.configure(conf)
|
65
71
|
end
|
66
|
-
@compressor.configure(conf)
|
67
72
|
|
68
73
|
@formatter = formatter_create
|
69
74
|
|
@@ -84,11 +89,15 @@ module Fluent::Plugin
|
|
84
89
|
@azure_storage_path = ''
|
85
90
|
@last_azure_storage_path = ''
|
86
91
|
@current_index = 0
|
87
|
-
if @store_as.nil? || @store_as == "none"
|
92
|
+
if @store_as.nil? || @store_as == "none" || @store_as == "text"
|
88
93
|
@blob_content_type = "text/plain"
|
94
|
+
else
|
95
|
+
@blob_content_type = "application/octet-stream"
|
96
|
+
end
|
97
|
+
|
98
|
+
if @store_as.nil? || @store_as == "none"
|
89
99
|
@final_file_extension = @file_extension
|
90
100
|
else
|
91
|
-
@blob_content_type = @compressor.content_type
|
92
101
|
@final_file_extension = @compressor.ext
|
93
102
|
end
|
94
103
|
|
@@ -100,10 +109,17 @@ module Fluent::Plugin
|
|
100
109
|
|
101
110
|
def start
|
102
111
|
setup_access_token
|
103
|
-
|
112
|
+
if !@skip_container_check
|
113
|
+
ensure_container
|
114
|
+
end
|
104
115
|
super
|
105
116
|
end
|
106
117
|
|
118
|
+
def format(tag, time, record)
|
119
|
+
r = inject_values_to_record(tag, time, record)
|
120
|
+
@formatter.format(tag, time, r)
|
121
|
+
end
|
122
|
+
|
107
123
|
def write(chunk)
|
108
124
|
metadata = chunk.metadata
|
109
125
|
if @store_as.nil? || @store_as == "none"
|
@@ -121,7 +137,6 @@ module Fluent::Plugin
|
|
121
137
|
raw_data << "#{line}\n"
|
122
138
|
end
|
123
139
|
end
|
124
|
-
raw_data = raw_data.chomp
|
125
140
|
unless raw_data.empty?
|
126
141
|
upload_blob(raw_data, metadata)
|
127
142
|
end
|
@@ -157,11 +172,6 @@ module Fluent::Plugin
|
|
157
172
|
append_blob(content, metadata, existing_content_length)
|
158
173
|
end
|
159
174
|
|
160
|
-
def format(tag, time, record)
|
161
|
-
r = inject_values_to_record(tag, time, record)
|
162
|
-
@formatter.format(tag, time, r)
|
163
|
-
end
|
164
|
-
|
165
175
|
private
|
166
176
|
def generate_log_name(metadata, index)
|
167
177
|
time_slice = if metadata.timekey.nil?
|
@@ -242,7 +252,7 @@ module Fluent::Plugin
|
|
242
252
|
params = { :"api-version" => ACCESS_TOKEN_API_VERSION, :resource => "https://storage.azure.com/"}
|
243
253
|
headers = {:"Content-Type" => "application/x-www-form-urlencoded"}
|
244
254
|
content = "grant_type=client_credentials&client_id=#{@azure_oauth_app_id}&client_secret=#{@azure_oauth_secret}&resource=https://storage.azure.com/"
|
245
|
-
request = Typhoeus::Request.new("https://login.microsoftonline.com/#{@azure_oauth_tenant_id}/oauth2/token", :body => content, :headers => headers
|
255
|
+
request = Typhoeus::Request.new("https://login.microsoftonline.com/#{@azure_oauth_tenant_id}/oauth2/token", :body => content, :headers => headers)
|
246
256
|
request.on_complete do |response|
|
247
257
|
if response.success?
|
248
258
|
data = JSON.parse(response.body)
|
@@ -312,7 +322,7 @@ module Fluent::Plugin
|
|
312
322
|
private
|
313
323
|
def create_blob(blob_path)
|
314
324
|
datestamp = create_request_date
|
315
|
-
headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp,:"Content-Length" => "0", :"Content-Type" => "
|
325
|
+
headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp,:"Content-Length" => "0", :"Content-Type" => "#{@blob_content_type}"}
|
316
326
|
params = {:resource => "file", :recursive => "false"}
|
317
327
|
auth_header = create_auth_header("put", datestamp, "#{@azure_container}#{blob_path}", headers, params)
|
318
328
|
headers[:Authorization] = auth_header
|
@@ -335,7 +345,7 @@ module Fluent::Plugin
|
|
335
345
|
def append_blob_block(blob_path, content, position)
|
336
346
|
log.debug "azurestorage_gen2: append_blob_block.start: Append blob ('#{blob_path}') called with position #{position}"
|
337
347
|
datestamp = create_request_date
|
338
|
-
headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp, :"
|
348
|
+
headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp, :"Content-Type" => "#{@blob_content_type}", :"Content-Length" => content.length}
|
339
349
|
params = {:action => "append", :position => "#{position}"}
|
340
350
|
auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}", headers, params)
|
341
351
|
headers[:Authorization] = auth_header
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-azurestorage-gen2
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Oliver Szabo
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-01-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|