fluent-plugin-azurestorage-gen2 0.1.4 → 0.1.5

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 13b891a2c06d9a6a18d088c8cf02eb781d3538eb
4
- data.tar.gz: cc89a73ec41eed32b5a7f3a4d104c3ac80422642
3
+ metadata.gz: c1f459cce5a616b607bcd07b2a0e60acb2b7f0e2
4
+ data.tar.gz: 6c162b6991996a2fb61f7f3f8af87a3e74454b6f
5
5
  SHA512:
6
- metadata.gz: da9ade031001af9fea91952af3da3fea6fa32dac029b4b2ae84cb9ac92c2b68e2fd7f9232380ec38fd96990ad53b07d9a16ec11699c12ed4b3e2978a72c03364
7
- data.tar.gz: 76d5f0f943d2647b388852d4c4d274c0af6c8cfae9b3440afaed6b5cb4db240106efe106e974cb8ad744fe7f3ecb4deee143271e42ef67f2f9db184427b96143
6
+ metadata.gz: 75e25084709b908aa16411d04262e828c52bbe367cc11d250ab8c9b7575cf819134b69aa0dc15c6c9ba8d902212c40bb79336ce9661378d4fc2968d87020f112
7
+ data.tar.gz: 460b141c22880446e7a6a15990f5c7d753646c372aefb41a7b13ecb6fb270fe9fe44528cb985c54fde282adba39179ca42a86bbd6713b530d66703c6ba3e425f
data/README.md CHANGED
@@ -9,7 +9,7 @@
9
9
 
10
10
  | fluent-plugin-azurestorage-gen2 | fluentd | ruby |
11
11
  |------------------------|---------|------|
12
- | >= 0.1.3 | >= v0.14.0 | >= 2.4 |
12
+ | >= 0.1.5 | >= v0.14.0 | >= 2.4 |
13
13
 
14
14
  ## Overview
15
15
 
@@ -117,6 +117,10 @@ Azure Storage Container name
117
117
 
118
118
  This plugin create container if not exist when you set 'auto_create_container' to true.
119
119
 
120
+ ### skip_container_check
121
+
122
+ You can skip the initial container listing (and container creation) operations at startup. That can be useful if the user is not allowed to perform these operations.
123
+
120
124
  ### azure_object_key_format
121
125
 
122
126
  The format of Azure Storage object keys. You can use several built-in variables:
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.1.4
1
+ 0.1.5
@@ -33,6 +33,7 @@ module Fluent::Plugin
33
33
  config_param :file_extension, :string, :default => "log"
34
34
  config_param :store_as, :string, :default => "none"
35
35
  config_param :auto_create_container, :bool, :default => false
36
+ config_param :skip_container_check, :bool, :default => false
36
37
  config_param :format, :string, :default => "out_file"
37
38
  config_param :time_slice_format, :string, :default => '%Y%m%d'
38
39
  config_param :command_parameter, :string, :default => nil
@@ -57,13 +58,17 @@ module Fluent::Plugin
57
58
  compat_parameters_convert(conf, :buffer, :formatter, :inject)
58
59
  super
59
60
 
60
- begin
61
- @compressor = COMPRESSOR_REGISTRY.lookup(@store_as).new(:buffer_type => @buffer_type, :log => log)
62
- rescue => e
63
- log.warn "#{@store_as} not found. Use 'text' instead"
64
- @compressor = TextCompressor.new
61
+ if @store_as.nil? || @store_as == "none"
62
+ log.info "azurestorage_gen2: Compression is disabled (store_as: #{@store_as})"
63
+ else
64
+ begin
65
+ @compressor = COMPRESSOR_REGISTRY.lookup(@store_as).new(:buffer_type => @buffer_type, :log => log)
66
+ rescue => e
67
+ log.warn "#{@store_as} not found. Use 'text' instead"
68
+ @compressor = TextCompressor.new
69
+ end
70
+ @compressor.configure(conf)
65
71
  end
66
- @compressor.configure(conf)
67
72
 
68
73
  @formatter = formatter_create
69
74
 
@@ -84,11 +89,15 @@ module Fluent::Plugin
84
89
  @azure_storage_path = ''
85
90
  @last_azure_storage_path = ''
86
91
  @current_index = 0
87
- if @store_as.nil? || @store_as == "none"
92
+ if @store_as.nil? || @store_as == "none" || @store_as == "text"
88
93
  @blob_content_type = "text/plain"
94
+ else
95
+ @blob_content_type = "application/octet-stream"
96
+ end
97
+
98
+ if @store_as.nil? || @store_as == "none"
89
99
  @final_file_extension = @file_extension
90
100
  else
91
- @blob_content_type = @compressor.content_type
92
101
  @final_file_extension = @compressor.ext
93
102
  end
94
103
 
@@ -100,10 +109,17 @@ module Fluent::Plugin
100
109
 
101
110
  def start
102
111
  setup_access_token
103
- ensure_container
112
+ if !@skip_container_check
113
+ ensure_container
114
+ end
104
115
  super
105
116
  end
106
117
 
118
+ def format(tag, time, record)
119
+ r = inject_values_to_record(tag, time, record)
120
+ @formatter.format(tag, time, r)
121
+ end
122
+
107
123
  def write(chunk)
108
124
  metadata = chunk.metadata
109
125
  if @store_as.nil? || @store_as == "none"
@@ -121,7 +137,6 @@ module Fluent::Plugin
121
137
  raw_data << "#{line}\n"
122
138
  end
123
139
  end
124
- raw_data = raw_data.chomp
125
140
  unless raw_data.empty?
126
141
  upload_blob(raw_data, metadata)
127
142
  end
@@ -157,11 +172,6 @@ module Fluent::Plugin
157
172
  append_blob(content, metadata, existing_content_length)
158
173
  end
159
174
 
160
- def format(tag, time, record)
161
- r = inject_values_to_record(tag, time, record)
162
- @formatter.format(tag, time, r)
163
- end
164
-
165
175
  private
166
176
  def generate_log_name(metadata, index)
167
177
  time_slice = if metadata.timekey.nil?
@@ -242,7 +252,7 @@ module Fluent::Plugin
242
252
  params = { :"api-version" => ACCESS_TOKEN_API_VERSION, :resource => "https://storage.azure.com/"}
243
253
  headers = {:"Content-Type" => "application/x-www-form-urlencoded"}
244
254
  content = "grant_type=client_credentials&client_id=#{@azure_oauth_app_id}&client_secret=#{@azure_oauth_secret}&resource=https://storage.azure.com/"
245
- request = Typhoeus::Request.new("https://login.microsoftonline.com/#{@azure_oauth_tenant_id}/oauth2/token", :body => content, :headers => headers, verbose: true)
255
+ request = Typhoeus::Request.new("https://login.microsoftonline.com/#{@azure_oauth_tenant_id}/oauth2/token", :body => content, :headers => headers)
246
256
  request.on_complete do |response|
247
257
  if response.success?
248
258
  data = JSON.parse(response.body)
@@ -312,7 +322,7 @@ module Fluent::Plugin
312
322
  private
313
323
  def create_blob(blob_path)
314
324
  datestamp = create_request_date
315
- headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp,:"Content-Length" => "0", :"Content-Type" => "application/json"}
325
+ headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp,:"Content-Length" => "0", :"Content-Type" => "#{@blob_content_type}"}
316
326
  params = {:resource => "file", :recursive => "false"}
317
327
  auth_header = create_auth_header("put", datestamp, "#{@azure_container}#{blob_path}", headers, params)
318
328
  headers[:Authorization] = auth_header
@@ -335,7 +345,7 @@ module Fluent::Plugin
335
345
  def append_blob_block(blob_path, content, position)
336
346
  log.debug "azurestorage_gen2: append_blob_block.start: Append blob ('#{blob_path}') called with position #{position}"
337
347
  datestamp = create_request_date
338
- headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp, :"x-ms-content-type" => "#{@blob_content_type}", :"Content-Length" => content.length}
348
+ headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp, :"Content-Type" => "#{@blob_content_type}", :"Content-Length" => content.length}
339
349
  params = {:action => "append", :position => "#{position}"}
340
350
  auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}", headers, params)
341
351
  headers[:Authorization] = auth_header
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-azurestorage-gen2
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.4
4
+ version: 0.1.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Oliver Szabo
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-12-23 00:00:00.000000000 Z
11
+ date: 2020-01-02 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd