fluent-plugin-azurestorage-gen2 0.1.4 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 13b891a2c06d9a6a18d088c8cf02eb781d3538eb
4
- data.tar.gz: cc89a73ec41eed32b5a7f3a4d104c3ac80422642
3
+ metadata.gz: c1f459cce5a616b607bcd07b2a0e60acb2b7f0e2
4
+ data.tar.gz: 6c162b6991996a2fb61f7f3f8af87a3e74454b6f
5
5
  SHA512:
6
- metadata.gz: da9ade031001af9fea91952af3da3fea6fa32dac029b4b2ae84cb9ac92c2b68e2fd7f9232380ec38fd96990ad53b07d9a16ec11699c12ed4b3e2978a72c03364
7
- data.tar.gz: 76d5f0f943d2647b388852d4c4d274c0af6c8cfae9b3440afaed6b5cb4db240106efe106e974cb8ad744fe7f3ecb4deee143271e42ef67f2f9db184427b96143
6
+ metadata.gz: 75e25084709b908aa16411d04262e828c52bbe367cc11d250ab8c9b7575cf819134b69aa0dc15c6c9ba8d902212c40bb79336ce9661378d4fc2968d87020f112
7
+ data.tar.gz: 460b141c22880446e7a6a15990f5c7d753646c372aefb41a7b13ecb6fb270fe9fe44528cb985c54fde282adba39179ca42a86bbd6713b530d66703c6ba3e425f
data/README.md CHANGED
@@ -9,7 +9,7 @@
9
9
 
10
10
  | fluent-plugin-azurestorage-gen2 | fluentd | ruby |
11
11
  |------------------------|---------|------|
12
- | >= 0.1.3 | >= v0.14.0 | >= 2.4 |
12
+ | >= 0.1.5 | >= v0.14.0 | >= 2.4 |
13
13
 
14
14
  ## Overview
15
15
 
@@ -117,6 +117,10 @@ Azure Storage Container name
117
117
 
118
118
  This plugin create container if not exist when you set 'auto_create_container' to true.
119
119
 
120
+ ### skip_container_check
121
+
122
+ You can skip the initial container listing (and container creation) operations at startup. That can be useful if the user is not allowed to perform these operations.
123
+
120
124
  ### azure_object_key_format
121
125
 
122
126
  The format of Azure Storage object keys. You can use several built-in variables:
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.1.4
1
+ 0.1.5
@@ -33,6 +33,7 @@ module Fluent::Plugin
33
33
  config_param :file_extension, :string, :default => "log"
34
34
  config_param :store_as, :string, :default => "none"
35
35
  config_param :auto_create_container, :bool, :default => false
36
+ config_param :skip_container_check, :bool, :default => false
36
37
  config_param :format, :string, :default => "out_file"
37
38
  config_param :time_slice_format, :string, :default => '%Y%m%d'
38
39
  config_param :command_parameter, :string, :default => nil
@@ -57,13 +58,17 @@ module Fluent::Plugin
57
58
  compat_parameters_convert(conf, :buffer, :formatter, :inject)
58
59
  super
59
60
 
60
- begin
61
- @compressor = COMPRESSOR_REGISTRY.lookup(@store_as).new(:buffer_type => @buffer_type, :log => log)
62
- rescue => e
63
- log.warn "#{@store_as} not found. Use 'text' instead"
64
- @compressor = TextCompressor.new
61
+ if @store_as.nil? || @store_as == "none"
62
+ log.info "azurestorage_gen2: Compression is disabled (store_as: #{@store_as})"
63
+ else
64
+ begin
65
+ @compressor = COMPRESSOR_REGISTRY.lookup(@store_as).new(:buffer_type => @buffer_type, :log => log)
66
+ rescue => e
67
+ log.warn "#{@store_as} not found. Use 'text' instead"
68
+ @compressor = TextCompressor.new
69
+ end
70
+ @compressor.configure(conf)
65
71
  end
66
- @compressor.configure(conf)
67
72
 
68
73
  @formatter = formatter_create
69
74
 
@@ -84,11 +89,15 @@ module Fluent::Plugin
84
89
  @azure_storage_path = ''
85
90
  @last_azure_storage_path = ''
86
91
  @current_index = 0
87
- if @store_as.nil? || @store_as == "none"
92
+ if @store_as.nil? || @store_as == "none" || @store_as == "text"
88
93
  @blob_content_type = "text/plain"
94
+ else
95
+ @blob_content_type = "application/octet-stream"
96
+ end
97
+
98
+ if @store_as.nil? || @store_as == "none"
89
99
  @final_file_extension = @file_extension
90
100
  else
91
- @blob_content_type = @compressor.content_type
92
101
  @final_file_extension = @compressor.ext
93
102
  end
94
103
 
@@ -100,10 +109,17 @@ module Fluent::Plugin
100
109
 
101
110
  def start
102
111
  setup_access_token
103
- ensure_container
112
+ if !@skip_container_check
113
+ ensure_container
114
+ end
104
115
  super
105
116
  end
106
117
 
118
+ def format(tag, time, record)
119
+ r = inject_values_to_record(tag, time, record)
120
+ @formatter.format(tag, time, r)
121
+ end
122
+
107
123
  def write(chunk)
108
124
  metadata = chunk.metadata
109
125
  if @store_as.nil? || @store_as == "none"
@@ -121,7 +137,6 @@ module Fluent::Plugin
121
137
  raw_data << "#{line}\n"
122
138
  end
123
139
  end
124
- raw_data = raw_data.chomp
125
140
  unless raw_data.empty?
126
141
  upload_blob(raw_data, metadata)
127
142
  end
@@ -157,11 +172,6 @@ module Fluent::Plugin
157
172
  append_blob(content, metadata, existing_content_length)
158
173
  end
159
174
 
160
- def format(tag, time, record)
161
- r = inject_values_to_record(tag, time, record)
162
- @formatter.format(tag, time, r)
163
- end
164
-
165
175
  private
166
176
  def generate_log_name(metadata, index)
167
177
  time_slice = if metadata.timekey.nil?
@@ -242,7 +252,7 @@ module Fluent::Plugin
242
252
  params = { :"api-version" => ACCESS_TOKEN_API_VERSION, :resource => "https://storage.azure.com/"}
243
253
  headers = {:"Content-Type" => "application/x-www-form-urlencoded"}
244
254
  content = "grant_type=client_credentials&client_id=#{@azure_oauth_app_id}&client_secret=#{@azure_oauth_secret}&resource=https://storage.azure.com/"
245
- request = Typhoeus::Request.new("https://login.microsoftonline.com/#{@azure_oauth_tenant_id}/oauth2/token", :body => content, :headers => headers, verbose: true)
255
+ request = Typhoeus::Request.new("https://login.microsoftonline.com/#{@azure_oauth_tenant_id}/oauth2/token", :body => content, :headers => headers)
246
256
  request.on_complete do |response|
247
257
  if response.success?
248
258
  data = JSON.parse(response.body)
@@ -312,7 +322,7 @@ module Fluent::Plugin
312
322
  private
313
323
  def create_blob(blob_path)
314
324
  datestamp = create_request_date
315
- headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp,:"Content-Length" => "0", :"Content-Type" => "application/json"}
325
+ headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp,:"Content-Length" => "0", :"Content-Type" => "#{@blob_content_type}"}
316
326
  params = {:resource => "file", :recursive => "false"}
317
327
  auth_header = create_auth_header("put", datestamp, "#{@azure_container}#{blob_path}", headers, params)
318
328
  headers[:Authorization] = auth_header
@@ -335,7 +345,7 @@ module Fluent::Plugin
335
345
  def append_blob_block(blob_path, content, position)
336
346
  log.debug "azurestorage_gen2: append_blob_block.start: Append blob ('#{blob_path}') called with position #{position}"
337
347
  datestamp = create_request_date
338
- headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp, :"x-ms-content-type" => "#{@blob_content_type}", :"Content-Length" => content.length}
348
+ headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp, :"Content-Type" => "#{@blob_content_type}", :"Content-Length" => content.length}
339
349
  params = {:action => "append", :position => "#{position}"}
340
350
  auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}", headers, params)
341
351
  headers[:Authorization] = auth_header
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-azurestorage-gen2
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.4
4
+ version: 0.1.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Oliver Szabo
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-12-23 00:00:00.000000000 Z
11
+ date: 2020-01-02 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd