fluent-plugin-azure-storage-append-blob 0.1.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 71879d8031373e9beacb9314ded1569bf606e18d4ba4cd7b1804bf6b197b130e
4
- data.tar.gz: e1073c5e8275a61cd18641254aa4d19446dad9e1673c8b8de5751604b63d85e6
3
+ metadata.gz: 7640e2e4b681f3d556b69dfd6cce4483c5af1e3a98a92b9261c5a2d9e1365277
4
+ data.tar.gz: 1fbc5232cbe8e50f6518fc41460e94ad59f9187d36fa8f3faf4800b5490b46d3
5
5
  SHA512:
6
- metadata.gz: 3e891db80e04fe02d1250f3c2d4a16705738c19fd945872b3ddde8d1f263e77d00fa19e6913890ed638aa5487f1f7f819d44336394ad73c82ee6c35637e897db
7
- data.tar.gz: c17efa89408c12fc5a991baaff846dd4e57f24e169e3f325c3d31df0b28a193123e5c8fbe835448b3c6ada157ceaad7c5005e6270591a42ed9ec1b3dbcd9f268
6
+ metadata.gz: cbc94f676ec13fec572539c98b82f200f6ef11659ca6745720eefcd2d71c7b34ac9287a9175b1a0bc1c84e6d70feaf7ceea5df8ad5c3af7600fffb52237356c2
7
+ data.tar.gz: 1989910ce48aaffd875d4226c097e85d25c04cea12dbcc9c4cfad13560a52ae1406675544be27bf02981fc20f78e4da88bf07d3ffd58fcc7744f7a0ddaeeee44
@@ -0,0 +1,27 @@
1
+ name: Ruby Gem
2
+
3
+ on:
4
+ workflow_dispatch:
5
+
6
+ jobs:
7
+ build:
8
+ name: Build + Publish
9
+ runs-on: ubuntu-latest
10
+
11
+ steps:
12
+ - uses: actions/checkout@v2
13
+ - name: Set up Ruby 2.6
14
+ uses: actions/setup-ruby@v1
15
+ with:
16
+ ruby-version: 2.6.x
17
+
18
+ - name: Publish to RubyGems
19
+ run: |
20
+ mkdir -p $HOME/.gem
21
+ touch $HOME/.gem/credentials
22
+ chmod 0600 $HOME/.gem/credentials
23
+ printf -- "---\n:rubygems_api_key: ${RUBYGEMS_API_KEY}\n" > $HOME/.gem/credentials
24
+ gem build *.gemspec
25
+ gem push *.gem
26
+ env:
27
+ RUBYGEMS_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"
@@ -0,0 +1,23 @@
1
+ name: Unittests
2
+
3
+ on:
4
+ push:
5
+ branches: [ master ]
6
+ pull_request:
7
+ branches: [ master ]
8
+
9
+ jobs:
10
+ test:
11
+
12
+ runs-on: ubuntu-latest
13
+
14
+ steps:
15
+ - uses: actions/checkout@v2
16
+ - name: Set up Ruby
17
+ uses: ruby/setup-ruby@v1
18
+ with:
19
+ ruby-version: 2.6
20
+ - name: Install dependencies
21
+ run: bundle install
22
+ - name: Run tests
23
+ run: bundle exec rake test
data/.gitignore CHANGED
@@ -42,7 +42,7 @@ build-iPhoneSimulator/
42
42
 
43
43
  # for a library or gem, you might want to ignore these files since the code is
44
44
  # intended to run in multiple environments; otherwise, check them in:
45
- # Gemfile.lock
45
+ Gemfile.lock
46
46
  # .ruby-version
47
47
  # .ruby-gemset
48
48
 
@@ -0,0 +1,2 @@
1
+ Style/FrozenStringLiteralComment:
2
+ Enabled: false
@@ -0,0 +1,36 @@
1
+ FROM ruby:latest
2
+
3
+ WORKDIR /plugin
4
+
5
+ ADD . /plugin
6
+
7
+ RUN gem install bundler && \
8
+ gem install fluentd --no-doc && \
9
+ fluent-gem build fluent-plugin-azure-storage-append-blob.gemspec && \
10
+ fluent-gem install fluent-plugin-azure-storage-append-blob-*.gem
11
+
12
+ RUN echo "<source>\n\
13
+ @type sample\n\
14
+ sample {\"hello\":\"world\"}\n\
15
+ tag pattern\n\
16
+ </source>\n\
17
+ <match pattern>\n\
18
+ @type azure-storage-append-blob\n\
19
+ azure_storage_account \"#{ENV['STORAGE_ACCOUNT']}\"\n\
20
+ azure_storage_access_key \"#{ENV['STORAGE_ACCESS_KEY']}\"\n\
21
+ azure_storage_sas_token \"#{ENV['STORAGE_SAS_TOKEN']}\"\n\
22
+ azure_container fluentd\n\
23
+ auto_create_container true\n\
24
+ path logs/\n\
25
+ azure_object_key_format %{path}%{time_slice}_%{index}.log\n\
26
+ time_slice_format %Y%m%d-%H\n\
27
+ <buffer tag,time>\n\
28
+ @type file\n\
29
+ path /var/log/fluent/azurestorageappendblob\n\
30
+ timekey 120 # 2 minutes\n\
31
+ timekey_wait 60\n\
32
+ timekey_use_utc true # use utc\n\
33
+ </buffer>\n\
34
+ </match>" > /plugin/fluent.conf
35
+
36
+ ENTRYPOINT ["fluentd", "-c", "fluent.conf"]
data/Gemfile CHANGED
File without changes
data/LICENSE CHANGED
File without changes
data/README.md CHANGED
@@ -8,67 +8,79 @@ Azure Storage Append Blob output plugin buffers logs in local file and uploads t
8
8
 
9
9
  ### RubyGems
10
10
 
11
- ```
12
- $ gem install fluent-plugin-azure-storage-append-blob
13
- ```
11
+ gem install fluent-plugin-azure-storage-append-blob
14
12
 
15
13
  ### Bundler
16
14
 
17
15
  Add following line to your Gemfile:
18
16
 
19
- ```ruby
20
- gem "fluent-plugin-azure-storage-append-blob"
21
- ```
17
+ gem "fluent-plugin-azure-storage-append-blob"
22
18
 
23
19
  And then execute:
24
20
 
25
- ```
26
- $ bundle
27
- ```
21
+ bundle
28
22
 
29
23
  ## Configuration
30
24
 
31
- ```
32
- <match pattern>
33
- type azure-storage-append-blob
25
+ <match pattern>
26
+ type azure-storage-append-blob
34
27
 
35
- azure_storage_account <your azure storage account>
36
- azure_storage_access_key <your azure storage access key>
37
- azure_container <your azure storage container>
38
- auto_create_container true
39
- path logs/
40
- azure_blob_name_format %{path}%{time_slice}_%{index}.log
41
- time_slice_format %Y%m%d-%H
42
- # if you want to use %{tag} or %Y/%m/%d/ like syntax in path / azure_blob_name_format,
43
- # need to specify tag for %{tag} and time for %Y/%m/%d in <buffer> argument.
44
- <buffer tag,time>
45
- @type file
46
- path /var/log/fluent/azurestorageappendblob
47
- timekey 120 # 2 minutes
48
- timekey_wait 60
49
- timekey_use_utc true # use utc
50
- </buffer>
51
- </match>
52
- ```
28
+ azure_storage_account <your azure storage account>
29
+ azure_storage_access_key <your azure storage access key> # leave empty to use MSI
30
+ azure_storage_sas_token <your azure storage sas token> # leave empty to use MSI
31
+ azure_imds_api_version <Azure Instance Metadata Service API Version> # only used for MSI
32
+ azure_token_refresh_interval <refresh interval in min> # only used for MSI
33
+ azure_container <your azure storage container>
34
+ auto_create_container true
35
+ path logs/
36
+ azure_object_key_format %{path}%{time_slice}_%{index}.log
37
+ time_slice_format %Y%m%d-%H
38
+ # if you want to use %{tag} or %Y/%m/%d/ like syntax in path / azure_blob_name_format,
39
+ # need to specify tag for %{tag} and time for %Y/%m/%d in <buffer> argument.
40
+ <buffer tag,time>
41
+ @type file
42
+ path /var/log/fluent/azurestorageappendblob
43
+ timekey 120 # 2 minutes
44
+ timekey_wait 60
45
+ timekey_use_utc true # use utc
46
+ </buffer>
47
+ </match>
53
48
 
54
- ### azure_storage_account (Required)
49
+ ### `azure_storage_account` (Required)
55
50
 
56
- Your Azure Storage Account Name. This can be retrieved from Azure Management potal.
51
+ Your Azure Storage Account Name. This can be retrieved from Azure Management portal.
57
52
 
58
- ### azure_storage_access_key (Required)
53
+ ### `azure_storage_access_key` or `azure_storage_sas_token` (Either required or both empty to use MSI)
59
54
 
60
- Your Azure Storage Access Key(Primary or Secondary). This also can be retrieved from Azure Management potal.
55
+ Your Azure Storage Access Key (Primary or Secondary) or shared access signature (SAS) token.
56
+ This also can be retrieved from Azure Management portal.
61
57
 
62
- ### azure_container (Required)
58
+ If both are empty, the plugin will use the local Managed Identity endpoint to obtain a token for the target storage account.
59
+
60
+ ### `azure_imds_api_version` (Optional, only for MSI)
61
+
62
+ Default: 2019-08-15
63
+
64
+ The Instance Metadata Service is used during the OAuth flow to obtain an access token. This API is versioned and specifying the version is mandatory.
65
+
66
+ See [here](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/instance-metadata-service#versioning) for more details.
67
+
68
+ ### `azure_token_refresh_interval` (Optional, only for MSI)
69
+
70
+ Default: 60 (1 hour)
71
+
72
+ When using MSI, the initial access token needs to be refreshed periodically.
73
+
74
+ ### `azure_container` (Required)
63
75
 
64
76
  Azure Storage Container name
65
77
 
66
- ### auto_create_container
78
+ ### `auto_create_container`
67
79
 
68
80
  This plugin creates the Azure container if it does not already exist exist when you set 'auto_create_container' to true.
69
81
  The default value is `true`
70
82
 
71
- ### azure_object_key_format
83
+ ### `azure_object_key_format`
72
84
 
73
85
  The format of Azure Storage object keys. You can use several built-in variables:
74
86
 
@@ -86,48 +98,68 @@ The default format is "%{path}%{time_slice}-%{index}.log".
86
98
 
87
99
  For instance, using the example configuration above, actual object keys on Azure Storage will be something like:
88
100
 
89
- ```
90
- "logs/20130111-22-0.log"
91
- "logs/20130111-23-0.log"
92
- "logs/20130112-00-0.log"
93
- ```
101
+ "logs/20130111-22-0.log"
102
+ "logs/20130111-23-0.log"
103
+ "logs/20130112-00-0.log"
94
104
 
95
105
  With the configuration:
96
106
 
97
- ```
98
- azure_object_key_format %{path}/events/ts=%{time_slice}/events.log
99
- path log
100
- time_slice_format %Y%m%d-%H
101
- ```
107
+ azure_object_key_format %{path}/events/ts=%{time_slice}/events.log
108
+ path log
109
+ time_slice_format %Y%m%d-%H
102
110
 
103
111
  You get:
104
112
 
105
- ```
106
- "log/events/ts=20130111-22/events.log"
107
- "log/events/ts=20130111-23/events.log"
108
- "log/events/ts=20130112-00/events.log"
109
- ```
113
+ "log/events/ts=20130111-22/events.log"
114
+ "log/events/ts=20130111-23/events.log"
115
+ "log/events/ts=20130112-00/events.log"
110
116
 
111
117
  The [fluent-mixin-config-placeholders](https://github.com/tagomoris/fluent-mixin-config-placeholders) mixin is also incorporated, so additional variables such as %{hostname}, etc. can be used in the `azure_object_key_format`. This is useful in preventing filename conflicts when writing from multiple servers.
112
118
 
113
- ```
114
- azure_object_key_format %{path}/events/ts=%{time_slice}/events-%{hostname}.log
115
- ```
119
+ azure_object_key_format %{path}/events/ts=%{time_slice}/events-%{hostname}.log
116
120
 
117
- ### time_slice_format
121
+ ### `time_slice_format`
118
122
 
119
123
  Format of the time used in the file name. Default is '%Y%m%d'. Use '%Y%m%d%H' to split files hourly.
120
124
 
121
125
  ### Run tests
122
- $ gem install bundler
123
- $ bundle install
124
- $ bundle exec rake test
125
126
 
126
- # Contributing
127
+ gem install bundler
128
+ bundle install
129
+ bundle exec rake test
130
+
131
+
132
+ ### Test Fluentd
133
+
134
+ 1. Create Storage Account and VM with enabled MSI
135
+ 2. Setup Docker ang Git
136
+ 3. SSH into VM
137
+ 4. Download this repo
138
+ ```
139
+ git clone https://github.com/microsoft/fluent-plugin-azure-storage-append-blob.git
140
+ cd fluent-plugin-azure-storage-append-blob
141
+ ```
142
+ 5. Build Docker image
143
+ `docker build -t fluent .`
144
+ 6. Run Docker image with different set of parameters:
145
+
146
+ 1. `STORAGE_ACCOUNT`: required, name of your storage account
147
+ 2. `STORAGE_ACCESS_KEY`: storage account access key
148
+ 3. `STORAGE_SAS_TOKEN`: storage sas token with enough permissions for the plugin
149
+
150
+ You need to specify `STORAGE_ACCOUNT` and one of auth ways. If you run it from VM with MSI,
151
+ just `STORAGE_ACCOUNT` is required. Keep in mind, there is no way to refresh MSI Token, so
152
+ ensure you setup proper permissions first.
153
+
154
+ ```bash
155
+ docker run -it -e STORAGE_ACCOUNT=<storage> -e STORAGE_ACCESS_KEY=<key> fluent
156
+ ```
157
+
158
+ ## Contributing
127
159
 
128
160
  This project welcomes contributions and suggestions. Most contributions require you to agree to a
129
161
  Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
130
- the rights to use your contribution. For details, visit https://cla.microsoft.com.
162
+ the rights to use your contribution. For details, visit [https://cla.microsoft.com](https://cla.microsoft.com).
131
163
 
132
164
  When you submit a pull request, a CLA-bot will automatically determine whether you need to provide
133
165
  a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions
data/Rakefile CHANGED
@@ -1,11 +1,11 @@
1
- require "bundler"
1
+ require 'bundler'
2
2
  Bundler::GemHelper.install_tasks
3
3
 
4
- require "rake/testtask"
4
+ require 'rake/testtask'
5
5
 
6
6
  Rake::TestTask.new(:test) do |t|
7
- t.libs.push("lib", "test")
8
- t.test_files = FileList["test/**/test_*.rb"]
7
+ t.libs.push('lib', 'test')
8
+ t.test_files = FileList['test/**/test_*.rb']
9
9
  t.verbose = true
10
10
  t.warning = true
11
11
  end
File without changes
@@ -2,27 +2,27 @@ lib = File.expand_path("../lib", __FILE__)
2
2
  $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
3
3
 
4
4
  Gem::Specification.new do |spec|
5
- spec.name = "fluent-plugin-azure-storage-append-blob"
6
- spec.version = "0.1.1"
7
- spec.authors = ["Microsoft Corporation"]
8
- spec.email = [""]
5
+ spec.name = 'fluent-plugin-azure-storage-append-blob'
6
+ spec.version = '0.2.0'
7
+ spec.authors = ['Microsoft Corporation']
8
+ spec.email = ['']
9
9
 
10
- spec.summary = "Azure Storage Append Blob output plugin for Fluentd event collector"
11
- spec.description = "Fluentd plugin to upload logs to Azure Storage append blobs."
12
- spec.homepage = "https://github.com/Microsoft/fluent-plugin-azure-storage-append-blob"
13
- spec.license = "MIT"
10
+ spec.summary = 'Azure Storage Append Blob output plugin for Fluentd event collector'
11
+ spec.description = 'Fluentd plugin to upload logs to Azure Storage append blobs.'
12
+ spec.homepage = 'https://github.com/Microsoft/fluent-plugin-azure-storage-append-blob'
13
+ spec.license = 'MIT'
14
14
 
15
- test_files, files = `git ls-files -z`.split("\x0").partition do |f|
15
+ test_files, files = `git ls-files -z`.split("\x0").partition do |f|
16
16
  f.match(%r{^(test|spec|features)/})
17
17
  end
18
- spec.files = files
19
- spec.executables = files.grep(%r{^bin/}) { |f| File.basename(f) }
20
- spec.test_files = test_files
21
- spec.require_paths = ["lib"]
18
+ spec.files = files
19
+ spec.executables = files.grep(%r{^bin/}) { |f| File.basename(f) }
20
+ spec.test_files = test_files
21
+ spec.require_paths = ['lib']
22
22
 
23
- spec.add_development_dependency "bundler", "~> 1.14"
24
- spec.add_development_dependency "rake", "~> 12.0"
25
- spec.add_development_dependency "test-unit", "~> 3.0"
26
- spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
27
- spec.add_runtime_dependency "azure-storage-blob", "~> 1.0"
23
+ spec.add_development_dependency 'bundler', '~> 2.0'
24
+ spec.add_development_dependency 'rake', '~> 13.0'
25
+ spec.add_development_dependency 'test-unit', '~> 3.0'
26
+ spec.add_runtime_dependency 'azure-storage-blob', '~> 2.0'
27
+ spec.add_runtime_dependency 'fluentd', ['>= 0.14.10', '< 2']
28
28
  end
@@ -3,90 +3,133 @@
3
3
  # Licensed under the MIT License. See License.txt in the project root for license information.
4
4
  #--------------------------------------------------------------------------------------------*/
5
5
 
6
- require 'fluent/plugin/output'
6
+ require 'azure/storage/common'
7
7
  require 'azure/storage/blob'
8
- require 'time'
9
- require 'tempfile'
8
+ require 'faraday'
9
+ require 'fluent/plugin/output'
10
+ require 'json'
10
11
 
11
12
  module Fluent
12
13
  module Plugin
13
14
  class AzureStorageAppendBlobOut < Fluent::Plugin::Output
14
- Fluent::Plugin.register_output("azure-storage-append-blob", self)
15
+ Fluent::Plugin.register_output('azure-storage-append-blob', self)
15
16
 
16
17
  helpers :formatter, :inject
17
18
 
18
- DEFAULT_FORMAT_TYPE = "out_file"
19
-
20
- config_param :path, :string, :default => ""
21
- config_param :azure_storage_account, :string, :default => nil
22
- config_param :azure_storage_access_key, :string, :default => nil, :secret => true
23
- config_param :azure_container, :string, :default => nil
24
- config_param :azure_object_key_format, :string, :default => "%{path}%{time_slice}-%{index}.log"
25
- config_param :auto_create_container, :bool, :default => true
26
- config_param :format, :string, :default => DEFAULT_FORMAT_TYPE
27
- config_param :time_slice_format, :string, :default => '%Y%m%d'
28
-
29
- DEFAULT_FORMAT_TYPE = "out_file"
19
+ DEFAULT_FORMAT_TYPE = 'out_file'.freeze
30
20
  AZURE_BLOCK_SIZE_LIMIT = 4 * 1024 * 1024 - 1
31
-
21
+
22
+ config_param :path, :string, default: ''
23
+ config_param :azure_storage_account, :string, default: nil
24
+ config_param :azure_storage_access_key, :string, default: nil, secret: true
25
+ config_param :azure_storage_sas_token, :string, default: nil, secret: true
26
+ config_param :azure_container, :string, default: nil
27
+ config_param :azure_imds_api_version, :string, default: '2019-08-15'
28
+ config_param :azure_token_refresh_interval, :integer, default: 60
29
+ config_param :use_msi, :bool, default: false
30
+ config_param :azure_object_key_format, :string, default: '%{path}%{time_slice}-%{index}.log'
31
+ config_param :auto_create_container, :bool, default: true
32
+ config_param :format, :string, default: DEFAULT_FORMAT_TYPE
33
+ config_param :time_slice_format, :string, default: '%Y%m%d'
34
+ config_param :localtime, :bool, default: false
35
+
32
36
  config_section :format do
33
37
  config_set_default :@type, DEFAULT_FORMAT_TYPE
34
38
  end
35
-
39
+
36
40
  config_section :buffer do
37
41
  config_set_default :chunk_keys, ['time']
38
42
  config_set_default :timekey, (60 * 60 * 24)
39
43
  end
40
-
44
+
41
45
  attr_reader :bs
42
-
46
+
43
47
  def configure(conf)
44
48
  super
45
-
49
+
46
50
  @formatter = formatter_create
47
-
48
- if @localtime
49
- @path_slicer = Proc.new {|path|
50
- Time.now.strftime(path)
51
- }
52
- else
53
- @path_slicer = Proc.new {|path|
54
- Time.now.utc.strftime(path)
55
- }
56
- end
57
-
58
- if @azure_container.nil?
59
- raise ConfigError, 'azure_container is needed'
51
+
52
+ @path_slicer = if @localtime
53
+ proc do |path|
54
+ Time.now.strftime(path)
55
+ end
56
+ else
57
+ proc do |path|
58
+ Time.now.utc.strftime(path)
59
+ end
60
+ end
61
+
62
+ raise ConfigError, 'azure_storage_account needs to be specified' if @azure_storage_account.nil?
63
+
64
+ raise ConfigError, 'azure_container needs to be specified' if @azure_container.nil?
65
+
66
+ if (@azure_storage_access_key.nil? || @azure_storage_access_key.empty?) && (@azure_storage_sas_token.nil? || @azure_storage_sas_token.empty?)
67
+ log.info 'Using MSI since neither azure_storage_access_key nor azure_storage_sas_token was provided.'
68
+ @use_msi = true
60
69
  end
61
70
  end
62
-
71
+
63
72
  def multi_workers_ready?
64
73
  true
65
74
  end
66
-
75
+
76
+ def get_access_token
77
+ access_key_request = Faraday.new('http://169.254.169.254/metadata/identity/oauth2/token?' \
78
+ "api-version=#{@azure_imds_api_version}" \
79
+ '&resource=https://storage.azure.com/',
80
+ headers: { 'Metadata' => 'true' })
81
+ .get
82
+ .body
83
+ JSON.parse(access_key_request)['access_token']
84
+ end
85
+
67
86
  def start
68
87
  super
69
-
70
- @bs = Azure::Storage::Blob::BlobService.create(storage_account_name: @azure_storage_account, storage_access_key: @azure_storage_access_key)
71
-
72
- ensure_container
88
+ if @use_msi
89
+ token_credential = Azure::Storage::Common::Core::TokenCredential.new get_access_token
90
+ token_signer = Azure::Storage::Common::Core::Auth::TokenSigner.new token_credential
91
+ @bs = Azure::Storage::Blob::BlobService.new(storage_account_name: @azure_storage_account, signer: token_signer)
92
+
93
+ refresh_interval = @azure_token_refresh_interval * 60
94
+ cancelled = false
95
+ renew_token = Thread.new do
96
+ Thread.stop
97
+ until cancelled
98
+ sleep(refresh_interval)
73
99
 
100
+ token_credential.renew_token get_access_token
101
+ end
102
+ end
103
+ sleep 0.1 while renew_token.status != 'sleep'
104
+ renew_token.run
105
+ else
106
+ @bs_params = { storage_account_name: @azure_storage_account }
107
+
108
+ if !@azure_storage_access_key.nil? && !@azure_storage_access_key.empty?
109
+ @bs_params.merge!({ storage_access_key: @azure_storage_access_key })
110
+ elsif !@azure_storage_sas_token.nil? && !@azure_storage_sas_token.empty?
111
+ @bs_params.merge!({ storage_sas_token: @azure_storage_sas_token })
112
+ end
113
+
114
+ @bs = Azure::Storage::Blob::BlobService.create(@bs_params)
115
+ end
116
+
117
+ ensure_container
74
118
  @azure_storage_path = ''
75
119
  @last_azure_storage_path = ''
76
120
  @current_index = 0
77
121
  end
78
-
122
+
79
123
  def format(tag, time, record)
80
124
  r = inject_values_to_record(tag, time, record)
81
125
  @formatter.format(tag, time, r)
82
126
  end
83
-
127
+
84
128
  def write(chunk)
85
129
  metadata = chunk.metadata
86
- tmp = Tempfile.new("azure-")
130
+ tmp = Tempfile.new('azure-')
87
131
  begin
88
132
  chunk.write_to(tmp)
89
- tmp.close
90
133
 
91
134
  generate_log_name(metadata, @current_index)
92
135
  if @last_azure_storage_path != @azure_storage_path
@@ -94,18 +137,23 @@ module Fluent
94
137
  generate_log_name(metadata, @current_index)
95
138
  end
96
139
 
97
- content = File.open(tmp.path, 'rb') { |file| file.read }
140
+ content = File.open(tmp.path, 'rb', &:read)
98
141
 
99
142
  append_blob(content, metadata)
100
143
  @last_azure_storage_path = @azure_storage_path
101
144
  ensure
102
- tmp.unlink
145
+ begin
146
+ tmp.close(true)
147
+ rescue StandardError
148
+ nil
149
+ end
103
150
  end
104
151
  end
105
-
152
+
106
153
  private
154
+
107
155
  def ensure_container
108
- if ! @bs.list_containers.find { |c| c.name == @azure_container }
156
+ unless @bs.list_containers.find { |c| c.name == @azure_container }
109
157
  if @auto_create_container
110
158
  @bs.create_container(@azure_container)
111
159
  else
@@ -115,24 +163,26 @@ module Fluent
115
163
  end
116
164
 
117
165
  private
166
+
118
167
  def generate_log_name(metadata, index)
119
168
  time_slice = if metadata.timekey.nil?
120
169
  ''.freeze
121
170
  else
122
171
  Time.at(metadata.timekey).utc.strftime(@time_slice_format)
123
- end
172
+ end
124
173
 
125
174
  path = @path_slicer.call(@path)
126
175
  values_for_object_key = {
127
- "%{path}" => path,
128
- "%{time_slice}" => time_slice,
129
- "%{index}" => index
176
+ '%{path}' => path,
177
+ '%{time_slice}' => time_slice,
178
+ '%{index}' => index
130
179
  }
131
- storage_path = @azure_object_key_format.gsub(%r(%{[^}]+}), values_for_object_key)
180
+ storage_path = @azure_object_key_format.gsub(/%{[^}]+}/, values_for_object_key)
132
181
  @azure_storage_path = extract_placeholders(storage_path, metadata)
133
182
  end
134
183
 
135
184
  private
185
+
136
186
  def append_blob(content, metadata)
137
187
  position = 0
138
188
  log.debug "azure_storage_append_blob: append_blob.start: Content size: #{content.length}"
@@ -143,8 +193,8 @@ module Fluent
143
193
  @bs.append_blob_block(@azure_container, @azure_storage_path, content[position..position + size])
144
194
  position += size
145
195
  break if position >= content.length
146
- rescue Azure::Core::Http::HTTPError => ex
147
- status_code = ex.status_code
196
+ rescue Azure::Core::Http::HTTPError => e
197
+ status_code = e.status_code
148
198
 
149
199
  if status_code == 409 # exceeds azure block limit
150
200
  @current_index += 1
@@ -153,7 +203,7 @@ module Fluent
153
203
 
154
204
  # If index is not a part of format, rethrow exception.
155
205
  if old_azure_storage_path == @azure_storage_path
156
- log.warn "azure_storage_append_blob: append_blob: blocks limit reached, you need to use %{index} for the format."
206
+ log.warn 'azure_storage_append_blob: append_blob: blocks limit reached, you need to use %{index} for the format.'
157
207
  raise
158
208
  end
159
209
 
@@ -167,9 +217,8 @@ module Fluent
167
217
  end
168
218
  end
169
219
  end
170
- log.debug "azure_storage_append_blob: append_blob.complete"
220
+ log.debug 'azure_storage_append_blob: append_blob.complete'
171
221
  end
172
-
173
222
  end
174
223
  end
175
224
  end
@@ -1,8 +1,8 @@
1
- $LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
2
- require "test-unit"
3
- require "fluent/test"
4
- require "fluent/test/driver/output"
5
- require "fluent/test/helpers"
1
+ $LOAD_PATH.unshift(File.expand_path('..', __dir__))
2
+ require 'test-unit'
3
+ require 'fluent/test'
4
+ require 'fluent/test/driver/output'
5
+ require 'fluent/test/helpers'
6
6
 
7
7
  Test::Unit::TestCase.include(Fluent::Test::Helpers)
8
8
  Test::Unit::TestCase.extend(Fluent::Test::Helpers)
@@ -8,32 +8,41 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
8
8
  Fluent::Test.setup
9
9
  end
10
10
 
11
- CONFIG = %[
11
+ CONFIG = %(
12
12
  azure_storage_account test_storage_account
13
13
  azure_storage_access_key MY_FAKE_SECRET
14
14
  azure_container test_container
15
15
  time_slice_format %Y%m%d-%H
16
16
  path log
17
- ]
17
+ ).freeze
18
18
 
19
- def create_driver(conf=CONFIG)
19
+ MSI_CONFIG = %(
20
+ azure_storage_account test_storage_account
21
+ azure_container test_container
22
+ azure_imds_api_version 1970-01-01
23
+ azure_token_refresh_interval 120
24
+ time_slice_format %Y%m%d-%H
25
+ path log
26
+ ).freeze
27
+
28
+ def create_driver(conf = CONFIG)
20
29
  Fluent::Test::Driver::Output.new(Fluent::Plugin::AzureStorageAppendBlobOut).configure(conf)
21
30
  end
22
31
 
23
32
  sub_test_case 'test config' do
24
33
  test 'config should reject with no azure container' do
25
34
  assert_raise Fluent::ConfigError do
26
- create_driver(%[
35
+ create_driver(%(
27
36
  azure_storage_account test_storage_account
28
37
  azure_storage_access_key MY_FAKE_SECRET
29
38
  time_slice_format %Y%m%d-%H
30
39
  time_slice_wait 10m
31
40
  path log
32
- ])
41
+ ))
33
42
  end
34
43
  end
35
44
 
36
- test 'config should set instance variables' do
45
+ test 'config with access key should set instance variables' do
37
46
  d = create_driver
38
47
  assert_equal 'test_storage_account', d.instance.azure_storage_account
39
48
  assert_equal 'MY_FAKE_SECRET', d.instance.azure_storage_access_key
@@ -41,26 +50,37 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
41
50
  assert_equal true, d.instance.auto_create_container
42
51
  assert_equal '%{path}%{time_slice}-%{index}.log', d.instance.azure_object_key_format
43
52
  end
53
+
54
+ test 'config with managed identity enabled should set instance variables' do
55
+ d = create_driver(MSI_CONFIG)
56
+ assert_equal 'test_storage_account', d.instance.azure_storage_account
57
+ assert_equal 'test_container', d.instance.azure_container
58
+ assert_equal true, d.instance.use_msi
59
+ assert_equal true, d.instance.auto_create_container
60
+ assert_equal '%{path}%{time_slice}-%{index}.log', d.instance.azure_object_key_format
61
+ assert_equal 120, d.instance.azure_token_refresh_interval
62
+ assert_equal '1970-01-01', d.instance.azure_imds_api_version
63
+ end
44
64
  end
45
65
 
46
66
  sub_test_case 'test path slicing' do
47
67
  test 'test path_slicing' do
48
- config = CONFIG.clone.gsub(/path\slog/, "path log/%Y/%m/%d")
68
+ config = CONFIG.clone.gsub(/path\slog/, 'path log/%Y/%m/%d')
49
69
  d = create_driver(config)
50
70
  path_slicer = d.instance.instance_variable_get(:@path_slicer)
51
71
  path = d.instance.instance_variable_get(:@path)
52
72
  slice = path_slicer.call(path)
53
- assert_equal slice, Time.now.utc.strftime("log/%Y/%m/%d")
73
+ assert_equal slice, Time.now.utc.strftime('log/%Y/%m/%d')
54
74
  end
55
75
 
56
76
  test 'path slicing utc' do
57
- config = CONFIG.clone.gsub(/path\slog/, "path log/%Y/%m/%d")
77
+ config = CONFIG.clone.gsub(/path\slog/, 'path log/%Y/%m/%d')
58
78
  config << "\nutc\n"
59
79
  d = create_driver(config)
60
80
  path_slicer = d.instance.instance_variable_get(:@path_slicer)
61
81
  path = d.instance.instance_variable_get(:@path)
62
82
  slice = path_slicer.call(path)
63
- assert_equal slice, Time.now.utc.strftime("log/%Y/%m/%d")
83
+ assert_equal slice, Time.now.utc.strftime('log/%Y/%m/%d')
64
84
  end
65
85
  end
66
86
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-azure-storage-append-blob
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.1
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Microsoft Corporation
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-11-29 00:00:00.000000000 Z
11
+ date: 2020-08-13 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -16,28 +16,28 @@ dependencies:
16
16
  requirements:
17
17
  - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: '1.14'
19
+ version: '2.0'
20
20
  type: :development
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: '1.14'
26
+ version: '2.0'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: rake
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
31
  - - "~>"
32
32
  - !ruby/object:Gem::Version
33
- version: '12.0'
33
+ version: '13.0'
34
34
  type: :development
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - "~>"
39
39
  - !ruby/object:Gem::Version
40
- version: '12.0'
40
+ version: '13.0'
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: test-unit
43
43
  requirement: !ruby/object:Gem::Requirement
@@ -52,6 +52,20 @@ dependencies:
52
52
  - - "~>"
53
53
  - !ruby/object:Gem::Version
54
54
  version: '3.0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: azure-storage-blob
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '2.0'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '2.0'
55
69
  - !ruby/object:Gem::Dependency
56
70
  name: fluentd
57
71
  requirement: !ruby/object:Gem::Requirement
@@ -72,20 +86,6 @@ dependencies:
72
86
  - - "<"
73
87
  - !ruby/object:Gem::Version
74
88
  version: '2'
75
- - !ruby/object:Gem::Dependency
76
- name: azure-storage-blob
77
- requirement: !ruby/object:Gem::Requirement
78
- requirements:
79
- - - "~>"
80
- - !ruby/object:Gem::Version
81
- version: '1.0'
82
- type: :runtime
83
- prerelease: false
84
- version_requirements: !ruby/object:Gem::Requirement
85
- requirements:
86
- - - "~>"
87
- - !ruby/object:Gem::Version
88
- version: '1.0'
89
89
  description: Fluentd plugin to upload logs to Azure Storage append blobs.
90
90
  email:
91
91
  - ''
@@ -93,7 +93,11 @@ executables: []
93
93
  extensions: []
94
94
  extra_rdoc_files: []
95
95
  files:
96
+ - ".github/workflows/gem-push.yml"
97
+ - ".github/workflows/ruby.yml"
96
98
  - ".gitignore"
99
+ - ".rubocop.yml"
100
+ - Dockerfile
97
101
  - Gemfile
98
102
  - LICENSE
99
103
  - README.md
@@ -122,8 +126,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
122
126
  - !ruby/object:Gem::Version
123
127
  version: '0'
124
128
  requirements: []
125
- rubyforge_project:
126
- rubygems_version: 2.7.8
129
+ rubygems_version: 3.0.3
127
130
  signing_key:
128
131
  specification_version: 4
129
132
  summary: Azure Storage Append Blob output plugin for Fluentd event collector