fluent-plugin-azure-storage-append-blob 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/gem-push.yml +27 -0
- data/.github/workflows/ruby.yml +23 -0
- data/.gitignore +1 -1
- data/.rubocop.yml +2 -0
- data/Dockerfile +36 -0
- data/Gemfile +0 -0
- data/LICENSE +0 -0
- data/README.md +93 -61
- data/Rakefile +4 -4
- data/azure-pipelines.yml +0 -0
- data/fluent-plugin-azure-storage-append-blob.gemspec +18 -18
- data/lib/fluent/plugin/out_azure-storage-append-blob.rb +107 -58
- data/test/helper.rb +5 -5
- data/test/plugin/test_out_azure_storage_append_blob.rb +30 -10
- metadata +25 -22
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7640e2e4b681f3d556b69dfd6cce4483c5af1e3a98a92b9261c5a2d9e1365277
|
4
|
+
data.tar.gz: 1fbc5232cbe8e50f6518fc41460e94ad59f9187d36fa8f3faf4800b5490b46d3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cbc94f676ec13fec572539c98b82f200f6ef11659ca6745720eefcd2d71c7b34ac9287a9175b1a0bc1c84e6d70feaf7ceea5df8ad5c3af7600fffb52237356c2
|
7
|
+
data.tar.gz: 1989910ce48aaffd875d4226c097e85d25c04cea12dbcc9c4cfad13560a52ae1406675544be27bf02981fc20f78e4da88bf07d3ffd58fcc7744f7a0ddaeeee44
|
@@ -0,0 +1,27 @@
|
|
1
|
+
name: Ruby Gem
|
2
|
+
|
3
|
+
on:
|
4
|
+
workflow_dispatch:
|
5
|
+
|
6
|
+
jobs:
|
7
|
+
build:
|
8
|
+
name: Build + Publish
|
9
|
+
runs-on: ubuntu-latest
|
10
|
+
|
11
|
+
steps:
|
12
|
+
- uses: actions/checkout@v2
|
13
|
+
- name: Set up Ruby 2.6
|
14
|
+
uses: actions/setup-ruby@v1
|
15
|
+
with:
|
16
|
+
ruby-version: 2.6.x
|
17
|
+
|
18
|
+
- name: Publish to RubyGems
|
19
|
+
run: |
|
20
|
+
mkdir -p $HOME/.gem
|
21
|
+
touch $HOME/.gem/credentials
|
22
|
+
chmod 0600 $HOME/.gem/credentials
|
23
|
+
printf -- "---\n:rubygems_api_key: ${RUBYGEMS_API_KEY}\n" > $HOME/.gem/credentials
|
24
|
+
gem build *.gemspec
|
25
|
+
gem push *.gem
|
26
|
+
env:
|
27
|
+
RUBYGEMS_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"
|
@@ -0,0 +1,23 @@
|
|
1
|
+
name: Unittests
|
2
|
+
|
3
|
+
on:
|
4
|
+
push:
|
5
|
+
branches: [ master ]
|
6
|
+
pull_request:
|
7
|
+
branches: [ master ]
|
8
|
+
|
9
|
+
jobs:
|
10
|
+
test:
|
11
|
+
|
12
|
+
runs-on: ubuntu-latest
|
13
|
+
|
14
|
+
steps:
|
15
|
+
- uses: actions/checkout@v2
|
16
|
+
- name: Set up Ruby
|
17
|
+
uses: ruby/setup-ruby@v1
|
18
|
+
with:
|
19
|
+
ruby-version: 2.6
|
20
|
+
- name: Install dependencies
|
21
|
+
run: bundle install
|
22
|
+
- name: Run tests
|
23
|
+
run: bundle exec rake test
|
data/.gitignore
CHANGED
data/.rubocop.yml
ADDED
data/Dockerfile
ADDED
@@ -0,0 +1,36 @@
|
|
1
|
+
FROM ruby:latest
|
2
|
+
|
3
|
+
WORKDIR /plugin
|
4
|
+
|
5
|
+
ADD . /plugin
|
6
|
+
|
7
|
+
RUN gem install bundler && \
|
8
|
+
gem install fluentd --no-doc && \
|
9
|
+
fluent-gem build fluent-plugin-azure-storage-append-blob.gemspec && \
|
10
|
+
fluent-gem install fluent-plugin-azure-storage-append-blob-*.gem
|
11
|
+
|
12
|
+
RUN echo "<source>\n\
|
13
|
+
@type sample\n\
|
14
|
+
sample {\"hello\":\"world\"}\n\
|
15
|
+
tag pattern\n\
|
16
|
+
</source>\n\
|
17
|
+
<match pattern>\n\
|
18
|
+
@type azure-storage-append-blob\n\
|
19
|
+
azure_storage_account \"#{ENV['STORAGE_ACCOUNT']}\"\n\
|
20
|
+
azure_storage_access_key \"#{ENV['STORAGE_ACCESS_KEY']}\"\n\
|
21
|
+
azure_storage_sas_token \"#{ENV['STORAGE_SAS_TOKEN']}\"\n\
|
22
|
+
azure_container fluentd\n\
|
23
|
+
auto_create_container true\n\
|
24
|
+
path logs/\n\
|
25
|
+
azure_object_key_format %{path}%{time_slice}_%{index}.log\n\
|
26
|
+
time_slice_format %Y%m%d-%H\n\
|
27
|
+
<buffer tag,time>\n\
|
28
|
+
@type file\n\
|
29
|
+
path /var/log/fluent/azurestorageappendblob\n\
|
30
|
+
timekey 120 # 2 minutes\n\
|
31
|
+
timekey_wait 60\n\
|
32
|
+
timekey_use_utc true # use utc\n\
|
33
|
+
</buffer>\n\
|
34
|
+
</match>" > /plugin/fluent.conf
|
35
|
+
|
36
|
+
ENTRYPOINT ["fluentd", "-c", "fluent.conf"]
|
data/Gemfile
CHANGED
File without changes
|
data/LICENSE
CHANGED
File without changes
|
data/README.md
CHANGED
@@ -8,67 +8,79 @@ Azure Storage Append Blob output plugin buffers logs in local file and uploads t
|
|
8
8
|
|
9
9
|
### RubyGems
|
10
10
|
|
11
|
-
|
12
|
-
$ gem install fluent-plugin-azure-storage-append-blob
|
13
|
-
```
|
11
|
+
gem install fluent-plugin-azure-storage-append-blob
|
14
12
|
|
15
13
|
### Bundler
|
16
14
|
|
17
15
|
Add following line to your Gemfile:
|
18
16
|
|
19
|
-
|
20
|
-
gem "fluent-plugin-azure-storage-append-blob"
|
21
|
-
```
|
17
|
+
gem "fluent-plugin-azure-storage-append-blob"
|
22
18
|
|
23
19
|
And then execute:
|
24
20
|
|
25
|
-
|
26
|
-
$ bundle
|
27
|
-
```
|
21
|
+
bundle
|
28
22
|
|
29
23
|
## Configuration
|
30
24
|
|
31
|
-
|
32
|
-
|
33
|
-
type azure-storage-append-blob
|
25
|
+
<match pattern>
|
26
|
+
type azure-storage-append-blob
|
34
27
|
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
28
|
+
azure_storage_account <your azure storage account>
|
29
|
+
azure_storage_access_key <your azure storage access key> # leave empty to use MSI
|
30
|
+
azure_storage_sas_token <your azure storage sas token> # leave empty to use MSI
|
31
|
+
azure_imds_api_version <Azure Instance Metadata Service API Version> # only used for MSI
|
32
|
+
azure_token_refresh_interval <refresh interval in min> # only used for MSI
|
33
|
+
azure_container <your azure storage container>
|
34
|
+
auto_create_container true
|
35
|
+
path logs/
|
36
|
+
azure_object_key_format %{path}%{time_slice}_%{index}.log
|
37
|
+
time_slice_format %Y%m%d-%H
|
38
|
+
# if you want to use %{tag} or %Y/%m/%d/ like syntax in path / azure_blob_name_format,
|
39
|
+
# need to specify tag for %{tag} and time for %Y/%m/%d in <buffer> argument.
|
40
|
+
<buffer tag,time>
|
41
|
+
@type file
|
42
|
+
path /var/log/fluent/azurestorageappendblob
|
43
|
+
timekey 120 # 2 minutes
|
44
|
+
timekey_wait 60
|
45
|
+
timekey_use_utc true # use utc
|
46
|
+
</buffer>
|
47
|
+
</match>
|
53
48
|
|
54
|
-
### azure_storage_account (Required)
|
49
|
+
### `azure_storage_account` (Required)
|
55
50
|
|
56
|
-
Your Azure Storage Account Name. This can be retrieved from Azure Management
|
51
|
+
Your Azure Storage Account Name. This can be retrieved from Azure Management portal.
|
57
52
|
|
58
|
-
### azure_storage_access_key (
|
53
|
+
### `azure_storage_access_key` or `azure_storage_sas_token` (Either required or both empty to use MSI)
|
59
54
|
|
60
|
-
Your Azure Storage Access Key(Primary or Secondary)
|
55
|
+
Your Azure Storage Access Key (Primary or Secondary) or shared access signature (SAS) token.
|
56
|
+
This also can be retrieved from Azure Management portal.
|
61
57
|
|
62
|
-
|
58
|
+
If both are empty, the plugin will use the local Managed Identity endpoint to obtain a token for the target storage account.
|
59
|
+
|
60
|
+
### `azure_imds_api_version` (Optional, only for MSI)
|
61
|
+
|
62
|
+
Default: 2019-08-15
|
63
|
+
|
64
|
+
The Instance Metadata Service is used during the OAuth flow to obtain an access token. This API is versioned and specifying the version is mandatory.
|
65
|
+
|
66
|
+
See [here](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/instance-metadata-service#versioning) for more details.
|
67
|
+
|
68
|
+
### `azure_token_refresh_interval` (Optional, only for MSI)
|
69
|
+
|
70
|
+
Default: 60 (1 hour)
|
71
|
+
|
72
|
+
When using MSI, the initial access token needs to be refreshed periodically.
|
73
|
+
|
74
|
+
### `azure_container` (Required)
|
63
75
|
|
64
76
|
Azure Storage Container name
|
65
77
|
|
66
|
-
### auto_create_container
|
78
|
+
### `auto_create_container`
|
67
79
|
|
68
80
|
This plugin creates the Azure container if it does not already exist exist when you set 'auto_create_container' to true.
|
69
81
|
The default value is `true`
|
70
82
|
|
71
|
-
### azure_object_key_format
|
83
|
+
### `azure_object_key_format`
|
72
84
|
|
73
85
|
The format of Azure Storage object keys. You can use several built-in variables:
|
74
86
|
|
@@ -86,48 +98,68 @@ The default format is "%{path}%{time_slice}-%{index}.log".
|
|
86
98
|
|
87
99
|
For instance, using the example configuration above, actual object keys on Azure Storage will be something like:
|
88
100
|
|
89
|
-
|
90
|
-
"logs/20130111-
|
91
|
-
"logs/
|
92
|
-
"logs/20130112-00-0.log"
|
93
|
-
```
|
101
|
+
"logs/20130111-22-0.log"
|
102
|
+
"logs/20130111-23-0.log"
|
103
|
+
"logs/20130112-00-0.log"
|
94
104
|
|
95
105
|
With the configuration:
|
96
106
|
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
time_slice_format %Y%m%d-%H
|
101
|
-
```
|
107
|
+
azure_object_key_format %{path}/events/ts=%{time_slice}/events.log
|
108
|
+
path log
|
109
|
+
time_slice_format %Y%m%d-%H
|
102
110
|
|
103
111
|
You get:
|
104
112
|
|
105
|
-
|
106
|
-
"log/events/ts=20130111-
|
107
|
-
"log/events/ts=
|
108
|
-
"log/events/ts=20130112-00/events.log"
|
109
|
-
```
|
113
|
+
"log/events/ts=20130111-22/events.log"
|
114
|
+
"log/events/ts=20130111-23/events.log"
|
115
|
+
"log/events/ts=20130112-00/events.log"
|
110
116
|
|
111
117
|
The [fluent-mixin-config-placeholders](https://github.com/tagomoris/fluent-mixin-config-placeholders) mixin is also incorporated, so additional variables such as %{hostname}, etc. can be used in the `azure_object_key_format`. This is useful in preventing filename conflicts when writing from multiple servers.
|
112
118
|
|
113
|
-
|
114
|
-
azure_object_key_format %{path}/events/ts=%{time_slice}/events-%{hostname}.log
|
115
|
-
```
|
119
|
+
azure_object_key_format %{path}/events/ts=%{time_slice}/events-%{hostname}.log
|
116
120
|
|
117
|
-
### time_slice_format
|
121
|
+
### `time_slice_format`
|
118
122
|
|
119
123
|
Format of the time used in the file name. Default is '%Y%m%d'. Use '%Y%m%d%H' to split files hourly.
|
120
124
|
|
121
125
|
### Run tests
|
122
|
-
$ gem install bundler
|
123
|
-
$ bundle install
|
124
|
-
$ bundle exec rake test
|
125
126
|
|
126
|
-
|
127
|
+
gem install bundler
|
128
|
+
bundle install
|
129
|
+
bundle exec rake test
|
130
|
+
|
131
|
+
|
132
|
+
### Test Fluentd
|
133
|
+
|
134
|
+
1. Create Storage Account and VM with enabled MSI
|
135
|
+
2. Setup Docker ang Git
|
136
|
+
3. SSH into VM
|
137
|
+
4. Download this repo
|
138
|
+
```
|
139
|
+
git clone https://github.com/microsoft/fluent-plugin-azure-storage-append-blob.git
|
140
|
+
cd fluent-plugin-azure-storage-append-blob
|
141
|
+
```
|
142
|
+
5. Build Docker image
|
143
|
+
`docker build -t fluent .`
|
144
|
+
6. Run Docker image with different set of parameters:
|
145
|
+
|
146
|
+
1. `STORAGE_ACCOUNT`: required, name of your storage account
|
147
|
+
2. `STORAGE_ACCESS_KEY`: storage account access key
|
148
|
+
3. `STORAGE_SAS_TOKEN`: storage sas token with enough permissions for the plugin
|
149
|
+
|
150
|
+
You need to specify `STORAGE_ACCOUNT` and one of auth ways. If you run it from VM with MSI,
|
151
|
+
just `STORAGE_ACCOUNT` is required. Keep in mind, there is no way to refresh MSI Token, so
|
152
|
+
ensure you setup proper permissions first.
|
153
|
+
|
154
|
+
```bash
|
155
|
+
docker run -it -e STORAGE_ACCOUNT=<storage> -e STORAGE_ACCESS_KEY=<key> fluent
|
156
|
+
```
|
157
|
+
|
158
|
+
## Contributing
|
127
159
|
|
128
160
|
This project welcomes contributions and suggestions. Most contributions require you to agree to a
|
129
161
|
Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
|
130
|
-
the rights to use your contribution. For details, visit https://cla.microsoft.com.
|
162
|
+
the rights to use your contribution. For details, visit [https://cla.microsoft.com](https://cla.microsoft.com).
|
131
163
|
|
132
164
|
When you submit a pull request, a CLA-bot will automatically determine whether you need to provide
|
133
165
|
a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions
|
data/Rakefile
CHANGED
@@ -1,11 +1,11 @@
|
|
1
|
-
require
|
1
|
+
require 'bundler'
|
2
2
|
Bundler::GemHelper.install_tasks
|
3
3
|
|
4
|
-
require
|
4
|
+
require 'rake/testtask'
|
5
5
|
|
6
6
|
Rake::TestTask.new(:test) do |t|
|
7
|
-
t.libs.push(
|
8
|
-
t.test_files = FileList[
|
7
|
+
t.libs.push('lib', 'test')
|
8
|
+
t.test_files = FileList['test/**/test_*.rb']
|
9
9
|
t.verbose = true
|
10
10
|
t.warning = true
|
11
11
|
end
|
data/azure-pipelines.yml
CHANGED
File without changes
|
@@ -2,27 +2,27 @@ lib = File.expand_path("../lib", __FILE__)
|
|
2
2
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
3
3
|
|
4
4
|
Gem::Specification.new do |spec|
|
5
|
-
spec.name
|
6
|
-
spec.version =
|
7
|
-
spec.authors = [
|
8
|
-
spec.email
|
5
|
+
spec.name = 'fluent-plugin-azure-storage-append-blob'
|
6
|
+
spec.version = '0.2.0'
|
7
|
+
spec.authors = ['Microsoft Corporation']
|
8
|
+
spec.email = ['']
|
9
9
|
|
10
|
-
spec.summary
|
11
|
-
spec.description
|
12
|
-
spec.homepage
|
13
|
-
spec.license
|
10
|
+
spec.summary = 'Azure Storage Append Blob output plugin for Fluentd event collector'
|
11
|
+
spec.description = 'Fluentd plugin to upload logs to Azure Storage append blobs.'
|
12
|
+
spec.homepage = 'https://github.com/Microsoft/fluent-plugin-azure-storage-append-blob'
|
13
|
+
spec.license = 'MIT'
|
14
14
|
|
15
|
-
test_files, files
|
15
|
+
test_files, files = `git ls-files -z`.split("\x0").partition do |f|
|
16
16
|
f.match(%r{^(test|spec|features)/})
|
17
17
|
end
|
18
|
-
spec.files
|
19
|
-
spec.executables
|
20
|
-
spec.test_files
|
21
|
-
spec.require_paths = [
|
18
|
+
spec.files = files
|
19
|
+
spec.executables = files.grep(%r{^bin/}) { |f| File.basename(f) }
|
20
|
+
spec.test_files = test_files
|
21
|
+
spec.require_paths = ['lib']
|
22
22
|
|
23
|
-
spec.add_development_dependency
|
24
|
-
spec.add_development_dependency
|
25
|
-
spec.add_development_dependency
|
26
|
-
spec.add_runtime_dependency
|
27
|
-
spec.add_runtime_dependency
|
23
|
+
spec.add_development_dependency 'bundler', '~> 2.0'
|
24
|
+
spec.add_development_dependency 'rake', '~> 13.0'
|
25
|
+
spec.add_development_dependency 'test-unit', '~> 3.0'
|
26
|
+
spec.add_runtime_dependency 'azure-storage-blob', '~> 2.0'
|
27
|
+
spec.add_runtime_dependency 'fluentd', ['>= 0.14.10', '< 2']
|
28
28
|
end
|
@@ -3,90 +3,133 @@
|
|
3
3
|
# Licensed under the MIT License. See License.txt in the project root for license information.
|
4
4
|
#--------------------------------------------------------------------------------------------*/
|
5
5
|
|
6
|
-
require '
|
6
|
+
require 'azure/storage/common'
|
7
7
|
require 'azure/storage/blob'
|
8
|
-
require '
|
9
|
-
require '
|
8
|
+
require 'faraday'
|
9
|
+
require 'fluent/plugin/output'
|
10
|
+
require 'json'
|
10
11
|
|
11
12
|
module Fluent
|
12
13
|
module Plugin
|
13
14
|
class AzureStorageAppendBlobOut < Fluent::Plugin::Output
|
14
|
-
Fluent::Plugin.register_output(
|
15
|
+
Fluent::Plugin.register_output('azure-storage-append-blob', self)
|
15
16
|
|
16
17
|
helpers :formatter, :inject
|
17
18
|
|
18
|
-
DEFAULT_FORMAT_TYPE =
|
19
|
-
|
20
|
-
config_param :path, :string, :default => ""
|
21
|
-
config_param :azure_storage_account, :string, :default => nil
|
22
|
-
config_param :azure_storage_access_key, :string, :default => nil, :secret => true
|
23
|
-
config_param :azure_container, :string, :default => nil
|
24
|
-
config_param :azure_object_key_format, :string, :default => "%{path}%{time_slice}-%{index}.log"
|
25
|
-
config_param :auto_create_container, :bool, :default => true
|
26
|
-
config_param :format, :string, :default => DEFAULT_FORMAT_TYPE
|
27
|
-
config_param :time_slice_format, :string, :default => '%Y%m%d'
|
28
|
-
|
29
|
-
DEFAULT_FORMAT_TYPE = "out_file"
|
19
|
+
DEFAULT_FORMAT_TYPE = 'out_file'.freeze
|
30
20
|
AZURE_BLOCK_SIZE_LIMIT = 4 * 1024 * 1024 - 1
|
31
|
-
|
21
|
+
|
22
|
+
config_param :path, :string, default: ''
|
23
|
+
config_param :azure_storage_account, :string, default: nil
|
24
|
+
config_param :azure_storage_access_key, :string, default: nil, secret: true
|
25
|
+
config_param :azure_storage_sas_token, :string, default: nil, secret: true
|
26
|
+
config_param :azure_container, :string, default: nil
|
27
|
+
config_param :azure_imds_api_version, :string, default: '2019-08-15'
|
28
|
+
config_param :azure_token_refresh_interval, :integer, default: 60
|
29
|
+
config_param :use_msi, :bool, default: false
|
30
|
+
config_param :azure_object_key_format, :string, default: '%{path}%{time_slice}-%{index}.log'
|
31
|
+
config_param :auto_create_container, :bool, default: true
|
32
|
+
config_param :format, :string, default: DEFAULT_FORMAT_TYPE
|
33
|
+
config_param :time_slice_format, :string, default: '%Y%m%d'
|
34
|
+
config_param :localtime, :bool, default: false
|
35
|
+
|
32
36
|
config_section :format do
|
33
37
|
config_set_default :@type, DEFAULT_FORMAT_TYPE
|
34
38
|
end
|
35
|
-
|
39
|
+
|
36
40
|
config_section :buffer do
|
37
41
|
config_set_default :chunk_keys, ['time']
|
38
42
|
config_set_default :timekey, (60 * 60 * 24)
|
39
43
|
end
|
40
|
-
|
44
|
+
|
41
45
|
attr_reader :bs
|
42
|
-
|
46
|
+
|
43
47
|
def configure(conf)
|
44
48
|
super
|
45
|
-
|
49
|
+
|
46
50
|
@formatter = formatter_create
|
47
|
-
|
48
|
-
if @localtime
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
if @
|
59
|
-
|
51
|
+
|
52
|
+
@path_slicer = if @localtime
|
53
|
+
proc do |path|
|
54
|
+
Time.now.strftime(path)
|
55
|
+
end
|
56
|
+
else
|
57
|
+
proc do |path|
|
58
|
+
Time.now.utc.strftime(path)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
raise ConfigError, 'azure_storage_account needs to be specified' if @azure_storage_account.nil?
|
63
|
+
|
64
|
+
raise ConfigError, 'azure_container needs to be specified' if @azure_container.nil?
|
65
|
+
|
66
|
+
if (@azure_storage_access_key.nil? || @azure_storage_access_key.empty?) && (@azure_storage_sas_token.nil? || @azure_storage_sas_token.empty?)
|
67
|
+
log.info 'Using MSI since neither azure_storage_access_key nor azure_storage_sas_token was provided.'
|
68
|
+
@use_msi = true
|
60
69
|
end
|
61
70
|
end
|
62
|
-
|
71
|
+
|
63
72
|
def multi_workers_ready?
|
64
73
|
true
|
65
74
|
end
|
66
|
-
|
75
|
+
|
76
|
+
def get_access_token
|
77
|
+
access_key_request = Faraday.new('http://169.254.169.254/metadata/identity/oauth2/token?' \
|
78
|
+
"api-version=#{@azure_imds_api_version}" \
|
79
|
+
'&resource=https://storage.azure.com/',
|
80
|
+
headers: { 'Metadata' => 'true' })
|
81
|
+
.get
|
82
|
+
.body
|
83
|
+
JSON.parse(access_key_request)['access_token']
|
84
|
+
end
|
85
|
+
|
67
86
|
def start
|
68
87
|
super
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
88
|
+
if @use_msi
|
89
|
+
token_credential = Azure::Storage::Common::Core::TokenCredential.new get_access_token
|
90
|
+
token_signer = Azure::Storage::Common::Core::Auth::TokenSigner.new token_credential
|
91
|
+
@bs = Azure::Storage::Blob::BlobService.new(storage_account_name: @azure_storage_account, signer: token_signer)
|
92
|
+
|
93
|
+
refresh_interval = @azure_token_refresh_interval * 60
|
94
|
+
cancelled = false
|
95
|
+
renew_token = Thread.new do
|
96
|
+
Thread.stop
|
97
|
+
until cancelled
|
98
|
+
sleep(refresh_interval)
|
73
99
|
|
100
|
+
token_credential.renew_token get_access_token
|
101
|
+
end
|
102
|
+
end
|
103
|
+
sleep 0.1 while renew_token.status != 'sleep'
|
104
|
+
renew_token.run
|
105
|
+
else
|
106
|
+
@bs_params = { storage_account_name: @azure_storage_account }
|
107
|
+
|
108
|
+
if !@azure_storage_access_key.nil? && !@azure_storage_access_key.empty?
|
109
|
+
@bs_params.merge!({ storage_access_key: @azure_storage_access_key })
|
110
|
+
elsif !@azure_storage_sas_token.nil? && !@azure_storage_sas_token.empty?
|
111
|
+
@bs_params.merge!({ storage_sas_token: @azure_storage_sas_token })
|
112
|
+
end
|
113
|
+
|
114
|
+
@bs = Azure::Storage::Blob::BlobService.create(@bs_params)
|
115
|
+
end
|
116
|
+
|
117
|
+
ensure_container
|
74
118
|
@azure_storage_path = ''
|
75
119
|
@last_azure_storage_path = ''
|
76
120
|
@current_index = 0
|
77
121
|
end
|
78
|
-
|
122
|
+
|
79
123
|
def format(tag, time, record)
|
80
124
|
r = inject_values_to_record(tag, time, record)
|
81
125
|
@formatter.format(tag, time, r)
|
82
126
|
end
|
83
|
-
|
127
|
+
|
84
128
|
def write(chunk)
|
85
129
|
metadata = chunk.metadata
|
86
|
-
tmp = Tempfile.new(
|
130
|
+
tmp = Tempfile.new('azure-')
|
87
131
|
begin
|
88
132
|
chunk.write_to(tmp)
|
89
|
-
tmp.close
|
90
133
|
|
91
134
|
generate_log_name(metadata, @current_index)
|
92
135
|
if @last_azure_storage_path != @azure_storage_path
|
@@ -94,18 +137,23 @@ module Fluent
|
|
94
137
|
generate_log_name(metadata, @current_index)
|
95
138
|
end
|
96
139
|
|
97
|
-
content = File.open(tmp.path, 'rb'
|
140
|
+
content = File.open(tmp.path, 'rb', &:read)
|
98
141
|
|
99
142
|
append_blob(content, metadata)
|
100
143
|
@last_azure_storage_path = @azure_storage_path
|
101
144
|
ensure
|
102
|
-
|
145
|
+
begin
|
146
|
+
tmp.close(true)
|
147
|
+
rescue StandardError
|
148
|
+
nil
|
149
|
+
end
|
103
150
|
end
|
104
151
|
end
|
105
|
-
|
152
|
+
|
106
153
|
private
|
154
|
+
|
107
155
|
def ensure_container
|
108
|
-
|
156
|
+
unless @bs.list_containers.find { |c| c.name == @azure_container }
|
109
157
|
if @auto_create_container
|
110
158
|
@bs.create_container(@azure_container)
|
111
159
|
else
|
@@ -115,24 +163,26 @@ module Fluent
|
|
115
163
|
end
|
116
164
|
|
117
165
|
private
|
166
|
+
|
118
167
|
def generate_log_name(metadata, index)
|
119
168
|
time_slice = if metadata.timekey.nil?
|
120
169
|
''.freeze
|
121
170
|
else
|
122
171
|
Time.at(metadata.timekey).utc.strftime(@time_slice_format)
|
123
|
-
|
172
|
+
end
|
124
173
|
|
125
174
|
path = @path_slicer.call(@path)
|
126
175
|
values_for_object_key = {
|
127
|
-
|
128
|
-
|
129
|
-
|
176
|
+
'%{path}' => path,
|
177
|
+
'%{time_slice}' => time_slice,
|
178
|
+
'%{index}' => index
|
130
179
|
}
|
131
|
-
storage_path = @azure_object_key_format.gsub(
|
180
|
+
storage_path = @azure_object_key_format.gsub(/%{[^}]+}/, values_for_object_key)
|
132
181
|
@azure_storage_path = extract_placeholders(storage_path, metadata)
|
133
182
|
end
|
134
183
|
|
135
184
|
private
|
185
|
+
|
136
186
|
def append_blob(content, metadata)
|
137
187
|
position = 0
|
138
188
|
log.debug "azure_storage_append_blob: append_blob.start: Content size: #{content.length}"
|
@@ -143,8 +193,8 @@ module Fluent
|
|
143
193
|
@bs.append_blob_block(@azure_container, @azure_storage_path, content[position..position + size])
|
144
194
|
position += size
|
145
195
|
break if position >= content.length
|
146
|
-
rescue Azure::Core::Http::HTTPError =>
|
147
|
-
status_code =
|
196
|
+
rescue Azure::Core::Http::HTTPError => e
|
197
|
+
status_code = e.status_code
|
148
198
|
|
149
199
|
if status_code == 409 # exceeds azure block limit
|
150
200
|
@current_index += 1
|
@@ -153,7 +203,7 @@ module Fluent
|
|
153
203
|
|
154
204
|
# If index is not a part of format, rethrow exception.
|
155
205
|
if old_azure_storage_path == @azure_storage_path
|
156
|
-
log.warn
|
206
|
+
log.warn 'azure_storage_append_blob: append_blob: blocks limit reached, you need to use %{index} for the format.'
|
157
207
|
raise
|
158
208
|
end
|
159
209
|
|
@@ -167,9 +217,8 @@ module Fluent
|
|
167
217
|
end
|
168
218
|
end
|
169
219
|
end
|
170
|
-
log.debug
|
220
|
+
log.debug 'azure_storage_append_blob: append_blob.complete'
|
171
221
|
end
|
172
|
-
|
173
222
|
end
|
174
223
|
end
|
175
224
|
end
|
data/test/helper.rb
CHANGED
@@ -1,8 +1,8 @@
|
|
1
|
-
$LOAD_PATH.unshift(File.expand_path(
|
2
|
-
require
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
1
|
+
$LOAD_PATH.unshift(File.expand_path('..', __dir__))
|
2
|
+
require 'test-unit'
|
3
|
+
require 'fluent/test'
|
4
|
+
require 'fluent/test/driver/output'
|
5
|
+
require 'fluent/test/helpers'
|
6
6
|
|
7
7
|
Test::Unit::TestCase.include(Fluent::Test::Helpers)
|
8
8
|
Test::Unit::TestCase.extend(Fluent::Test::Helpers)
|
@@ -8,32 +8,41 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
|
|
8
8
|
Fluent::Test.setup
|
9
9
|
end
|
10
10
|
|
11
|
-
CONFIG = %
|
11
|
+
CONFIG = %(
|
12
12
|
azure_storage_account test_storage_account
|
13
13
|
azure_storage_access_key MY_FAKE_SECRET
|
14
14
|
azure_container test_container
|
15
15
|
time_slice_format %Y%m%d-%H
|
16
16
|
path log
|
17
|
-
|
17
|
+
).freeze
|
18
18
|
|
19
|
-
|
19
|
+
MSI_CONFIG = %(
|
20
|
+
azure_storage_account test_storage_account
|
21
|
+
azure_container test_container
|
22
|
+
azure_imds_api_version 1970-01-01
|
23
|
+
azure_token_refresh_interval 120
|
24
|
+
time_slice_format %Y%m%d-%H
|
25
|
+
path log
|
26
|
+
).freeze
|
27
|
+
|
28
|
+
def create_driver(conf = CONFIG)
|
20
29
|
Fluent::Test::Driver::Output.new(Fluent::Plugin::AzureStorageAppendBlobOut).configure(conf)
|
21
30
|
end
|
22
31
|
|
23
32
|
sub_test_case 'test config' do
|
24
33
|
test 'config should reject with no azure container' do
|
25
34
|
assert_raise Fluent::ConfigError do
|
26
|
-
create_driver(%
|
35
|
+
create_driver(%(
|
27
36
|
azure_storage_account test_storage_account
|
28
37
|
azure_storage_access_key MY_FAKE_SECRET
|
29
38
|
time_slice_format %Y%m%d-%H
|
30
39
|
time_slice_wait 10m
|
31
40
|
path log
|
32
|
-
|
41
|
+
))
|
33
42
|
end
|
34
43
|
end
|
35
44
|
|
36
|
-
|
45
|
+
test 'config with access key should set instance variables' do
|
37
46
|
d = create_driver
|
38
47
|
assert_equal 'test_storage_account', d.instance.azure_storage_account
|
39
48
|
assert_equal 'MY_FAKE_SECRET', d.instance.azure_storage_access_key
|
@@ -41,26 +50,37 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
|
|
41
50
|
assert_equal true, d.instance.auto_create_container
|
42
51
|
assert_equal '%{path}%{time_slice}-%{index}.log', d.instance.azure_object_key_format
|
43
52
|
end
|
53
|
+
|
54
|
+
test 'config with managed identity enabled should set instance variables' do
|
55
|
+
d = create_driver(MSI_CONFIG)
|
56
|
+
assert_equal 'test_storage_account', d.instance.azure_storage_account
|
57
|
+
assert_equal 'test_container', d.instance.azure_container
|
58
|
+
assert_equal true, d.instance.use_msi
|
59
|
+
assert_equal true, d.instance.auto_create_container
|
60
|
+
assert_equal '%{path}%{time_slice}-%{index}.log', d.instance.azure_object_key_format
|
61
|
+
assert_equal 120, d.instance.azure_token_refresh_interval
|
62
|
+
assert_equal '1970-01-01', d.instance.azure_imds_api_version
|
63
|
+
end
|
44
64
|
end
|
45
65
|
|
46
66
|
sub_test_case 'test path slicing' do
|
47
67
|
test 'test path_slicing' do
|
48
|
-
config = CONFIG.clone.gsub(/path\slog/,
|
68
|
+
config = CONFIG.clone.gsub(/path\slog/, 'path log/%Y/%m/%d')
|
49
69
|
d = create_driver(config)
|
50
70
|
path_slicer = d.instance.instance_variable_get(:@path_slicer)
|
51
71
|
path = d.instance.instance_variable_get(:@path)
|
52
72
|
slice = path_slicer.call(path)
|
53
|
-
assert_equal slice, Time.now.utc.strftime(
|
73
|
+
assert_equal slice, Time.now.utc.strftime('log/%Y/%m/%d')
|
54
74
|
end
|
55
75
|
|
56
76
|
test 'path slicing utc' do
|
57
|
-
config = CONFIG.clone.gsub(/path\slog/,
|
77
|
+
config = CONFIG.clone.gsub(/path\slog/, 'path log/%Y/%m/%d')
|
58
78
|
config << "\nutc\n"
|
59
79
|
d = create_driver(config)
|
60
80
|
path_slicer = d.instance.instance_variable_get(:@path_slicer)
|
61
81
|
path = d.instance.instance_variable_get(:@path)
|
62
82
|
slice = path_slicer.call(path)
|
63
|
-
assert_equal slice, Time.now.utc.strftime(
|
83
|
+
assert_equal slice, Time.now.utc.strftime('log/%Y/%m/%d')
|
64
84
|
end
|
65
85
|
end
|
66
86
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-azure-storage-append-blob
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Microsoft Corporation
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-08-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -16,28 +16,28 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '
|
19
|
+
version: '2.0'
|
20
20
|
type: :development
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '
|
26
|
+
version: '2.0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: rake
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
33
|
+
version: '13.0'
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: '
|
40
|
+
version: '13.0'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: test-unit
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -52,6 +52,20 @@ dependencies:
|
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '3.0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: azure-storage-blob
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '2.0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '2.0'
|
55
69
|
- !ruby/object:Gem::Dependency
|
56
70
|
name: fluentd
|
57
71
|
requirement: !ruby/object:Gem::Requirement
|
@@ -72,20 +86,6 @@ dependencies:
|
|
72
86
|
- - "<"
|
73
87
|
- !ruby/object:Gem::Version
|
74
88
|
version: '2'
|
75
|
-
- !ruby/object:Gem::Dependency
|
76
|
-
name: azure-storage-blob
|
77
|
-
requirement: !ruby/object:Gem::Requirement
|
78
|
-
requirements:
|
79
|
-
- - "~>"
|
80
|
-
- !ruby/object:Gem::Version
|
81
|
-
version: '1.0'
|
82
|
-
type: :runtime
|
83
|
-
prerelease: false
|
84
|
-
version_requirements: !ruby/object:Gem::Requirement
|
85
|
-
requirements:
|
86
|
-
- - "~>"
|
87
|
-
- !ruby/object:Gem::Version
|
88
|
-
version: '1.0'
|
89
89
|
description: Fluentd plugin to upload logs to Azure Storage append blobs.
|
90
90
|
email:
|
91
91
|
- ''
|
@@ -93,7 +93,11 @@ executables: []
|
|
93
93
|
extensions: []
|
94
94
|
extra_rdoc_files: []
|
95
95
|
files:
|
96
|
+
- ".github/workflows/gem-push.yml"
|
97
|
+
- ".github/workflows/ruby.yml"
|
96
98
|
- ".gitignore"
|
99
|
+
- ".rubocop.yml"
|
100
|
+
- Dockerfile
|
97
101
|
- Gemfile
|
98
102
|
- LICENSE
|
99
103
|
- README.md
|
@@ -122,8 +126,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
122
126
|
- !ruby/object:Gem::Version
|
123
127
|
version: '0'
|
124
128
|
requirements: []
|
125
|
-
|
126
|
-
rubygems_version: 2.7.8
|
129
|
+
rubygems_version: 3.0.3
|
127
130
|
signing_key:
|
128
131
|
specification_version: 4
|
129
132
|
summary: Azure Storage Append Blob output plugin for Fluentd event collector
|