fluent-plugin-azurestorage-gen2 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +58 -0
- data/.travis.yml +14 -0
- data/Gemfile +3 -0
- data/LICENSE +21 -0
- data/Makefile +15 -0
- data/README.md +216 -0
- data/Rakefile +14 -0
- data/VERSION +1 -0
- data/fluent-plugin-azurestorage-gen2.gemspec +27 -0
- data/lib/fluent/plugin/out_azurestorage_gen2.rb +366 -0
- data/test/helper.rb +8 -0
- data/test/plugin/test_out_azurestorage_gen2.rb +54 -0
- metadata +212 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 6f15931cc13f12c4aeb6723ba2a44c604998e819
|
4
|
+
data.tar.gz: 557964c5e84e0274a840af3e18500d9197ea8cd4
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 7d0e97ee7d6b1429e98f91aa72135c4f84bf4cfd8b573cc8ec747cb36fe0e65fb98bce5d02d5416db83a4946c406c93f27ede48a1ba989d25bf675771311e6cd
|
7
|
+
data.tar.gz: 6ed3f231452a94eb0e66244648be66500af2fd82577b3796df11d0cda363238b95b50cc478ae01b54d1e8deb4cdb12101151ccfecb8177f53923db97b395248b
|
data/.gitignore
ADDED
@@ -0,0 +1,58 @@
|
|
1
|
+
*.gem
|
2
|
+
*.rbc
|
3
|
+
/.config
|
4
|
+
/coverage/
|
5
|
+
/InstalledFiles
|
6
|
+
/pkg/
|
7
|
+
/spec/reports/
|
8
|
+
/spec/examples.txt
|
9
|
+
/test/tmp/
|
10
|
+
/test/version_tmp/
|
11
|
+
/tmp/
|
12
|
+
GEMSPEC_ARGS
|
13
|
+
Gemfile.lock
|
14
|
+
|
15
|
+
# Used by dotenv library to load environment variables.
|
16
|
+
# .env
|
17
|
+
|
18
|
+
# Ignore Byebug command history file.
|
19
|
+
.byebug_history
|
20
|
+
|
21
|
+
## Specific to RubyMotion:
|
22
|
+
.dat*
|
23
|
+
.repl_history
|
24
|
+
build/
|
25
|
+
*.bridgesupport
|
26
|
+
build-iPhoneOS/
|
27
|
+
build-iPhoneSimulator/
|
28
|
+
|
29
|
+
## Specific to RubyMotion (use of CocoaPods):
|
30
|
+
#
|
31
|
+
# We recommend against adding the Pods directory to your .gitignore. However
|
32
|
+
# you should judge for yourself, the pros and cons are mentioned at:
|
33
|
+
# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
|
34
|
+
#
|
35
|
+
# vendor/Pods/
|
36
|
+
|
37
|
+
## Documentation cache and generated files:
|
38
|
+
/.yardoc/
|
39
|
+
/_yardoc/
|
40
|
+
/doc/
|
41
|
+
/rdoc/
|
42
|
+
|
43
|
+
## Environment normalization:
|
44
|
+
/.bundle/
|
45
|
+
/vendor/bundle
|
46
|
+
/lib/bundler/man/
|
47
|
+
|
48
|
+
# for a library or gem, you might want to ignore these files since the code is
|
49
|
+
# intended to run in multiple environments; otherwise, check them in:
|
50
|
+
# Gemfile.lock
|
51
|
+
# .ruby-version
|
52
|
+
# .ruby-gemset
|
53
|
+
|
54
|
+
# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
|
55
|
+
.rvmrc
|
56
|
+
|
57
|
+
# Used by RuboCop. Remote config files pulled in from inherit_from directive.
|
58
|
+
# .rubocop-https?--*
|
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
MIT License
|
2
|
+
|
3
|
+
Copyright (c) 2019 Olivér Sz,
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
13
|
+
copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
21
|
+
SOFTWARE.
|
data/Makefile
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
gem:
|
2
|
+
rm -f fluent-plugin-azurestorage-gen2*.gem
|
3
|
+
gem build fluent-plugin-azurestorage-gen2.gemspec
|
4
|
+
|
5
|
+
install: gem
|
6
|
+
gem install fluent-plugin-azurestorage-gen2*.gem
|
7
|
+
|
8
|
+
push: gem
|
9
|
+
gem push fluent-plugin-azurestorage-gen2*.gem
|
10
|
+
|
11
|
+
tag-and-branch:
|
12
|
+
git tag "v$$(cat VERSION)" $(RELEASE_COMMIT)
|
13
|
+
git checkout -b "release/$$(cat VERSION)" $(RELEASE_COMMIT)
|
14
|
+
git push origin "v$$(cat VERSION)"
|
15
|
+
git push -u origin "release/$$(cat VERSION)"
|
data/README.md
ADDED
@@ -0,0 +1,216 @@
|
|
1
|
+
# Azure Datalake Storage Gen2 Fluentd Output Plugin (IN PROGRESS)
|
2
|
+
|
3
|
+
[![Build Status](https://travis-ci.org/oleewere/fluent-plugin-azurestorage-gen2.svg?branch=master)](https://travis-ci.org/oleewere/fluent-plugin-azurestorage-gen2)
|
4
|
+
[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
|
5
|
+
[![Gem Version](https://badge.fury.io/rb/fluent-plugin-azurestorage-gen2.svg)](http://badge.fury.io/rb/fluent-plugin-azurestorage-gen2)
|
6
|
+
![](https://ruby-gem-downloads-badge.herokuapp.com/fluent-plugin-azurestorage-gen2?extension=png)
|
7
|
+
|
8
|
+
## Requirements
|
9
|
+
|
10
|
+
| fluent-plugin-azurestorage-gen2 | fluentd | ruby |
|
11
|
+
|------------------------|---------|------|
|
12
|
+
| >= 0.1.1 | >= v0.14.0 | >= 2.4 |
|
13
|
+
|
14
|
+
## Overview
|
15
|
+
|
16
|
+
Fluent output plugin that can use ABFS api and append blobs with MSI support
|
17
|
+
|
18
|
+
## Installation
|
19
|
+
|
20
|
+
Install from RubyGems:
|
21
|
+
```
|
22
|
+
$ gem install fluent-plugin-azurestorage-gen2
|
23
|
+
```
|
24
|
+
|
25
|
+
## Configuration
|
26
|
+
|
27
|
+
```
|
28
|
+
<match **>
|
29
|
+
@type azurestorage_gen2
|
30
|
+
azure_storage_account mystorageabfs
|
31
|
+
azure_container mycontainer
|
32
|
+
azure_instance_msi /subscriptions/mysubscriptionid
|
33
|
+
azure_object_key_format %{path}-%{index}.%{file_extension}
|
34
|
+
azure_oauth_refresh_interval 3600
|
35
|
+
time_slice_format %Y%m%d-%H
|
36
|
+
file_extension log
|
37
|
+
path "/cluster-logs/myfolder/${tag[1]}-#{Socket.gethostname}-%M"
|
38
|
+
auto_create_container true
|
39
|
+
<buffer tag,time>
|
40
|
+
@type file
|
41
|
+
path /var/log/fluent/azurestorage-buffer
|
42
|
+
timekey 5m
|
43
|
+
timekey_wait 0s
|
44
|
+
timekey_use_utc true
|
45
|
+
chunk_limit_size 64m
|
46
|
+
</buffer>
|
47
|
+
</match>
|
48
|
+
```
|
49
|
+
|
50
|
+
### Configuration options
|
51
|
+
|
52
|
+
### azure_storage_account
|
53
|
+
|
54
|
+
Your Azure Storage Account Name. This can be got from Azure Management potal.
|
55
|
+
This parameter is required when environment variable 'AZURE_STORAGE_ACCOUNT' is not set.
|
56
|
+
|
57
|
+
### azure_storage_access_key (not implemented yet - use msi)
|
58
|
+
|
59
|
+
Your Azure Storage Access Key(Primary or Secondary). This also can be got from Azure Management potal. Storage access key authentication is used when this parameter is provided or environment variable 'AZURE_STORAGE_ACCESS_KEY' is set.
|
60
|
+
|
61
|
+
### azure_instance_msi
|
62
|
+
|
63
|
+
Your Azure Managed Service Identity ID. When storage key authentication is not used, the plugin uses OAuth2 to authenticate as given MSI. This authentication method only works on Azure VM. If the VM has only one MSI assigned, this parameter becomes optional and the only MSI will be used. Otherwise this parameter is required.
|
64
|
+
|
65
|
+
### azure_oauth_refresh_interval
|
66
|
+
|
67
|
+
OAuth2 access token refreshment interval in second. Only applies when MSI authentication is used.
|
68
|
+
|
69
|
+
### azure_container (Required)
|
70
|
+
|
71
|
+
Azure Storage Container name
|
72
|
+
|
73
|
+
### auto_create_container
|
74
|
+
|
75
|
+
This plugin create container if not exist when you set 'auto_create_container' to true.
|
76
|
+
|
77
|
+
### azure_object_key_format
|
78
|
+
|
79
|
+
The format of Azure Storage object keys. You can use several built-in variables:
|
80
|
+
|
81
|
+
- %{path}
|
82
|
+
- %{time_slice}
|
83
|
+
- %{index}
|
84
|
+
- %{file_extension}
|
85
|
+
|
86
|
+
to decide keys dynamically.
|
87
|
+
|
88
|
+
%{path} is exactly the value of *path* configured in the configuration file. E.g., "logs/" in the example configuration above.
|
89
|
+
%{time_slice} is the time-slice in text that are formatted with *time_slice_format*.
|
90
|
+
%{index} is the sequential number starts from 0, increments when multiple files are uploaded to Azure Storage in the same time slice.
|
91
|
+
%{file_extention} is always "gz" for now.
|
92
|
+
|
93
|
+
The default format is "%{path}%{time_slice}_%{index}.%{file_extension}".
|
94
|
+
|
95
|
+
For instance, using the example configuration above, actual object keys on Azure Storage will be something like:
|
96
|
+
|
97
|
+
```
|
98
|
+
"logs/20130111-22_0.gz"
|
99
|
+
"logs/20130111-23_0.gz"
|
100
|
+
"logs/20130111-23_1.gz"
|
101
|
+
"logs/20130112-00_0.gz"
|
102
|
+
```
|
103
|
+
|
104
|
+
With the configuration:
|
105
|
+
|
106
|
+
```
|
107
|
+
azure_object_key_format %{path}/events/ts=%{time_slice}/events_%{index}.%{file_extension}
|
108
|
+
path log
|
109
|
+
time_slice_format %Y%m%d-%H
|
110
|
+
```
|
111
|
+
|
112
|
+
You get:
|
113
|
+
|
114
|
+
```
|
115
|
+
"log/events/ts=20130111-22/events_0.gz"
|
116
|
+
"log/events/ts=20130111-23/events_0.gz"
|
117
|
+
"log/events/ts=20130111-23/events_1.gz"
|
118
|
+
"log/events/ts=20130112-00/events_0.gz"
|
119
|
+
```
|
120
|
+
|
121
|
+
The [fluent-mixin-config-placeholders](https://github.com/tagomoris/fluent-mixin-config-placeholders) mixin is also incorporated, so additional variables such as %{hostname}, %{uuid}, etc. can be used in the azure_object_key_format. This could prove useful in preventing filename conflicts when writing from multiple servers.
|
122
|
+
|
123
|
+
```
|
124
|
+
azure_object_key_format %{path}/events/ts=%{time_slice}/events_%{index}-%{hostname}.%{file_extension}
|
125
|
+
```
|
126
|
+
|
127
|
+
### format
|
128
|
+
|
129
|
+
Change one line format in the Azure Storage object. Supported formats are 'out_file', 'json', 'ltsv' and 'single_value'.
|
130
|
+
|
131
|
+
- out_file (default)
|
132
|
+
|
133
|
+
```
|
134
|
+
time\ttag\t{..json1..}
|
135
|
+
time\ttag\t{..json2..}
|
136
|
+
...
|
137
|
+
```
|
138
|
+
|
139
|
+
- json
|
140
|
+
|
141
|
+
```
|
142
|
+
{..json1..}
|
143
|
+
{..json2..}
|
144
|
+
...
|
145
|
+
```
|
146
|
+
|
147
|
+
At this format, "time" and "tag" are omitted.
|
148
|
+
But you can set these information to the record by setting "include_tag_key" / "tag_key" and "include_time_key" / "time_key" option.
|
149
|
+
If you set following configuration in AzureStorage output:
|
150
|
+
|
151
|
+
```
|
152
|
+
format json
|
153
|
+
include_time_key true
|
154
|
+
time_key log_time # default is time
|
155
|
+
```
|
156
|
+
|
157
|
+
then the record has log_time field.
|
158
|
+
|
159
|
+
```
|
160
|
+
{"log_time":"time string",...}
|
161
|
+
```
|
162
|
+
|
163
|
+
- ltsv
|
164
|
+
|
165
|
+
```
|
166
|
+
key1:value1\tkey2:value2
|
167
|
+
key1:value1\tkey2:value2
|
168
|
+
...
|
169
|
+
```
|
170
|
+
|
171
|
+
"ltsv" format also accepts "include_xxx" related options. See "json" section.
|
172
|
+
|
173
|
+
- single_value
|
174
|
+
|
175
|
+
Use specified value instead of entire recode. If you get '{"message":"my log"}', then contents are
|
176
|
+
|
177
|
+
```
|
178
|
+
my log1
|
179
|
+
my log2
|
180
|
+
...
|
181
|
+
```
|
182
|
+
|
183
|
+
You can change key name by "message_key" option.
|
184
|
+
|
185
|
+
### path
|
186
|
+
|
187
|
+
path prefix of the files on Azure Storage. Default is "" (no prefix).
|
188
|
+
|
189
|
+
### buffer_path (required)
|
190
|
+
|
191
|
+
path prefix of the files to buffer logs.
|
192
|
+
|
193
|
+
### time_slice_format
|
194
|
+
|
195
|
+
Format of the time used as the file name. Default is '%Y%m%d'. Use '%Y%m%d%H' to split files hourly.
|
196
|
+
|
197
|
+
### time_slice_wait
|
198
|
+
|
199
|
+
The time to wait old logs. Default is 10 minutes.
|
200
|
+
|
201
|
+
### utc
|
202
|
+
|
203
|
+
Use UTC instead of local time.
|
204
|
+
|
205
|
+
## TODOs
|
206
|
+
|
207
|
+
- add storage key support
|
208
|
+
- add compression (if append is not used)
|
209
|
+
|
210
|
+
## Contributing
|
211
|
+
|
212
|
+
1. Fork it
|
213
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
214
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
215
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
216
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
|
2
|
+
require 'bundler'
|
3
|
+
Bundler::GemHelper.install_tasks
|
4
|
+
|
5
|
+
require 'rake/testtask'
|
6
|
+
|
7
|
+
Rake::TestTask.new(:test) do |test|
|
8
|
+
test.libs << 'lib' << 'test'
|
9
|
+
test.test_files = FileList['test/plugin/test_*.rb']
|
10
|
+
test.verbose = true
|
11
|
+
end
|
12
|
+
|
13
|
+
task :default => [:build]
|
14
|
+
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.1.1
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
$:.push File.expand_path('../lib', __FILE__)
|
3
|
+
|
4
|
+
Gem::Specification.new do |gem|
|
5
|
+
gem.name = "fluent-plugin-azurestorage-gen2"
|
6
|
+
gem.description = "Azure Storage output plugin for Fluentd event collector"
|
7
|
+
gem.license = "Apache-2.0"
|
8
|
+
gem.homepage = "https://github.com/oleewere/fluent-plugin-azurestorage-gen2"
|
9
|
+
gem.summary = gem.description
|
10
|
+
gem.version = File.read("VERSION").strip
|
11
|
+
gem.authors = ["Oliver Szabo"]
|
12
|
+
gem.email = ["oleewere@gmail.com"]
|
13
|
+
#gem.platform = Gem::Platform::RUBY
|
14
|
+
gem.files = `git ls-files`.split("\n")
|
15
|
+
gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
16
|
+
gem.executables = `git ls-files -- bin/*`.split("\n").map {|f| File.basename(f)}
|
17
|
+
gem.require_paths = ['lib']
|
18
|
+
|
19
|
+
gem.add_runtime_dependency 'fluentd', ['>= 1.0', '< 2']
|
20
|
+
gem.add_runtime_dependency 'uuidtools', '~> 2.1', '>= 2.1.5'
|
21
|
+
gem.add_runtime_dependency 'typhoeus', '~> 1.0', '>= 1.0.1'
|
22
|
+
gem.add_runtime_dependency 'json', '~> 2.1', '>= 2.1.0'
|
23
|
+
gem.add_runtime_dependency "yajl-ruby", '~> 1.4'
|
24
|
+
gem.add_development_dependency 'rake', '~> 12.3', '>= 12.3.1'
|
25
|
+
gem.add_development_dependency 'test-unit', '~> 3.3', '>= 3.3.3'
|
26
|
+
gem.add_development_dependency 'test-unit-rr', '~> 1.0', '>= 1.0.5'
|
27
|
+
end
|
@@ -0,0 +1,366 @@
|
|
1
|
+
require 'net/http'
|
2
|
+
require 'json'
|
3
|
+
require 'tempfile'
|
4
|
+
require 'time'
|
5
|
+
require 'typhoeus'
|
6
|
+
require 'fluent/plugin/output'
|
7
|
+
require 'zlib'
|
8
|
+
|
9
|
+
module Fluent::Plugin
|
10
|
+
class AzureStorageGen2Output < Fluent::Plugin::Output
|
11
|
+
Fluent::Plugin.register_output('azurestorage_gen2', self)
|
12
|
+
|
13
|
+
helpers :compat_parameters, :formatter, :inject
|
14
|
+
|
15
|
+
def initialize
|
16
|
+
super
|
17
|
+
@compressor = nil
|
18
|
+
end
|
19
|
+
|
20
|
+
config_param :path, :string, :default => ""
|
21
|
+
config_param :azure_storage_account, :string, :default => nil
|
22
|
+
config_param :azure_storage_access_key, :string, :default => nil, :secret => true
|
23
|
+
config_param :azure_instance_msi, :string, :default => nil
|
24
|
+
config_param :azure_oauth_refresh_interval, :integer, :default => 60 * 60 # one hour
|
25
|
+
config_param :azure_container, :string, :default => nil
|
26
|
+
config_param :azure_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
|
27
|
+
config_param :file_extension, :string, :default => "log"
|
28
|
+
config_param :store_as, :string, :default => "gzip"
|
29
|
+
config_param :auto_create_container, :bool, :default => false
|
30
|
+
config_param :format, :string, :default => "out_file"
|
31
|
+
config_param :time_slice_format, :string, :default => '%Y%m%d'
|
32
|
+
|
33
|
+
DEFAULT_FORMAT_TYPE = "out_file"
|
34
|
+
URL_DOMAIN_SUFFIX = '.dfs.core.windows.net'
|
35
|
+
ACCESS_TOKEN_API_VERSION = "2018-02-01"
|
36
|
+
ABFS_API_VERSION = "2018-11-09"
|
37
|
+
AZURE_BLOCK_SIZE_LIMIT = 4 * 1024 * 1024 - 1
|
38
|
+
|
39
|
+
config_section :format do
|
40
|
+
config_set_default :@type, DEFAULT_FORMAT_TYPE
|
41
|
+
end
|
42
|
+
|
43
|
+
config_section :buffer do
|
44
|
+
config_set_default :chunk_keys, ['time']
|
45
|
+
config_set_default :timekey, (60 * 60 * 24)
|
46
|
+
end
|
47
|
+
|
48
|
+
def configure(conf)
|
49
|
+
compat_parameters_convert(conf, :buffer, :formatter, :inject)
|
50
|
+
super
|
51
|
+
|
52
|
+
@formatter = formatter_create
|
53
|
+
|
54
|
+
if @localtime
|
55
|
+
@path_slicer = Proc.new {|path|
|
56
|
+
Time.now.strftime(path)
|
57
|
+
}
|
58
|
+
else
|
59
|
+
@path_slicer = Proc.new {|path|
|
60
|
+
Time.now.utc.strftime(path)
|
61
|
+
}
|
62
|
+
end
|
63
|
+
|
64
|
+
if @azure_container.nil?
|
65
|
+
raise Fluent::ConfigError, "azure_container is needed"
|
66
|
+
end
|
67
|
+
|
68
|
+
@azure_storage_path = ''
|
69
|
+
@last_azure_storage_path = ''
|
70
|
+
@current_index = 0
|
71
|
+
end
|
72
|
+
|
73
|
+
def multi_workers_ready?
|
74
|
+
true
|
75
|
+
end
|
76
|
+
|
77
|
+
def start
|
78
|
+
setup_access_token
|
79
|
+
ensure_container
|
80
|
+
super
|
81
|
+
end
|
82
|
+
|
83
|
+
def write(chunk)
|
84
|
+
metadata = chunk.metadata
|
85
|
+
|
86
|
+
#tmp = Tempfile.new("azure-")
|
87
|
+
#begin
|
88
|
+
# tmp.close
|
89
|
+
# generate_log_name(metadata, @current_index)
|
90
|
+
# if @last_azure_storage_path != @azure_storage_path
|
91
|
+
# @current_index = 0
|
92
|
+
# generate_log_name(metadata, @current_index)
|
93
|
+
# end
|
94
|
+
# content = File.open(tmp.path, 'rb') { |file| file.read }
|
95
|
+
# raw_data = raw_data.chomp
|
96
|
+
# log.debug "Content: #{content}"
|
97
|
+
# upload_blob(content, metadata)
|
98
|
+
# @last_azure_storage_path = @azure_storage_path
|
99
|
+
#ensure
|
100
|
+
# tmp.close(true) rescue nil
|
101
|
+
# tmp.unlink
|
102
|
+
#end
|
103
|
+
raw_data=''
|
104
|
+
generate_log_name(metadata, @current_index)
|
105
|
+
if @last_azure_storage_path != @azure_storage_path
|
106
|
+
@current_index = 0
|
107
|
+
generate_log_name(metadata, @current_index)
|
108
|
+
end
|
109
|
+
chunk.each do |emit_time, record|
|
110
|
+
line = record["message"].chomp
|
111
|
+
raw_data << "#{line}\n"
|
112
|
+
end
|
113
|
+
raw_data = raw_data.chomp
|
114
|
+
unless raw_data.empty?
|
115
|
+
upload_blob(raw_data, metadata)
|
116
|
+
end
|
117
|
+
@last_azure_storage_path = @azure_storage_path
|
118
|
+
|
119
|
+
end
|
120
|
+
|
121
|
+
private
|
122
|
+
def upload_blob(content, metadata)
|
123
|
+
log.debug "azurestorage_gen2: Uploading blob: #{@azure_storage_path}"
|
124
|
+
existing_content_length = get_blob_properties(@azure_storage_path)
|
125
|
+
if existing_content_length == 0
|
126
|
+
create_blob(@azure_storage_path)
|
127
|
+
end
|
128
|
+
append_blob(content, metadata, existing_content_length)
|
129
|
+
end
|
130
|
+
|
131
|
+
def format(tag, time, record)
|
132
|
+
r = inject_values_to_record(tag, time, record)
|
133
|
+
@formatter.format(tag, time, r)
|
134
|
+
end
|
135
|
+
|
136
|
+
private
|
137
|
+
def generate_log_name(metadata, index)
|
138
|
+
time_slice = if metadata.timekey.nil?
|
139
|
+
''.freeze
|
140
|
+
else
|
141
|
+
Time.at(metadata.timekey).utc.strftime(@time_slice_format)
|
142
|
+
end
|
143
|
+
path = @path_slicer.call(@path)
|
144
|
+
values_for_object_key = {
|
145
|
+
"%{path}" => path,
|
146
|
+
"%{time_slice}" => time_slice,
|
147
|
+
"%{index}" => index,
|
148
|
+
"%{file_extension}" => @file_extension
|
149
|
+
}
|
150
|
+
storage_path = @azure_object_key_format.gsub(%r(%{[^}]+}), values_for_object_key)
|
151
|
+
extracted_path = extract_placeholders(storage_path, metadata)
|
152
|
+
extracted_path = "/" + extracted_path unless extracted_path.start_with?("/")
|
153
|
+
@azure_storage_path = extracted_path
|
154
|
+
end
|
155
|
+
|
156
|
+
def setup_access_token
|
157
|
+
@get_token_lock = Concurrent::ReadWriteLock.new
|
158
|
+
acquire_access_token
|
159
|
+
if @azure_oauth_refresh_interval > 0
|
160
|
+
log.info("azurestorage_gen2: Start getting access token every #{@azure_oauth_refresh_interval} seconds.")
|
161
|
+
@get_token_task = Concurrent::TimerTask.new(
|
162
|
+
execution_interval: @azure_oauth_refresh_interval) {
|
163
|
+
begin
|
164
|
+
acquire_access_token
|
165
|
+
rescue Exception => e
|
166
|
+
log.warn("#{e.message}, continue with previous credentials.")
|
167
|
+
end
|
168
|
+
}
|
169
|
+
@get_token_task.execute
|
170
|
+
end
|
171
|
+
end
|
172
|
+
|
173
|
+
# Referenced from azure doc.
|
174
|
+
# https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/tutorial-linux-vm-access-storage#get-an-access-token-and-use-it-to-call-azure-storage
|
175
|
+
def acquire_access_token
|
176
|
+
params = { :"api-version" => ACCESS_TOKEN_API_VERSION, :resource => "https://storage.azure.com/" }
|
177
|
+
unless @azure_instance_msi.nil?
|
178
|
+
params[:msi_res_id] = @azure_instance_msi
|
179
|
+
end
|
180
|
+
request = Typhoeus::Request.new("http://169.254.169.254/metadata/identity/oauth2/token", params: params, headers: { Metadata: "true"})
|
181
|
+
request.on_complete do |response|
|
182
|
+
if response.success?
|
183
|
+
data = JSON.parse(response.body)
|
184
|
+
log.debug "azurestorage_gen2: Token response: #{data}"
|
185
|
+
@azure_access_token = data["access_token"]
|
186
|
+
else
|
187
|
+
raise Fluent::UnrecoverableError, "Failed to acquire access token. #{response.code}: #{response.body}"
|
188
|
+
end
|
189
|
+
end
|
190
|
+
request.run
|
191
|
+
end
|
192
|
+
|
193
|
+
private
|
194
|
+
def ensure_container
|
195
|
+
headers = {:"x-ms-version" => ABFS_API_VERSION, :"Authorization" => "Bearer #{@azure_access_token}",:"Content-Length" => "0"}
|
196
|
+
params = {:resource => "filesystem" }
|
197
|
+
request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}", :method => :head, :params => params, :headers=> headers)
|
198
|
+
request.on_complete do |response|
|
199
|
+
if response.success?
|
200
|
+
log.info "azurestorage_gen2: Container '#{@azure_container}' exists."
|
201
|
+
elsif response.timed_out?
|
202
|
+
raise Fluent::UnrecoverableError, "Get container '#{@azure_container}' request timed out."
|
203
|
+
elsif response.code == 404
|
204
|
+
log.info "azurestorage_gen2: Container '#{@azure_container}' does not exist. Creating it if needed..."
|
205
|
+
if @auto_create_container
|
206
|
+
create_container
|
207
|
+
else
|
208
|
+
raise Fluent::ConfigError, "The specified container does not exist: container = #{@azure_container}"
|
209
|
+
end
|
210
|
+
else
|
211
|
+
raise Fluent::UnrecoverableError, "Get container request failed - code: #{response.code}, body: #{response.body}"
|
212
|
+
end
|
213
|
+
end
|
214
|
+
request.run
|
215
|
+
end
|
216
|
+
|
217
|
+
private
|
218
|
+
def create_container
|
219
|
+
headers = {:"x-ms-version" => ABFS_API_VERSION, :"Authorization" => "Bearer #{@azure_access_token}",:"Content-Length" => "0"}
|
220
|
+
params = {:resource => "filesystem" }
|
221
|
+
request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}", :method => :put, :params => params, :headers=> headers)
|
222
|
+
request.on_complete do |response|
|
223
|
+
if response.success?
|
224
|
+
log.debug "azurestorage_gen2: Container '#{@azure_container}' created, response code: #{response.code}"
|
225
|
+
elsif response.timed_out?
|
226
|
+
raise Fluent::UnrecoverableError, "Creating container '#{@azure_container}' request timed out."
|
227
|
+
else
|
228
|
+
raise Fluent::UnrecoverableError, "Creating container request failed - code: #{response.code}, body: #{response.body}"
|
229
|
+
end
|
230
|
+
end
|
231
|
+
request.run
|
232
|
+
end
|
233
|
+
|
234
|
+
private
|
235
|
+
def create_blob(blob_path)
|
236
|
+
headers = {:"x-ms-version" => ABFS_API_VERSION, :"Authorization" => "Bearer #{@azure_access_token}",:"Content-Length" => "0", :"Content-Type" => "application/json"}
|
237
|
+
params = {:resource => "file", :recursive => "false"}
|
238
|
+
request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :put, :params => params, :headers=> headers)
|
239
|
+
request.on_complete do |response|
|
240
|
+
if response.success?
|
241
|
+
log.debug "azurestorage_gen2: Blob '#{blob_path}' has been created, response code: #{response.code}"
|
242
|
+
elsif response.timed_out?
|
243
|
+
raise Fluent::UnrecoverableError, "Creating blob '#{blob_path}' request timed out."
|
244
|
+
elsif response.code == 409
|
245
|
+
log.debug "azurestorage_gen2: Blob already exists: #{blob_path}"
|
246
|
+
else
|
247
|
+
raise Fluent::UnrecoverableError, "Creating blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}"
|
248
|
+
end
|
249
|
+
end
|
250
|
+
request.run
|
251
|
+
end
|
252
|
+
|
253
|
+
private
|
254
|
+
def append_blob_block(blob_path, content, position)
|
255
|
+
log.debug "azurestorage_gen2: append_blob_block.start: Append blob ('#{blob_path}') called with position #{position}"
|
256
|
+
headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-content-type" => "text/plain", :"Authorization" => "Bearer #{@azure_access_token}"}
|
257
|
+
params = {:action => "append", :position => "#{position}"}
|
258
|
+
request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :patch, :body => content, :params => params, :headers=> headers)
|
259
|
+
request.on_complete do |response|
|
260
|
+
if response.success?
|
261
|
+
log.debug "azurestorage_gen2: Blob '#{blob_path}' has been appended, response code: #{response.code}"
|
262
|
+
elsif response.timed_out?
|
263
|
+
raise Fluent::UnrecoverableError, "Appending blob #{blob_path}' request timed out."
|
264
|
+
elsif response.code == 404
|
265
|
+
raise AppendBlobResponseError.new("Blob '#{blob_path}' has not found. Error code: #{response.code}", 404)
|
266
|
+
elsif response.code == 409
|
267
|
+
raise AppendBlobResponseError.new("Blob '#{blob_path}' has conflict. Error code: #{response.code}", 409)
|
268
|
+
else
|
269
|
+
raise Fluent::UnrecoverableError, "Appending blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}"
|
270
|
+
end
|
271
|
+
end
|
272
|
+
request.run
|
273
|
+
end
|
274
|
+
|
275
|
+
private
|
276
|
+
def flush(blob_path, position)
|
277
|
+
log.debug "azurestorage_gen2: flush_blob.start: Flush blob ('#{blob_path}') called with position #{position}"
|
278
|
+
headers = {:"x-ms-version" => ABFS_API_VERSION, :"Authorization" => "Bearer #{@azure_access_token}",:"Content-Length" => "0"}
|
279
|
+
params = {:action => "flush", :position => "#{position}"}
|
280
|
+
request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :patch, :params => params, :headers=> headers)
|
281
|
+
request.on_complete do |response|
|
282
|
+
if response.success?
|
283
|
+
log.debug "azurestorage_gen2: Blob '#{blob_path}' flush was successful, response code: #{response.code}"
|
284
|
+
elsif response.timed_out?
|
285
|
+
raise Fluent::UnrecoverableError, "Bloub '#{blob_path}' flush request timed out."
|
286
|
+
else
|
287
|
+
raise Fluent::UnrecoverableError, "Blob flush request failed - code: #{response.code}, body: #{response.body}"
|
288
|
+
end
|
289
|
+
end
|
290
|
+
request.run
|
291
|
+
end
|
292
|
+
|
293
|
+
private
|
294
|
+
def get_blob_properties(blob_path)
|
295
|
+
headers = {:"x-ms-version" => ABFS_API_VERSION, :"Authorization" => "Bearer #{@azure_access_token}",:"Content-Length" => "0"}
|
296
|
+
params = {}
|
297
|
+
content_length = -1
|
298
|
+
request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :head, :params => params, :headers=> headers)
|
299
|
+
request.on_complete do |response|
|
300
|
+
if response.success?
|
301
|
+
log.debug "azurestorage_gen2: Get blob properties for '#{blob_path}', response headers: #{response.headers}"
|
302
|
+
content_length = response.headers['Content-Length'].to_i
|
303
|
+
elsif response.timed_out?
|
304
|
+
raise Fluent::UnrecoverableError, "Get blob properties '#{blob_path}' request timed out."
|
305
|
+
elsif response.code == 404
|
306
|
+
log.debug "azurestorage_gen2: Blob '#{blob_path}' does not exist. Creating it if needed..."
|
307
|
+
content_length = 0
|
308
|
+
else
|
309
|
+
raise Fluent::UnrecoverableError, "Get blob properties '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}"
|
310
|
+
end
|
311
|
+
end
|
312
|
+
request.run
|
313
|
+
content_length
|
314
|
+
end
|
315
|
+
|
316
|
+
private
|
317
|
+
def append_blob(content, metadata, existing_content_length)
|
318
|
+
position = 0
|
319
|
+
log.debug "azurestorage_gen2: append_blob.start: Content size: #{content.length}"
|
320
|
+
loop do
|
321
|
+
begin
|
322
|
+
size = [content.length - position, AZURE_BLOCK_SIZE_LIMIT].min
|
323
|
+
log.debug "azurestorage_gen2: append_blob.chunk: content[#{position}..#{position + size}]"
|
324
|
+
append_blob_block(@azure_storage_path, content[position..position + size], existing_content_length)
|
325
|
+
|
326
|
+
position += size
|
327
|
+
existing_content_length += size
|
328
|
+
break if position >= content.length
|
329
|
+
rescue AppendBlobResponseError => ex
|
330
|
+
status_code = ex.status_code
|
331
|
+
|
332
|
+
if status_code == 409 # exceeds azure block limit
|
333
|
+
@current_index += 1
|
334
|
+
old_azure_storage_path = @azure_storage_path
|
335
|
+
generate_log_name(metadata, @current_index)
|
336
|
+
|
337
|
+
# If index is not a part of format, rethrow exception.
|
338
|
+
if old_azure_storage_path == @azure_storage_path
|
339
|
+
log.warn "azurestorage_gen2: append_blob: blocks limit reached, you need to use %{index} for the format."
|
340
|
+
raise
|
341
|
+
end
|
342
|
+
flush(old_azure_storage_path, existing_content_length)
|
343
|
+
|
344
|
+
log.info "azurestorage_gen2: append_blob: blocks limit reached, creating new blob #{@azure_storage_path}."
|
345
|
+
create_blob(@azure_storage_path)
|
346
|
+
elsif status_code == 404 # blob not found
|
347
|
+
log.debug "azurestorage_gen2: append_blob: #{@azure_storage_path} blob doesn't exist, creating new blob."
|
348
|
+
create_blob(@azure_storage_path)
|
349
|
+
else
|
350
|
+
raise
|
351
|
+
end
|
352
|
+
end
|
353
|
+
end
|
354
|
+
flush(@azure_storage_path, existing_content_length)
|
355
|
+
log.debug "azurestorage_gen2: append_blob.complete"
|
356
|
+
end
|
357
|
+
end
|
358
|
+
|
359
|
+
class AppendBlobResponseError < StandardError
|
360
|
+
attr_reader :status_code
|
361
|
+
def initialize(message="Default message", status_code=0)
|
362
|
+
@status_code = status_code
|
363
|
+
super(message)
|
364
|
+
end
|
365
|
+
end
|
366
|
+
end
|
data/test/helper.rb
ADDED
@@ -0,0 +1,8 @@
|
|
1
|
+
$LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
|
2
|
+
require "test-unit"
|
3
|
+
require "fluent/test"
|
4
|
+
require "fluent/test/driver/output"
|
5
|
+
require "fluent/test/helpers"
|
6
|
+
|
7
|
+
Test::Unit::TestCase.include(Fluent::Test::Helpers)
|
8
|
+
Test::Unit::TestCase.extend(Fluent::Test::Helpers)
|
@@ -0,0 +1,54 @@
|
|
1
|
+
require 'fluent/test'
|
2
|
+
require 'fluent/test/driver/output'
|
3
|
+
require 'fluent/test/helpers'
|
4
|
+
require 'fluent/plugin/out_azurestorage_gen2'
|
5
|
+
|
6
|
+
require 'test/unit/rr'
|
7
|
+
require 'zlib'
|
8
|
+
require 'fileutils'
|
9
|
+
|
10
|
+
include Fluent::Test::Helpers
|
11
|
+
|
12
|
+
class AzureStorageGen2OutputTest < Test::Unit::TestCase
|
13
|
+
def setup
|
14
|
+
Fluent::Test.setup
|
15
|
+
end
|
16
|
+
|
17
|
+
CONFIG = %[
|
18
|
+
azure_storage_account test_storage_account
|
19
|
+
azure_storage_access_key dGVzdF9zdG9yYWdlX2FjY2Vzc19rZXk=
|
20
|
+
azure_container test_container
|
21
|
+
path log
|
22
|
+
utc
|
23
|
+
buffer_type memory
|
24
|
+
]
|
25
|
+
|
26
|
+
def create_driver(conf = CONFIG)
|
27
|
+
Fluent::Test::Driver::Output.new(Fluent::Plugin::AzureStorageGen2Output) do
|
28
|
+
# for testing.
|
29
|
+
def contents
|
30
|
+
@emit_streams
|
31
|
+
end
|
32
|
+
|
33
|
+
def write(chunk)
|
34
|
+
@emit_streams = []
|
35
|
+
event = chunk.read
|
36
|
+
@emit_streams << event
|
37
|
+
end
|
38
|
+
|
39
|
+
private
|
40
|
+
def ensure_container
|
41
|
+
end
|
42
|
+
|
43
|
+
end.configure(conf)
|
44
|
+
end
|
45
|
+
|
46
|
+
def test_configure
|
47
|
+
# TODO write tests
|
48
|
+
d = create_driver
|
49
|
+
assert_equal 'test_storage_account', d.instance.azure_storage_account
|
50
|
+
assert_equal 'dGVzdF9zdG9yYWdlX2FjY2Vzc19rZXk=', d.instance.azure_storage_access_key
|
51
|
+
assert_equal 'test_container', d.instance.azure_container
|
52
|
+
assert_equal 'log', d.instance.path
|
53
|
+
end
|
54
|
+
end
|
metadata
ADDED
@@ -0,0 +1,212 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-azurestorage-gen2
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Oliver Szabo
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2019-12-16 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: fluentd
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.0'
|
20
|
+
- - "<"
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: '2'
|
23
|
+
type: :runtime
|
24
|
+
prerelease: false
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
26
|
+
requirements:
|
27
|
+
- - ">="
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '1.0'
|
30
|
+
- - "<"
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '2'
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
name: uuidtools
|
35
|
+
requirement: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - "~>"
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: '2.1'
|
40
|
+
- - ">="
|
41
|
+
- !ruby/object:Gem::Version
|
42
|
+
version: 2.1.5
|
43
|
+
type: :runtime
|
44
|
+
prerelease: false
|
45
|
+
version_requirements: !ruby/object:Gem::Requirement
|
46
|
+
requirements:
|
47
|
+
- - "~>"
|
48
|
+
- !ruby/object:Gem::Version
|
49
|
+
version: '2.1'
|
50
|
+
- - ">="
|
51
|
+
- !ruby/object:Gem::Version
|
52
|
+
version: 2.1.5
|
53
|
+
- !ruby/object:Gem::Dependency
|
54
|
+
name: typhoeus
|
55
|
+
requirement: !ruby/object:Gem::Requirement
|
56
|
+
requirements:
|
57
|
+
- - "~>"
|
58
|
+
- !ruby/object:Gem::Version
|
59
|
+
version: '1.0'
|
60
|
+
- - ">="
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: 1.0.1
|
63
|
+
type: :runtime
|
64
|
+
prerelease: false
|
65
|
+
version_requirements: !ruby/object:Gem::Requirement
|
66
|
+
requirements:
|
67
|
+
- - "~>"
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '1.0'
|
70
|
+
- - ">="
|
71
|
+
- !ruby/object:Gem::Version
|
72
|
+
version: 1.0.1
|
73
|
+
- !ruby/object:Gem::Dependency
|
74
|
+
name: json
|
75
|
+
requirement: !ruby/object:Gem::Requirement
|
76
|
+
requirements:
|
77
|
+
- - "~>"
|
78
|
+
- !ruby/object:Gem::Version
|
79
|
+
version: '2.1'
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: 2.1.0
|
83
|
+
type: :runtime
|
84
|
+
prerelease: false
|
85
|
+
version_requirements: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '2.1'
|
90
|
+
- - ">="
|
91
|
+
- !ruby/object:Gem::Version
|
92
|
+
version: 2.1.0
|
93
|
+
- !ruby/object:Gem::Dependency
|
94
|
+
name: yajl-ruby
|
95
|
+
requirement: !ruby/object:Gem::Requirement
|
96
|
+
requirements:
|
97
|
+
- - "~>"
|
98
|
+
- !ruby/object:Gem::Version
|
99
|
+
version: '1.4'
|
100
|
+
type: :runtime
|
101
|
+
prerelease: false
|
102
|
+
version_requirements: !ruby/object:Gem::Requirement
|
103
|
+
requirements:
|
104
|
+
- - "~>"
|
105
|
+
- !ruby/object:Gem::Version
|
106
|
+
version: '1.4'
|
107
|
+
- !ruby/object:Gem::Dependency
|
108
|
+
name: rake
|
109
|
+
requirement: !ruby/object:Gem::Requirement
|
110
|
+
requirements:
|
111
|
+
- - "~>"
|
112
|
+
- !ruby/object:Gem::Version
|
113
|
+
version: '12.3'
|
114
|
+
- - ">="
|
115
|
+
- !ruby/object:Gem::Version
|
116
|
+
version: 12.3.1
|
117
|
+
type: :development
|
118
|
+
prerelease: false
|
119
|
+
version_requirements: !ruby/object:Gem::Requirement
|
120
|
+
requirements:
|
121
|
+
- - "~>"
|
122
|
+
- !ruby/object:Gem::Version
|
123
|
+
version: '12.3'
|
124
|
+
- - ">="
|
125
|
+
- !ruby/object:Gem::Version
|
126
|
+
version: 12.3.1
|
127
|
+
- !ruby/object:Gem::Dependency
|
128
|
+
name: test-unit
|
129
|
+
requirement: !ruby/object:Gem::Requirement
|
130
|
+
requirements:
|
131
|
+
- - "~>"
|
132
|
+
- !ruby/object:Gem::Version
|
133
|
+
version: '3.3'
|
134
|
+
- - ">="
|
135
|
+
- !ruby/object:Gem::Version
|
136
|
+
version: 3.3.3
|
137
|
+
type: :development
|
138
|
+
prerelease: false
|
139
|
+
version_requirements: !ruby/object:Gem::Requirement
|
140
|
+
requirements:
|
141
|
+
- - "~>"
|
142
|
+
- !ruby/object:Gem::Version
|
143
|
+
version: '3.3'
|
144
|
+
- - ">="
|
145
|
+
- !ruby/object:Gem::Version
|
146
|
+
version: 3.3.3
|
147
|
+
- !ruby/object:Gem::Dependency
|
148
|
+
name: test-unit-rr
|
149
|
+
requirement: !ruby/object:Gem::Requirement
|
150
|
+
requirements:
|
151
|
+
- - "~>"
|
152
|
+
- !ruby/object:Gem::Version
|
153
|
+
version: '1.0'
|
154
|
+
- - ">="
|
155
|
+
- !ruby/object:Gem::Version
|
156
|
+
version: 1.0.5
|
157
|
+
type: :development
|
158
|
+
prerelease: false
|
159
|
+
version_requirements: !ruby/object:Gem::Requirement
|
160
|
+
requirements:
|
161
|
+
- - "~>"
|
162
|
+
- !ruby/object:Gem::Version
|
163
|
+
version: '1.0'
|
164
|
+
- - ">="
|
165
|
+
- !ruby/object:Gem::Version
|
166
|
+
version: 1.0.5
|
167
|
+
description: Azure Storage output plugin for Fluentd event collector
|
168
|
+
email:
|
169
|
+
- oleewere@gmail.com
|
170
|
+
executables: []
|
171
|
+
extensions: []
|
172
|
+
extra_rdoc_files: []
|
173
|
+
files:
|
174
|
+
- ".gitignore"
|
175
|
+
- ".travis.yml"
|
176
|
+
- Gemfile
|
177
|
+
- LICENSE
|
178
|
+
- Makefile
|
179
|
+
- README.md
|
180
|
+
- Rakefile
|
181
|
+
- VERSION
|
182
|
+
- fluent-plugin-azurestorage-gen2.gemspec
|
183
|
+
- lib/fluent/plugin/out_azurestorage_gen2.rb
|
184
|
+
- test/helper.rb
|
185
|
+
- test/plugin/test_out_azurestorage_gen2.rb
|
186
|
+
homepage: https://github.com/oleewere/fluent-plugin-azurestorage-gen2
|
187
|
+
licenses:
|
188
|
+
- Apache-2.0
|
189
|
+
metadata: {}
|
190
|
+
post_install_message:
|
191
|
+
rdoc_options: []
|
192
|
+
require_paths:
|
193
|
+
- lib
|
194
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
195
|
+
requirements:
|
196
|
+
- - ">="
|
197
|
+
- !ruby/object:Gem::Version
|
198
|
+
version: '0'
|
199
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
200
|
+
requirements:
|
201
|
+
- - ">="
|
202
|
+
- !ruby/object:Gem::Version
|
203
|
+
version: '0'
|
204
|
+
requirements: []
|
205
|
+
rubyforge_project:
|
206
|
+
rubygems_version: 2.5.1
|
207
|
+
signing_key:
|
208
|
+
specification_version: 4
|
209
|
+
summary: Azure Storage output plugin for Fluentd event collector
|
210
|
+
test_files:
|
211
|
+
- test/helper.rb
|
212
|
+
- test/plugin/test_out_azurestorage_gen2.rb
|