fluent-plugin-azure-storage-append-blob 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +50 -0
- data/Gemfile +3 -0
- data/LICENSE +21 -0
- data/README.md +138 -0
- data/Rakefile +13 -0
- data/fluent-plugin-azure-storage-append-blob.gemspec +28 -0
- data/lib/fluent/plugin/out_azure-storage-append-blob.rb +175 -0
- data/test/helper.rb +8 -0
- data/test/plugin/test_out_azure_storage_append_blob.rb +66 -0
- metadata +131 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 86a7a8f0c9164564d064640c1db1ccf3f97a3d0fb99fb6c14c65cdc0a104619c
|
4
|
+
data.tar.gz: c144705806010e9e6feb1d7bd84070156f413139a80ffdebad9325c639112329
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: e8ff1cf315f5f330f1d202db8cd0ca74ba77f41cdf11f961926f97eb9be8a3ecd963cba06ad7a8bad20fc4151e454f396ccc2c5b1568251127e99e1e4b2416ab
|
7
|
+
data.tar.gz: c27e7414feec0113b1d3b7bcc613c0a351e3eaac60e37bca3f8f04b9c66248f92c7b8f2a81ea71de375f62008d5f6d40cf29937829dd3a2e4c45d3b27fa1c6f1
|
data/.gitignore
ADDED
@@ -0,0 +1,50 @@
|
|
1
|
+
*.gem
|
2
|
+
*.rbc
|
3
|
+
/.config
|
4
|
+
/coverage/
|
5
|
+
/InstalledFiles
|
6
|
+
/pkg/
|
7
|
+
/spec/reports/
|
8
|
+
/spec/examples.txt
|
9
|
+
/test/tmp/
|
10
|
+
/test/version_tmp/
|
11
|
+
/tmp/
|
12
|
+
|
13
|
+
# Used by dotenv library to load environment variables.
|
14
|
+
# .env
|
15
|
+
|
16
|
+
## Specific to RubyMotion:
|
17
|
+
.dat*
|
18
|
+
.repl_history
|
19
|
+
build/
|
20
|
+
*.bridgesupport
|
21
|
+
build-iPhoneOS/
|
22
|
+
build-iPhoneSimulator/
|
23
|
+
|
24
|
+
## Specific to RubyMotion (use of CocoaPods):
|
25
|
+
#
|
26
|
+
# We recommend against adding the Pods directory to your .gitignore. However
|
27
|
+
# you should judge for yourself, the pros and cons are mentioned at:
|
28
|
+
# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
|
29
|
+
#
|
30
|
+
# vendor/Pods/
|
31
|
+
|
32
|
+
## Documentation cache and generated files:
|
33
|
+
/.yardoc/
|
34
|
+
/_yardoc/
|
35
|
+
/doc/
|
36
|
+
/rdoc/
|
37
|
+
|
38
|
+
## Environment normalization:
|
39
|
+
/.bundle/
|
40
|
+
/vendor/bundle
|
41
|
+
/lib/bundler/man/
|
42
|
+
|
43
|
+
# for a library or gem, you might want to ignore these files since the code is
|
44
|
+
# intended to run in multiple environments; otherwise, check them in:
|
45
|
+
# Gemfile.lock
|
46
|
+
# .ruby-version
|
47
|
+
# .ruby-gemset
|
48
|
+
|
49
|
+
# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
|
50
|
+
.rvmrc
|
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
MIT License
|
2
|
+
|
3
|
+
Copyright (c) Microsoft Corporation. All rights reserved.
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
13
|
+
copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
21
|
+
SOFTWARE
|
data/README.md
ADDED
@@ -0,0 +1,138 @@
|
|
1
|
+
# fluent-plugin-azure-storage-append-blob
|
2
|
+
|
3
|
+
[Fluentd](https://fluentd.org/) out plugin to do something.
|
4
|
+
|
5
|
+
Azure Storage Append Blob output plugin buffers logs in local file and uploads them to Azure Storage Append Blob periodically.
|
6
|
+
|
7
|
+
## Installation
|
8
|
+
|
9
|
+
### RubyGems
|
10
|
+
|
11
|
+
```
|
12
|
+
$ gem install fluent-plugin-azure-storage-append-blob
|
13
|
+
```
|
14
|
+
|
15
|
+
### Bundler
|
16
|
+
|
17
|
+
Add following line to your Gemfile:
|
18
|
+
|
19
|
+
```ruby
|
20
|
+
gem "fluent-plugin-azure-storage-append-blob"
|
21
|
+
```
|
22
|
+
|
23
|
+
And then execute:
|
24
|
+
|
25
|
+
```
|
26
|
+
$ bundle
|
27
|
+
```
|
28
|
+
|
29
|
+
## Configuration
|
30
|
+
|
31
|
+
```
|
32
|
+
<match pattern>
|
33
|
+
type azure-storage-append-blob
|
34
|
+
|
35
|
+
azure_storage_account <your azure storage account>
|
36
|
+
azure_storage_access_key <your azure storage access key>
|
37
|
+
azure_container <your azure storage container>
|
38
|
+
auto_create_container true
|
39
|
+
path logs/
|
40
|
+
azure_blob_name_format %{path}%{time_slice}_%{index}.log
|
41
|
+
time_slice_format %Y%m%d-%H
|
42
|
+
# if you want to use %{tag} or %Y/%m/%d/ like syntax in path / azure_blob_name_format,
|
43
|
+
# need to specify tag for %{tag} and time for %Y/%m/%d in <buffer> argument.
|
44
|
+
<buffer tag,time>
|
45
|
+
@type file
|
46
|
+
path /var/log/fluent/azurestorageappendblob
|
47
|
+
timekey 120 # 2 minutes
|
48
|
+
timekey_wait 60
|
49
|
+
timekey_use_utc true # use utc
|
50
|
+
</buffer>
|
51
|
+
</match>
|
52
|
+
```
|
53
|
+
|
54
|
+
### azure_storage_account (Required)
|
55
|
+
|
56
|
+
Your Azure Storage Account Name. This can be retrieved from Azure Management potal.
|
57
|
+
|
58
|
+
### azure_storage_access_key (Required)
|
59
|
+
|
60
|
+
Your Azure Storage Access Key(Primary or Secondary). This also can be retrieved from Azure Management potal.
|
61
|
+
|
62
|
+
### azure_container (Required)
|
63
|
+
|
64
|
+
Azure Storage Container name
|
65
|
+
|
66
|
+
### auto_create_container
|
67
|
+
|
68
|
+
This plugin creates the Azure container if it does not already exist exist when you set 'auto_create_container' to true.
|
69
|
+
The default value is `true`
|
70
|
+
|
71
|
+
### azure_object_key_format
|
72
|
+
|
73
|
+
The format of Azure Storage object keys. You can use several built-in variables:
|
74
|
+
|
75
|
+
- %{path}
|
76
|
+
- %{time_slice}
|
77
|
+
- %{index}
|
78
|
+
|
79
|
+
to decide keys dynamically.
|
80
|
+
|
81
|
+
%{path} is exactly the value of *path* configured in the configuration file. E.g., "logs/" in the example configuration above.
|
82
|
+
%{time_slice} is the time-slice in text that are formatted with *time_slice_format*.
|
83
|
+
%{index} is used only if your blob exceed Azure 50000 blocks limit per blob to prevent data loss. Its not required to use this parameter.
|
84
|
+
|
85
|
+
The default format is "%{path}%{time_slice}-%{index}.log".
|
86
|
+
|
87
|
+
For instance, using the example configuration above, actual object keys on Azure Storage will be something like:
|
88
|
+
|
89
|
+
```
|
90
|
+
"logs/20130111-22-0.log"
|
91
|
+
"logs/20130111-23-0.log"
|
92
|
+
"logs/20130112-00-0.log"
|
93
|
+
```
|
94
|
+
|
95
|
+
With the configuration:
|
96
|
+
|
97
|
+
```
|
98
|
+
azure_object_key_format %{path}/events/ts=%{time_slice}/events.log
|
99
|
+
path log
|
100
|
+
time_slice_format %Y%m%d-%H
|
101
|
+
```
|
102
|
+
|
103
|
+
You get:
|
104
|
+
|
105
|
+
```
|
106
|
+
"log/events/ts=20130111-22/events.log"
|
107
|
+
"log/events/ts=20130111-23/events.log"
|
108
|
+
"log/events/ts=20130112-00/events.log"
|
109
|
+
```
|
110
|
+
|
111
|
+
The [fluent-mixin-config-placeholders](https://github.com/tagomoris/fluent-mixin-config-placeholders) mixin is also incorporated, so additional variables such as %{hostname}, etc. can be used in the `azure_object_key_format`. This is useful in preventing filename conflicts when writing from multiple servers.
|
112
|
+
|
113
|
+
```
|
114
|
+
azure_object_key_format %{path}/events/ts=%{time_slice}/events-%{hostname}.log
|
115
|
+
```
|
116
|
+
|
117
|
+
### time_slice_format
|
118
|
+
|
119
|
+
Format of the time used in the file name. Default is '%Y%m%d'. Use '%Y%m%d%H' to split files hourly.
|
120
|
+
|
121
|
+
### Run tests
|
122
|
+
$ gem install bundler
|
123
|
+
$ bundle install
|
124
|
+
$ bundle exec rake test
|
125
|
+
|
126
|
+
# Contributing
|
127
|
+
|
128
|
+
This project welcomes contributions and suggestions. Most contributions require you to agree to a
|
129
|
+
Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
|
130
|
+
the rights to use your contribution. For details, visit https://cla.microsoft.com.
|
131
|
+
|
132
|
+
When you submit a pull request, a CLA-bot will automatically determine whether you need to provide
|
133
|
+
a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions
|
134
|
+
provided by the bot. You will only need to do this once across all repos using our CLA.
|
135
|
+
|
136
|
+
This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
|
137
|
+
For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
|
138
|
+
contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
|
data/Rakefile
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
require "bundler"
|
2
|
+
Bundler::GemHelper.install_tasks
|
3
|
+
|
4
|
+
require "rake/testtask"
|
5
|
+
|
6
|
+
Rake::TestTask.new(:test) do |t|
|
7
|
+
t.libs.push("lib", "test")
|
8
|
+
t.test_files = FileList["test/**/test_*.rb"]
|
9
|
+
t.verbose = true
|
10
|
+
t.warning = true
|
11
|
+
end
|
12
|
+
|
13
|
+
task default: [:test]
|
@@ -0,0 +1,28 @@
|
|
1
|
+
lib = File.expand_path("../lib", __FILE__)
|
2
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
3
|
+
|
4
|
+
Gem::Specification.new do |spec|
|
5
|
+
spec.name = "fluent-plugin-azure-storage-append-blob"
|
6
|
+
spec.version = "0.1.0"
|
7
|
+
spec.authors = ["Microsoft"]
|
8
|
+
spec.email = [""]
|
9
|
+
|
10
|
+
spec.summary = "Azure Storage Append Blob output plugin for Fluentd event collector"
|
11
|
+
spec.description = "Fluentd plugin to upload logs to Azure Storage append blobs."
|
12
|
+
spec.homepage = "https://github.com/Microsoft/fluent-plugin-azure-storage-append-blob"
|
13
|
+
spec.license = "MIT"
|
14
|
+
|
15
|
+
test_files, files = `git ls-files -z`.split("\x0").partition do |f|
|
16
|
+
f.match(%r{^(test|spec|features)/})
|
17
|
+
end
|
18
|
+
spec.files = files
|
19
|
+
spec.executables = files.grep(%r{^bin/}) { |f| File.basename(f) }
|
20
|
+
spec.test_files = test_files
|
21
|
+
spec.require_paths = ["lib"]
|
22
|
+
|
23
|
+
spec.add_development_dependency "bundler", "~> 1.14"
|
24
|
+
spec.add_development_dependency "rake", "~> 12.0"
|
25
|
+
spec.add_development_dependency "test-unit", "~> 3.0"
|
26
|
+
spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
|
27
|
+
spec.add_runtime_dependency "azure-storage-blob", "~> 1.0"
|
28
|
+
end
|
@@ -0,0 +1,175 @@
|
|
1
|
+
#---------------------------------------------------------------------------------------------
|
2
|
+
# Copyright (c) Microsoft Corporation. All rights reserved.
|
3
|
+
# Licensed under the MIT License. See License.txt in the project root for license information.
|
4
|
+
#--------------------------------------------------------------------------------------------*/
|
5
|
+
|
6
|
+
require 'fluent/plugin/output'
|
7
|
+
require 'azure/storage/blob'
|
8
|
+
require 'time'
|
9
|
+
require 'tempfile'
|
10
|
+
|
11
|
+
module Fluent
|
12
|
+
module Plugin
|
13
|
+
class AzureStorageAppendBlobOut < Fluent::Plugin::Output
|
14
|
+
Fluent::Plugin.register_output("azure-storage-append-blob", self)
|
15
|
+
|
16
|
+
helpers :formatter, :inject
|
17
|
+
|
18
|
+
DEFAULT_FORMAT_TYPE = "out_file"
|
19
|
+
|
20
|
+
config_param :path, :string, :default => ""
|
21
|
+
config_param :azure_storage_account, :string, :default => nil
|
22
|
+
config_param :azure_storage_access_key, :string, :default => nil, :secret => true
|
23
|
+
config_param :azure_container, :string, :default => nil
|
24
|
+
config_param :azure_object_key_format, :string, :default => "%{path}%{time_slice}-%{index}.log"
|
25
|
+
config_param :auto_create_container, :bool, :default => true
|
26
|
+
config_param :format, :string, :default => DEFAULT_FORMAT_TYPE
|
27
|
+
config_param :time_slice_format, :string, :default => '%Y%m%d'
|
28
|
+
|
29
|
+
DEFAULT_FORMAT_TYPE = "out_file"
|
30
|
+
AZURE_BLOCK_SIZE_LIMIT = 4 * 1024 * 1024 - 1
|
31
|
+
|
32
|
+
config_section :format do
|
33
|
+
config_set_default :@type, DEFAULT_FORMAT_TYPE
|
34
|
+
end
|
35
|
+
|
36
|
+
config_section :buffer do
|
37
|
+
config_set_default :chunk_keys, ['time']
|
38
|
+
config_set_default :timekey, (60 * 60 * 24)
|
39
|
+
end
|
40
|
+
|
41
|
+
attr_reader :bs
|
42
|
+
|
43
|
+
def configure(conf)
|
44
|
+
super
|
45
|
+
|
46
|
+
@formatter = formatter_create
|
47
|
+
|
48
|
+
if @localtime
|
49
|
+
@path_slicer = Proc.new {|path|
|
50
|
+
Time.now.strftime(path)
|
51
|
+
}
|
52
|
+
else
|
53
|
+
@path_slicer = Proc.new {|path|
|
54
|
+
Time.now.utc.strftime(path)
|
55
|
+
}
|
56
|
+
end
|
57
|
+
|
58
|
+
if @azure_container.nil?
|
59
|
+
raise ConfigError, 'azure_container is needed'
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def multi_workers_ready?
|
64
|
+
true
|
65
|
+
end
|
66
|
+
|
67
|
+
def start
|
68
|
+
super
|
69
|
+
|
70
|
+
@bs = Azure::Storage::Blob::BlobService.create(storage_account_name: @azure_storage_account, storage_access_key: @azure_storage_access_key)
|
71
|
+
|
72
|
+
ensure_container
|
73
|
+
|
74
|
+
@azure_storage_path = ''
|
75
|
+
@last_azure_storage_path = ''
|
76
|
+
@current_index = 0
|
77
|
+
end
|
78
|
+
|
79
|
+
def format(tag, time, record)
|
80
|
+
r = inject_values_to_record(tag, time, record)
|
81
|
+
@formatter.format(tag, time, r)
|
82
|
+
end
|
83
|
+
|
84
|
+
def write(chunk)
|
85
|
+
metadata = chunk.metadata
|
86
|
+
tmp = Tempfile.new("azure-")
|
87
|
+
begin
|
88
|
+
chunk.write_to(tmp)
|
89
|
+
tmp.close
|
90
|
+
|
91
|
+
generate_log_name(metadata, @current_index)
|
92
|
+
if @last_azure_storage_path != @azure_storage_path
|
93
|
+
@current_index = 0
|
94
|
+
generate_log_name(metadata, @current_index)
|
95
|
+
end
|
96
|
+
|
97
|
+
content = File.open(tmp.path, 'rb') { |file| file.read }
|
98
|
+
|
99
|
+
append_blob(content)
|
100
|
+
@last_azure_storage_path = @azure_storage_path
|
101
|
+
ensure
|
102
|
+
tmp.unlink
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
private
|
107
|
+
def ensure_container
|
108
|
+
if ! @bs.list_containers.find { |c| c.name == @azure_container }
|
109
|
+
if @auto_create_container
|
110
|
+
@bs.create_container(@azure_container)
|
111
|
+
else
|
112
|
+
raise "The specified container does not exist: container = #{@azure_container}"
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
private
|
118
|
+
def generate_log_name(metadata, index)
|
119
|
+
time_slice = if metadata.timekey.nil?
|
120
|
+
''.freeze
|
121
|
+
else
|
122
|
+
Time.at(metadata.timekey).utc.strftime(@time_slice_format)
|
123
|
+
end
|
124
|
+
|
125
|
+
path = @path_slicer.call(@path)
|
126
|
+
values_for_object_key = {
|
127
|
+
"%{path}" => path,
|
128
|
+
"%{time_slice}" => time_slice,
|
129
|
+
"%{index}" => index
|
130
|
+
}
|
131
|
+
storage_path = @azure_object_key_format.gsub(%r(%{[^}]+}), values_for_object_key)
|
132
|
+
@azure_storage_path = extract_placeholders(storage_path, metadata)
|
133
|
+
end
|
134
|
+
|
135
|
+
private
|
136
|
+
def append_blob(content)
|
137
|
+
position = 0
|
138
|
+
log.debug "azure_storage_append_blob: append_blob.start: Content size: #{content.length}"
|
139
|
+
loop do
|
140
|
+
begin
|
141
|
+
size = [content.length - position, AZURE_BLOCK_SIZE_LIMIT].min
|
142
|
+
log.debug "azure_storage_append_blob: append_blob.chunk: content[#{position}..#{position + size}]"
|
143
|
+
@bs.append_blob_block(@azure_container, @azure_storage_path, content[position..position + size])
|
144
|
+
position += size
|
145
|
+
break if position >= content.length
|
146
|
+
rescue Azure::Core::Http::HTTPError => ex
|
147
|
+
status_code = ex.status_code
|
148
|
+
|
149
|
+
if status_code == 409 # exceeds azure block limit
|
150
|
+
@current_index += 1
|
151
|
+
old_azure_storage_path = @azure_storage_path
|
152
|
+
generate_log_name(metadata, time_slice, @current_index)
|
153
|
+
|
154
|
+
# If index is not a part of format, rethrow exception.
|
155
|
+
if old_azure_storage_path == @azure_storage_path
|
156
|
+
log.warn "azure_storage_append_blob: append_blob: blocks limit reached, you need to use %{index} for the format."
|
157
|
+
raise
|
158
|
+
end
|
159
|
+
|
160
|
+
log.debug "azure_storage_append_blob: append_blob: blocks limit reached, creating new blob #{@azure_storage_path}."
|
161
|
+
@bs.create_append_blob(@azure_container, @azure_storage_path)
|
162
|
+
elsif status_code == 404 # blob not found
|
163
|
+
log.debug "azure_storage_append_blob: append_blob: #{@azure_storage_path} blob doesn't exist, creating new blob."
|
164
|
+
@bs.create_append_blob(@azure_container, @azure_storage_path)
|
165
|
+
else
|
166
|
+
raise
|
167
|
+
end
|
168
|
+
end
|
169
|
+
end
|
170
|
+
log.debug "azure_storage_append_blob: append_blob.complete"
|
171
|
+
end
|
172
|
+
|
173
|
+
end
|
174
|
+
end
|
175
|
+
end
|
data/test/helper.rb
ADDED
@@ -0,0 +1,8 @@
|
|
1
|
+
$LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
|
2
|
+
require "test-unit"
|
3
|
+
require "fluent/test"
|
4
|
+
require "fluent/test/driver/output"
|
5
|
+
require "fluent/test/helpers"
|
6
|
+
|
7
|
+
Test::Unit::TestCase.include(Fluent::Test::Helpers)
|
8
|
+
Test::Unit::TestCase.extend(Fluent::Test::Helpers)
|
@@ -0,0 +1,66 @@
|
|
1
|
+
require 'helper'
|
2
|
+
require 'fluent/plugin/out_azure-storage-append-blob.rb'
|
3
|
+
|
4
|
+
include Fluent::Test::Helpers
|
5
|
+
|
6
|
+
class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
|
7
|
+
setup do
|
8
|
+
Fluent::Test.setup
|
9
|
+
end
|
10
|
+
|
11
|
+
CONFIG = %[
|
12
|
+
azure_storage_account test_storage_account
|
13
|
+
azure_storage_access_key MY_FAKE_SECRET
|
14
|
+
azure_container test_container
|
15
|
+
time_slice_format %Y%m%d-%H
|
16
|
+
path log
|
17
|
+
]
|
18
|
+
|
19
|
+
def create_driver(conf=CONFIG)
|
20
|
+
Fluent::Test::Driver::Output.new(Fluent::Plugin::AzureStorageAppendBlobOut).configure(conf)
|
21
|
+
end
|
22
|
+
|
23
|
+
sub_test_case 'test config' do
|
24
|
+
test 'config should reject with no azure container' do
|
25
|
+
assert_raise Fluent::ConfigError do
|
26
|
+
create_driver(%[
|
27
|
+
azure_storage_account test_storage_account
|
28
|
+
azure_storage_access_key MY_FAKE_SECRET
|
29
|
+
time_slice_format %Y%m%d-%H
|
30
|
+
time_slice_wait 10m
|
31
|
+
path log
|
32
|
+
])
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
test 'config should set instance variables' do
|
37
|
+
d = create_driver
|
38
|
+
assert_equal 'test_storage_account', d.instance.azure_storage_account
|
39
|
+
assert_equal 'MY_FAKE_SECRET', d.instance.azure_storage_access_key
|
40
|
+
assert_equal 'test_container', d.instance.azure_container
|
41
|
+
assert_equal true, d.instance.auto_create_container
|
42
|
+
assert_equal '%{path}%{time_slice}-%{index}.log', d.instance.azure_object_key_format
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
sub_test_case 'test path slicing' do
|
47
|
+
test 'test path_slicing' do
|
48
|
+
config = CONFIG.clone.gsub(/path\slog/, "path log/%Y/%m/%d")
|
49
|
+
d = create_driver(config)
|
50
|
+
path_slicer = d.instance.instance_variable_get(:@path_slicer)
|
51
|
+
path = d.instance.instance_variable_get(:@path)
|
52
|
+
slice = path_slicer.call(path)
|
53
|
+
assert_equal slice, Time.now.utc.strftime("log/%Y/%m/%d")
|
54
|
+
end
|
55
|
+
|
56
|
+
test 'path slicing utc' do
|
57
|
+
config = CONFIG.clone.gsub(/path\slog/, "path log/%Y/%m/%d")
|
58
|
+
config << "\nutc\n"
|
59
|
+
d = create_driver(config)
|
60
|
+
path_slicer = d.instance.instance_variable_get(:@path_slicer)
|
61
|
+
path = d.instance.instance_variable_get(:@path)
|
62
|
+
slice = path_slicer.call(path)
|
63
|
+
assert_equal slice, Time.now.utc.strftime("log/%Y/%m/%d")
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
metadata
ADDED
@@ -0,0 +1,131 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-azure-storage-append-blob
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Microsoft
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2018-11-29 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.14'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.14'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '12.0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '12.0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: test-unit
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '3.0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '3.0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: fluentd
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 0.14.10
|
62
|
+
- - "<"
|
63
|
+
- !ruby/object:Gem::Version
|
64
|
+
version: '2'
|
65
|
+
type: :runtime
|
66
|
+
prerelease: false
|
67
|
+
version_requirements: !ruby/object:Gem::Requirement
|
68
|
+
requirements:
|
69
|
+
- - ">="
|
70
|
+
- !ruby/object:Gem::Version
|
71
|
+
version: 0.14.10
|
72
|
+
- - "<"
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: '2'
|
75
|
+
- !ruby/object:Gem::Dependency
|
76
|
+
name: azure-storage-blob
|
77
|
+
requirement: !ruby/object:Gem::Requirement
|
78
|
+
requirements:
|
79
|
+
- - "~>"
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
version: '1.0'
|
82
|
+
type: :runtime
|
83
|
+
prerelease: false
|
84
|
+
version_requirements: !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - "~>"
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: '1.0'
|
89
|
+
description: Fluentd plugin to upload logs to Azure Storage append blobs.
|
90
|
+
email:
|
91
|
+
- ''
|
92
|
+
executables: []
|
93
|
+
extensions: []
|
94
|
+
extra_rdoc_files: []
|
95
|
+
files:
|
96
|
+
- ".gitignore"
|
97
|
+
- Gemfile
|
98
|
+
- LICENSE
|
99
|
+
- README.md
|
100
|
+
- Rakefile
|
101
|
+
- fluent-plugin-azure-storage-append-blob.gemspec
|
102
|
+
- lib/fluent/plugin/out_azure-storage-append-blob.rb
|
103
|
+
- test/helper.rb
|
104
|
+
- test/plugin/test_out_azure_storage_append_blob.rb
|
105
|
+
homepage: https://github.com/Microsoft/fluent-plugin-azure-storage-append-blob
|
106
|
+
licenses:
|
107
|
+
- MIT
|
108
|
+
metadata: {}
|
109
|
+
post_install_message:
|
110
|
+
rdoc_options: []
|
111
|
+
require_paths:
|
112
|
+
- lib
|
113
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
119
|
+
requirements:
|
120
|
+
- - ">="
|
121
|
+
- !ruby/object:Gem::Version
|
122
|
+
version: '0'
|
123
|
+
requirements: []
|
124
|
+
rubyforge_project:
|
125
|
+
rubygems_version: 2.7.8
|
126
|
+
signing_key:
|
127
|
+
specification_version: 4
|
128
|
+
summary: Azure Storage Append Blob output plugin for Fluentd event collector
|
129
|
+
test_files:
|
130
|
+
- test/helper.rb
|
131
|
+
- test/plugin/test_out_azure_storage_append_blob.rb
|