fluent-plugin-azurestorage 0.0.5 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +8 -14
- data/.travis.yml +21 -0
- data/Gemfile +1 -2
- data/Rakefile +14 -1
- data/VERSION +1 -0
- data/fluent-plugin-azurestorage.gemspec +21 -23
- data/lib/fluent/plugin/azurestorage_compressor_gzip_command.rb +51 -0
- data/lib/fluent/plugin/azurestorage_compressor_lzma2.rb +34 -0
- data/lib/fluent/plugin/azurestorage_compressor_lzo.rb +34 -0
- data/lib/fluent/plugin/out_azurestorage.rb +122 -66
- data/test/test_out_azurestorage.rb +224 -0
- metadata +36 -43
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8bb81cd14c71e59c606798aacf0fcbf3816d65c8
|
4
|
+
data.tar.gz: b520ba98690443f89e9f683248ba321675de0ffa
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 98704f9ebff04476910ff2292e141fd8bf11f6831a27d429cdcecf16a8a2053bee071dbb2c8310b7ed600a42d71d7ddf52111b019b4cf968ef6704eda3de0113
|
7
|
+
data.tar.gz: 06eac631498041b2c523b926587afc49dd90d13c39ba92a527beb8e04ea9369449885ab2dc0a055282af090491c7b5ca3fe8a016bc2569a083628d99e815ae39
|
data/.gitignore
CHANGED
@@ -1,17 +1,11 @@
|
|
1
|
+
~*
|
2
|
+
#*
|
3
|
+
*~
|
4
|
+
[._]*.s[a-w][a-z]
|
5
|
+
.DS_Store
|
6
|
+
|
1
7
|
*.gem
|
2
|
-
*.rbc
|
3
8
|
.bundle
|
4
|
-
.config
|
5
|
-
.yardoc
|
6
9
|
Gemfile.lock
|
7
|
-
|
8
|
-
|
9
|
-
coverage
|
10
|
-
doc/
|
11
|
-
lib/bundler/man
|
12
|
-
pkg
|
13
|
-
rdoc
|
14
|
-
spec/reports
|
15
|
-
test/tmp
|
16
|
-
test/version_tmp
|
17
|
-
tmp
|
10
|
+
vendor
|
11
|
+
.ruby-version
|
data/.travis.yml
ADDED
data/Gemfile
CHANGED
data/Rakefile
CHANGED
@@ -1 +1,14 @@
|
|
1
|
-
|
1
|
+
|
2
|
+
require 'bundler'
|
3
|
+
Bundler::GemHelper.install_tasks
|
4
|
+
|
5
|
+
require 'rake/testtask'
|
6
|
+
|
7
|
+
Rake::TestTask.new(:test) do |test|
|
8
|
+
test.libs << 'lib' << 'test'
|
9
|
+
test.test_files = FileList['test/test_*.rb']
|
10
|
+
test.verbose = true
|
11
|
+
end
|
12
|
+
|
13
|
+
task :default => [:build]
|
14
|
+
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.0.6
|
@@ -1,26 +1,24 @@
|
|
1
|
-
#
|
2
|
-
|
3
|
-
#$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
1
|
+
# encoding: utf-8
|
2
|
+
$:.push File.expand_path('../lib', __FILE__)
|
4
3
|
|
5
|
-
Gem::Specification.new do |
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
4
|
+
Gem::Specification.new do |gem|
|
5
|
+
gem.name = "fluent-plugin-azurestorage"
|
6
|
+
gem.description = "Azure Storage output plugin for Fluentd event collector"
|
7
|
+
gem.license = "Apache-2.0"
|
8
|
+
gem.homepage = "https://github.com/htgc/fluent-plugin-azurestorage"
|
9
|
+
gem.summary = gem.description
|
10
|
+
gem.version = File.read("VERSION").strip
|
11
|
+
gem.authors = ["Hidemasa Togashi"]
|
12
|
+
gem.email = ["togachiro@gmail.com"]
|
13
|
+
gem.has_rdoc = false
|
14
|
+
#gem.platform = Gem::Platform::RUBY
|
15
|
+
gem.files = `git ls-files`.split("\n")
|
16
|
+
gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
17
|
+
gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
18
|
+
gem.require_paths = ['lib']
|
14
19
|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
spec.add_development_dependency "bundler", "~> 1.3"
|
21
|
-
spec.add_development_dependency "rake"
|
22
|
-
|
23
|
-
spec.add_runtime_dependency "fluentd", [">= 0.10.58", "< 2"]
|
24
|
-
spec.add_runtime_dependency "fluent-mixin-config-placeholders"
|
25
|
-
spec.add_runtime_dependency "azure", "0.6.4"
|
20
|
+
gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
|
21
|
+
gem.add_dependency "azure", "0.6.4"
|
22
|
+
gem.add_dependency "fluent-mixin-config-placeholders", ">= 0.3.0"
|
23
|
+
gem.add_development_dependency "rake", ">= 0.9.2"
|
26
24
|
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
module Fluent
|
2
|
+
class AzureStorageOutput
|
3
|
+
class GzipCommandCompressor < Compressor
|
4
|
+
AzureStorageOutput.register_compressor('gzip_command', self)
|
5
|
+
|
6
|
+
config_param :command_parameter, :string, :default => ''
|
7
|
+
|
8
|
+
def configure(conf)
|
9
|
+
super
|
10
|
+
check_command('gzip')
|
11
|
+
end
|
12
|
+
|
13
|
+
def ext
|
14
|
+
'gz'.freeze
|
15
|
+
end
|
16
|
+
|
17
|
+
def content_type
|
18
|
+
'application/x-gzip'.freeze
|
19
|
+
end
|
20
|
+
|
21
|
+
def compress(chunk, tmp)
|
22
|
+
chunk_is_file = @buffer_type == 'file'
|
23
|
+
path = if chunk_is_file
|
24
|
+
chunk.path
|
25
|
+
else
|
26
|
+
w = Tempfile.new("chunk-gzip-tmp")
|
27
|
+
chunk.write_to(w)
|
28
|
+
w.close
|
29
|
+
w.path
|
30
|
+
end
|
31
|
+
|
32
|
+
res = system "gzip #{@command_parameter} -c #{path} > #{tmp.path}"
|
33
|
+
unless res
|
34
|
+
log.warn "failed to execute gzip command. Fallback to GzipWriter. status = #{$?}"
|
35
|
+
begin
|
36
|
+
tmp.truncate(0)
|
37
|
+
gw = Zlib::GzipWriter.new(tmp)
|
38
|
+
chunk.write_to(gw)
|
39
|
+
gw.close
|
40
|
+
ensure
|
41
|
+
gw.close rescue nil
|
42
|
+
end
|
43
|
+
end
|
44
|
+
ensure
|
45
|
+
unless chunk_is_file
|
46
|
+
w.close(true) rescue nil
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
module Fluent
|
2
|
+
class AzureStorageOutput
|
3
|
+
class LZMA2Compressor < Compressor
|
4
|
+
AzureStorageOutput.register_compressor('lzma2', self)
|
5
|
+
|
6
|
+
config_param :command_parameter, :string, :default => '-qf0'
|
7
|
+
|
8
|
+
def configure(conf)
|
9
|
+
super
|
10
|
+
check_command('xz', 'LZMA2')
|
11
|
+
end
|
12
|
+
|
13
|
+
def ext
|
14
|
+
'xz'.freeze
|
15
|
+
end
|
16
|
+
|
17
|
+
def content_type
|
18
|
+
'application/x-xz'.freeze
|
19
|
+
end
|
20
|
+
|
21
|
+
def compress(chunk, tmp)
|
22
|
+
w = Tempfile.new("chunk-xz-tmp")
|
23
|
+
chunk.write_to(w)
|
24
|
+
w.close
|
25
|
+
|
26
|
+
# We don't check the return code because we can't recover lzop failure.
|
27
|
+
system "xz #{@command_parameter} -c #{w.path} > #{tmp.path}"
|
28
|
+
ensure
|
29
|
+
w.close rescue nil
|
30
|
+
w.unlink rescue nil
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
module Fluent
|
2
|
+
class AzureStorageOutput
|
3
|
+
class LZOCompressor < Compressor
|
4
|
+
AzureStorageOutput.register_compressor('lzo', self)
|
5
|
+
|
6
|
+
config_param :command_parameter, :string, :default => '-qf1'
|
7
|
+
|
8
|
+
def configure(conf)
|
9
|
+
super
|
10
|
+
check_command('lzop', 'LZO')
|
11
|
+
end
|
12
|
+
|
13
|
+
def ext
|
14
|
+
'lzo'.freeze
|
15
|
+
end
|
16
|
+
|
17
|
+
def content_type
|
18
|
+
'application/x-lzop'.freeze
|
19
|
+
end
|
20
|
+
|
21
|
+
def compress(chunk, tmp)
|
22
|
+
w = Tempfile.new("chunk-tmp")
|
23
|
+
chunk.write_to(w)
|
24
|
+
w.close
|
25
|
+
|
26
|
+
# We don't check the return code because we can't recover lzop failure.
|
27
|
+
system "lzop #{@command_parameter} -o #{tmp.path} #{w.path}"
|
28
|
+
ensure
|
29
|
+
w.close rescue nil
|
30
|
+
w.unlink rescue nil
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -10,20 +10,23 @@ module Fluent
|
|
10
10
|
require 'zlib'
|
11
11
|
require 'time'
|
12
12
|
require 'tempfile'
|
13
|
-
|
13
|
+
|
14
|
+
@compressor = nil
|
14
15
|
end
|
15
16
|
|
16
|
-
config_param :path, :string, :default =>
|
17
|
+
config_param :path, :string, :default => ""
|
17
18
|
config_param :azure_storage_account, :string, :default => nil
|
18
19
|
config_param :azure_storage_access_key, :string, :default => nil, :secret => true
|
19
20
|
config_param :azure_container, :string, :default => nil
|
20
|
-
config_param :azure_storage_type, :string, :default =>
|
21
|
+
config_param :azure_storage_type, :string, :default => "blob"
|
21
22
|
config_param :azure_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
|
22
|
-
config_param :store_as, :string, :default =>
|
23
|
+
config_param :store_as, :string, :default => "gzip"
|
23
24
|
config_param :auto_create_container, :bool, :default => true
|
24
|
-
config_param :format, :string, :default =>
|
25
|
+
config_param :format, :string, :default => "out_file"
|
25
26
|
config_param :command_parameter, :string, :default => nil
|
26
27
|
|
28
|
+
attr_reader :bs
|
29
|
+
|
27
30
|
include Fluent::Mixin::ConfigPlaceholders
|
28
31
|
|
29
32
|
def placeholders
|
@@ -33,31 +36,13 @@ module Fluent
|
|
33
36
|
def configure(conf)
|
34
37
|
super
|
35
38
|
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
when 'lzma2'
|
44
|
-
check_command('xz', 'LZMA2')
|
45
|
-
@command_paramter = '-qf0' if @command_parameter.nil?
|
46
|
-
'xz'
|
47
|
-
when 'json'
|
48
|
-
'json'
|
49
|
-
else
|
50
|
-
'txt'
|
51
|
-
end
|
52
|
-
|
53
|
-
@storage_type = case @azure_storage_type
|
54
|
-
when 'tables'
|
55
|
-
raise NotImplementedError
|
56
|
-
when 'queues'
|
57
|
-
raise NotImplementedError
|
58
|
-
else
|
59
|
-
'blob'
|
60
|
-
end
|
39
|
+
begin
|
40
|
+
@compressor = COMPRESSOR_REGISTRY.lookup(@store_as).new(:buffer_type => @buffer_type, :log => log)
|
41
|
+
rescue => e
|
42
|
+
$log.warn "#{@store_as} not found. Use 'text' instead"
|
43
|
+
@compressor = TextCompressor.new
|
44
|
+
end
|
45
|
+
@compressor.configure(conf)
|
61
46
|
|
62
47
|
@formatter = Plugin.new_formatter(@format)
|
63
48
|
@formatter.configure(conf)
|
@@ -75,6 +60,15 @@ module Fluent
|
|
75
60
|
if @azure_container.nil?
|
76
61
|
raise ConfigError, 'azure_container is needed'
|
77
62
|
end
|
63
|
+
|
64
|
+
@storage_type = case @azure_storage_type
|
65
|
+
when 'tables'
|
66
|
+
raise NotImplementedError
|
67
|
+
when 'queues'
|
68
|
+
raise NotImplementedError
|
69
|
+
else
|
70
|
+
'blob'
|
71
|
+
end
|
78
72
|
end
|
79
73
|
|
80
74
|
def start
|
@@ -102,15 +96,15 @@ module Fluent
|
|
102
96
|
begin
|
103
97
|
path = @path_slicer.call(@path)
|
104
98
|
values_for_object_key = {
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
99
|
+
"path" => path,
|
100
|
+
"time_slice" => chunk.key,
|
101
|
+
"file_extension" => @compressor.ext,
|
102
|
+
"index" => i,
|
103
|
+
"uuid_flush" => uuid_random
|
109
104
|
}
|
110
105
|
storage_path = @azure_object_key_format.gsub(%r(%{[^}]+})) { |expr|
|
111
106
|
values_for_object_key[expr[2...expr.size-1]]
|
112
107
|
}
|
113
|
-
|
114
108
|
if (i > 0) && (storage_path == previous_path)
|
115
109
|
raise "duplicated path is generated. use %{index} in azure_object_key_format: path = #{storage_path}"
|
116
110
|
end
|
@@ -118,36 +112,13 @@ module Fluent
|
|
118
112
|
i += 1
|
119
113
|
previous_path = storage_path
|
120
114
|
end while blob_exists?(@azure_container, storage_path)
|
121
|
-
|
115
|
+
|
122
116
|
tmp = Tempfile.new("azure-")
|
123
117
|
begin
|
124
|
-
|
125
|
-
|
126
|
-
w = Zlib::GzipWriter.new(tmp)
|
127
|
-
chunk.write_to(w)
|
128
|
-
w.close
|
129
|
-
when 'lzo'
|
130
|
-
w = Tempfile.new('chunk-tmp')
|
131
|
-
chunk.write_to(w)
|
132
|
-
w.close
|
133
|
-
tmp.close
|
134
|
-
system "lzop #{@command_parameter} -o #{tmp.path} #{w.path}"
|
135
|
-
when 'lzma2'
|
136
|
-
w = Tempfile.new('chunk-xz-tmp')
|
137
|
-
chunk.write_to(w)
|
138
|
-
w.close
|
139
|
-
tmp.close
|
140
|
-
system "xz #{@command_parameter} -c #{w.path} > #{tmp.path}"
|
141
|
-
else
|
142
|
-
chunk.write_to(tmp)
|
143
|
-
tmp.close
|
144
|
-
end
|
118
|
+
@compressor.compress(chunk, tmp)
|
119
|
+
tmp.close
|
145
120
|
content = File.open(tmp.path, 'rb') { |file| file.read }
|
146
121
|
@bs.create_block_blob(@azure_container, storage_path, content)
|
147
|
-
ensure
|
148
|
-
tmp.close(true) rescue nil
|
149
|
-
w.close rescue nil
|
150
|
-
w.unlink rescue nil
|
151
122
|
end
|
152
123
|
end
|
153
124
|
|
@@ -162,12 +133,97 @@ module Fluent
|
|
162
133
|
end
|
163
134
|
end
|
164
135
|
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
136
|
+
class Compressor
|
137
|
+
include Configurable
|
138
|
+
|
139
|
+
def initialize(opts = {})
|
140
|
+
super()
|
141
|
+
@buffer_type = opts[:buffer_type]
|
142
|
+
@log = opts[:log]
|
170
143
|
end
|
144
|
+
|
145
|
+
attr_reader :buffer_type, :log
|
146
|
+
|
147
|
+
def configure(conf)
|
148
|
+
super
|
149
|
+
end
|
150
|
+
|
151
|
+
def ext
|
152
|
+
end
|
153
|
+
|
154
|
+
def content_type
|
155
|
+
end
|
156
|
+
|
157
|
+
def compress(chunk, tmp)
|
158
|
+
end
|
159
|
+
|
160
|
+
private
|
161
|
+
|
162
|
+
def check_command(command, algo = nil)
|
163
|
+
require 'open3'
|
164
|
+
|
165
|
+
algo = command if algo.nil?
|
166
|
+
begin
|
167
|
+
Open3.capture3("#{command} -V")
|
168
|
+
rescue Errno::ENOENT
|
169
|
+
raise ConfigError, "'#{command}' utility must be in PATH for #{algo} compression"
|
170
|
+
end
|
171
|
+
end
|
172
|
+
end
|
173
|
+
|
174
|
+
class GzipCompressor < Compressor
|
175
|
+
def ext
|
176
|
+
'gz'.freeze
|
177
|
+
end
|
178
|
+
|
179
|
+
def content_type
|
180
|
+
'application/x-gzip'.freeze
|
181
|
+
end
|
182
|
+
|
183
|
+
def compress(chunk, tmp)
|
184
|
+
w = Zlib::GzipWriter.new(tmp)
|
185
|
+
chunk.write_to(w)
|
186
|
+
w.finish
|
187
|
+
ensure
|
188
|
+
w.finish rescue nil
|
189
|
+
end
|
190
|
+
end
|
191
|
+
|
192
|
+
class TextCompressor < Compressor
|
193
|
+
def ext
|
194
|
+
'txt'.freeze
|
195
|
+
end
|
196
|
+
|
197
|
+
def content_type
|
198
|
+
'text/plain'.freeze
|
199
|
+
end
|
200
|
+
|
201
|
+
def compress(chunk, tmp)
|
202
|
+
chunk.write_to(tmp)
|
203
|
+
end
|
204
|
+
end
|
205
|
+
|
206
|
+
class JsonCompressor < TextCompressor
|
207
|
+
def ext
|
208
|
+
'json'.freeze
|
209
|
+
end
|
210
|
+
|
211
|
+
def content_type
|
212
|
+
'application/json'.freeze
|
213
|
+
end
|
214
|
+
end
|
215
|
+
|
216
|
+
COMPRESSOR_REGISTRY = Registry.new(:azurestorage_compressor_type, 'fluent/plugin/azurestorage_compressor_')
|
217
|
+
{
|
218
|
+
'gzip' => GzipCompressor,
|
219
|
+
'json' => JsonCompressor,
|
220
|
+
'text' => TextCompressor
|
221
|
+
}.each { |name, compressor|
|
222
|
+
COMPRESSOR_REGISTRY.register(name, compressor)
|
223
|
+
}
|
224
|
+
|
225
|
+
def self.register_compressor(name, compressor)
|
226
|
+
COMPRESSOR_REGISTRY.register(name, compressor)
|
171
227
|
end
|
172
228
|
|
173
229
|
def blob_exists?(container, blob)
|
@@ -0,0 +1,224 @@
|
|
1
|
+
require 'fluent/test'
|
2
|
+
require 'fluent/plugin/out_azurestorage'
|
3
|
+
|
4
|
+
require 'test/unit/rr'
|
5
|
+
require 'zlib'
|
6
|
+
require 'fileutils'
|
7
|
+
|
8
|
+
class AzureStorageOutputTest < Test::Unit::TestCase
|
9
|
+
def setup
|
10
|
+
require 'azure'
|
11
|
+
Fluent::Test.setup
|
12
|
+
end
|
13
|
+
|
14
|
+
CONFIG = %[
|
15
|
+
azure_storage_account test_storage_account
|
16
|
+
azure_storage_access_key dGVzdF9zdG9yYWdlX2FjY2Vzc19rZXk=
|
17
|
+
azure_container test_container
|
18
|
+
path log
|
19
|
+
utc
|
20
|
+
buffer_type memory
|
21
|
+
]
|
22
|
+
|
23
|
+
def create_driver(conf = CONFIG)
|
24
|
+
Fluent::Test::BufferedOutputTestDriver.new(Fluent::AzureStorageOutput) do
|
25
|
+
def write(chunk)
|
26
|
+
chunk.read
|
27
|
+
end
|
28
|
+
|
29
|
+
private
|
30
|
+
|
31
|
+
def ensure_container
|
32
|
+
end
|
33
|
+
|
34
|
+
end.configure(conf)
|
35
|
+
end
|
36
|
+
|
37
|
+
def test_configure
|
38
|
+
d = create_driver
|
39
|
+
assert_equal 'test_storage_account', d.instance.azure_storage_account
|
40
|
+
assert_equal 'dGVzdF9zdG9yYWdlX2FjY2Vzc19rZXk=', d.instance.azure_storage_access_key
|
41
|
+
assert_equal 'test_container', d.instance.azure_container
|
42
|
+
assert_equal 'log', d.instance.path
|
43
|
+
assert_equal 'gz', d.instance.instance_variable_get(:@compressor).ext
|
44
|
+
assert_equal 'application/x-gzip', d.instance.instance_variable_get(:@compressor).content_type
|
45
|
+
end
|
46
|
+
|
47
|
+
def test_configure_with_mime_type_json
|
48
|
+
conf = CONFIG.clone
|
49
|
+
conf << "\nstore_as json\n"
|
50
|
+
d = create_driver(conf)
|
51
|
+
assert_equal 'json', d.instance.instance_variable_get(:@compressor).ext
|
52
|
+
assert_equal 'application/json', d.instance.instance_variable_get(:@compressor).content_type
|
53
|
+
end
|
54
|
+
|
55
|
+
def test_configure_with_mime_type_text
|
56
|
+
conf = CONFIG.clone
|
57
|
+
conf << "\nstore_as text\n"
|
58
|
+
d = create_driver(conf)
|
59
|
+
assert_equal 'txt', d.instance.instance_variable_get(:@compressor).ext
|
60
|
+
assert_equal 'text/plain', d.instance.instance_variable_get(:@compressor).content_type
|
61
|
+
end
|
62
|
+
|
63
|
+
def test_configure_with_mime_type_lzo
|
64
|
+
conf = CONFIG.clone
|
65
|
+
conf << "\nstore_as lzo\n"
|
66
|
+
d = create_driver(conf)
|
67
|
+
assert_equal 'lzo', d.instance.instance_variable_get(:@compressor).ext
|
68
|
+
assert_equal 'application/x-lzop', d.instance.instance_variable_get(:@compressor).content_type
|
69
|
+
rescue => e
|
70
|
+
# TODO: replace code with disable lzop command
|
71
|
+
assert(e.is_a?(Fluent::ConfigError))
|
72
|
+
end
|
73
|
+
|
74
|
+
def test_path_slicing
|
75
|
+
config = CONFIG.clone.gsub(/path\slog/, "path log/%Y/%m/%d")
|
76
|
+
d = create_driver(config)
|
77
|
+
path_slicer = d.instance.instance_variable_get(:@path_slicer)
|
78
|
+
path = d.instance.instance_variable_get(:@path)
|
79
|
+
slice = path_slicer.call(path)
|
80
|
+
assert_equal slice, Time.now.utc.strftime("log/%Y/%m/%d")
|
81
|
+
end
|
82
|
+
|
83
|
+
def test_path_slicing_utc
|
84
|
+
config = CONFIG.clone.gsub(/path\slog/, "path log/%Y/%m/%d")
|
85
|
+
config << "\nutc\n"
|
86
|
+
d = create_driver(config)
|
87
|
+
path_slicer = d.instance.instance_variable_get(:@path_slicer)
|
88
|
+
path = d.instance.instance_variable_get(:@path)
|
89
|
+
slice = path_slicer.call(path)
|
90
|
+
assert_equal slice, Time.now.utc.strftime("log/%Y/%m/%d")
|
91
|
+
end
|
92
|
+
|
93
|
+
def test_format
|
94
|
+
d = create_driver
|
95
|
+
|
96
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
97
|
+
d.emit({"a"=>1}, time)
|
98
|
+
d.emit({"a"=>2}, time)
|
99
|
+
|
100
|
+
d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n]
|
101
|
+
d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
|
102
|
+
|
103
|
+
d.run
|
104
|
+
end
|
105
|
+
|
106
|
+
def test_format_included_tag_and_time
|
107
|
+
config = [CONFIG, 'include_tag_key true', 'include_time_key true'].join("\n")
|
108
|
+
d = create_driver(config)
|
109
|
+
|
110
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
111
|
+
d.emit({"a"=>1}, time)
|
112
|
+
d.emit({"a"=>2}, time)
|
113
|
+
|
114
|
+
d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":1,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
|
115
|
+
d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":2,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
|
116
|
+
|
117
|
+
d.run
|
118
|
+
end
|
119
|
+
|
120
|
+
def test_format_with_format_ltsv
|
121
|
+
config = [CONFIG, 'format ltsv'].join("\n")
|
122
|
+
d = create_driver(config)
|
123
|
+
|
124
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
125
|
+
d.emit({"a"=>1, "b"=>1}, time)
|
126
|
+
d.emit({"a"=>2, "b"=>2}, time)
|
127
|
+
|
128
|
+
d.expect_format %[a:1\tb:1\n]
|
129
|
+
d.expect_format %[a:2\tb:2\n]
|
130
|
+
|
131
|
+
d.run
|
132
|
+
end
|
133
|
+
|
134
|
+
def test_format_with_format_json
|
135
|
+
config = [CONFIG, 'format json'].join("\n")
|
136
|
+
d = create_driver(config)
|
137
|
+
|
138
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
139
|
+
d.emit({"a"=>1}, time)
|
140
|
+
d.emit({"a"=>2}, time)
|
141
|
+
|
142
|
+
d.expect_format %[{"a":1}\n]
|
143
|
+
d.expect_format %[{"a":2}\n]
|
144
|
+
|
145
|
+
d.run
|
146
|
+
end
|
147
|
+
|
148
|
+
def test_format_with_format_json_included_tag
|
149
|
+
config = [CONFIG, 'format json', 'include_tag_key true'].join("\n")
|
150
|
+
d = create_driver(config)
|
151
|
+
|
152
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
153
|
+
d.emit({"a"=>1}, time)
|
154
|
+
d.emit({"a"=>2}, time)
|
155
|
+
|
156
|
+
d.expect_format %[{"a":1,"tag":"test"}\n]
|
157
|
+
d.expect_format %[{"a":2,"tag":"test"}\n]
|
158
|
+
|
159
|
+
d.run
|
160
|
+
end
|
161
|
+
|
162
|
+
def test_format_with_format_json_included_time
|
163
|
+
config = [CONFIG, 'format json', 'include_time_key true'].join("\n")
|
164
|
+
d = create_driver(config)
|
165
|
+
|
166
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
167
|
+
d.emit({"a"=>1}, time)
|
168
|
+
d.emit({"a"=>2}, time)
|
169
|
+
|
170
|
+
d.expect_format %[{"a":1,"time":"2011-01-02T13:14:15Z"}\n]
|
171
|
+
d.expect_format %[{"a":2,"time":"2011-01-02T13:14:15Z"}\n]
|
172
|
+
|
173
|
+
d.run
|
174
|
+
end
|
175
|
+
|
176
|
+
def test_format_with_format_json_included_tag_and_time
|
177
|
+
config = [CONFIG, 'format json', 'include_tag_key true', 'include_time_key true'].join("\n")
|
178
|
+
d = create_driver(config)
|
179
|
+
|
180
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
181
|
+
d.emit({"a"=>1}, time)
|
182
|
+
d.emit({"a"=>2}, time)
|
183
|
+
|
184
|
+
d.expect_format %[{"a":1,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
|
185
|
+
d.expect_format %[{"a":2,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
|
186
|
+
|
187
|
+
d.run
|
188
|
+
end
|
189
|
+
|
190
|
+
def test_chunk_to_write
|
191
|
+
d = create_driver
|
192
|
+
|
193
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
194
|
+
d.emit({"a"=>1}, time)
|
195
|
+
d.emit({"a"=>2}, time)
|
196
|
+
|
197
|
+
# AzureStorageOutputTest#write returns chunk.read
|
198
|
+
data = d.run
|
199
|
+
|
200
|
+
assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
|
201
|
+
%[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
|
202
|
+
data
|
203
|
+
end
|
204
|
+
|
205
|
+
CONFIG_TIME_SLICE = %[
|
206
|
+
hostname testing.node.local
|
207
|
+
azure_storage_account test_storage_account
|
208
|
+
azure_storage_access_key dGVzdF9zdG9yYWdlX2FjY2Vzc19rZXk=
|
209
|
+
azure_container test_container
|
210
|
+
azure_object_key_format %{path}/events/ts=%{time_slice}/events_%{index}-%{hostname}.%{file_extension}
|
211
|
+
time_slice_format %Y%m%d-%H
|
212
|
+
path log
|
213
|
+
utc
|
214
|
+
buffer_type memory
|
215
|
+
log_level debug
|
216
|
+
]
|
217
|
+
|
218
|
+
def create_time_sliced_driver(conf = CONFIG_TIME_SLICE)
|
219
|
+
d = Fluent::Test::TimeSlicedOutputTestDriver.new(Fluent::AzureStorageOutput) do
|
220
|
+
end.configure(conf)
|
221
|
+
d
|
222
|
+
end
|
223
|
+
|
224
|
+
end
|
metadata
CHANGED
@@ -1,43 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-azurestorage
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Hidemasa Togashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-09
|
11
|
+
date: 2015-10-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
|
-
- !ruby/object:Gem::Dependency
|
14
|
-
name: bundler
|
15
|
-
requirement: !ruby/object:Gem::Requirement
|
16
|
-
requirements:
|
17
|
-
- - ~>
|
18
|
-
- !ruby/object:Gem::Version
|
19
|
-
version: '1.3'
|
20
|
-
type: :development
|
21
|
-
prerelease: false
|
22
|
-
version_requirements: !ruby/object:Gem::Requirement
|
23
|
-
requirements:
|
24
|
-
- - ~>
|
25
|
-
- !ruby/object:Gem::Version
|
26
|
-
version: '1.3'
|
27
|
-
- !ruby/object:Gem::Dependency
|
28
|
-
name: rake
|
29
|
-
requirement: !ruby/object:Gem::Requirement
|
30
|
-
requirements:
|
31
|
-
- - '>='
|
32
|
-
- !ruby/object:Gem::Version
|
33
|
-
version: '0'
|
34
|
-
type: :development
|
35
|
-
prerelease: false
|
36
|
-
version_requirements: !ruby/object:Gem::Requirement
|
37
|
-
requirements:
|
38
|
-
- - '>='
|
39
|
-
- !ruby/object:Gem::Version
|
40
|
-
version: '0'
|
41
13
|
- !ruby/object:Gem::Dependency
|
42
14
|
name: fluentd
|
43
15
|
requirement: !ruby/object:Gem::Requirement
|
@@ -58,35 +30,49 @@ dependencies:
|
|
58
30
|
- - <
|
59
31
|
- !ruby/object:Gem::Version
|
60
32
|
version: '2'
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
name: azure
|
35
|
+
requirement: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - '='
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: 0.6.4
|
40
|
+
type: :runtime
|
41
|
+
prerelease: false
|
42
|
+
version_requirements: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - '='
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: 0.6.4
|
61
47
|
- !ruby/object:Gem::Dependency
|
62
48
|
name: fluent-mixin-config-placeholders
|
63
49
|
requirement: !ruby/object:Gem::Requirement
|
64
50
|
requirements:
|
65
51
|
- - '>='
|
66
52
|
- !ruby/object:Gem::Version
|
67
|
-
version:
|
53
|
+
version: 0.3.0
|
68
54
|
type: :runtime
|
69
55
|
prerelease: false
|
70
56
|
version_requirements: !ruby/object:Gem::Requirement
|
71
57
|
requirements:
|
72
58
|
- - '>='
|
73
59
|
- !ruby/object:Gem::Version
|
74
|
-
version:
|
60
|
+
version: 0.3.0
|
75
61
|
- !ruby/object:Gem::Dependency
|
76
|
-
name:
|
62
|
+
name: rake
|
77
63
|
requirement: !ruby/object:Gem::Requirement
|
78
64
|
requirements:
|
79
|
-
- - '
|
65
|
+
- - '>='
|
80
66
|
- !ruby/object:Gem::Version
|
81
|
-
version: 0.
|
82
|
-
type: :
|
67
|
+
version: 0.9.2
|
68
|
+
type: :development
|
83
69
|
prerelease: false
|
84
70
|
version_requirements: !ruby/object:Gem::Requirement
|
85
71
|
requirements:
|
86
|
-
- - '
|
72
|
+
- - '>='
|
87
73
|
- !ruby/object:Gem::Version
|
88
|
-
version: 0.
|
89
|
-
description:
|
74
|
+
version: 0.9.2
|
75
|
+
description: Azure Storage output plugin for Fluentd event collector
|
90
76
|
email:
|
91
77
|
- togachiro@gmail.com
|
92
78
|
executables: []
|
@@ -94,15 +80,21 @@ extensions: []
|
|
94
80
|
extra_rdoc_files: []
|
95
81
|
files:
|
96
82
|
- .gitignore
|
83
|
+
- .travis.yml
|
97
84
|
- Gemfile
|
98
85
|
- LICENSE.txt
|
99
86
|
- README.md
|
100
87
|
- Rakefile
|
88
|
+
- VERSION
|
101
89
|
- fluent-plugin-azurestorage.gemspec
|
90
|
+
- lib/fluent/plugin/azurestorage_compressor_gzip_command.rb
|
91
|
+
- lib/fluent/plugin/azurestorage_compressor_lzma2.rb
|
92
|
+
- lib/fluent/plugin/azurestorage_compressor_lzo.rb
|
102
93
|
- lib/fluent/plugin/out_azurestorage.rb
|
103
|
-
|
94
|
+
- test/test_out_azurestorage.rb
|
95
|
+
homepage: https://github.com/htgc/fluent-plugin-azurestorage
|
104
96
|
licenses:
|
105
|
-
- Apache
|
97
|
+
- Apache-2.0
|
106
98
|
metadata: {}
|
107
99
|
post_install_message:
|
108
100
|
rdoc_options: []
|
@@ -123,5 +115,6 @@ rubyforge_project:
|
|
123
115
|
rubygems_version: 2.0.14
|
124
116
|
signing_key:
|
125
117
|
specification_version: 4
|
126
|
-
summary:
|
127
|
-
test_files:
|
118
|
+
summary: Azure Storage output plugin for Fluentd event collector
|
119
|
+
test_files:
|
120
|
+
- test/test_out_azurestorage.rb
|