fluent-plugin-azurestorage 0.0.5 → 0.0.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +8 -14
- data/.travis.yml +21 -0
- data/Gemfile +1 -2
- data/Rakefile +14 -1
- data/VERSION +1 -0
- data/fluent-plugin-azurestorage.gemspec +21 -23
- data/lib/fluent/plugin/azurestorage_compressor_gzip_command.rb +51 -0
- data/lib/fluent/plugin/azurestorage_compressor_lzma2.rb +34 -0
- data/lib/fluent/plugin/azurestorage_compressor_lzo.rb +34 -0
- data/lib/fluent/plugin/out_azurestorage.rb +122 -66
- data/test/test_out_azurestorage.rb +224 -0
- metadata +36 -43
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8bb81cd14c71e59c606798aacf0fcbf3816d65c8
|
4
|
+
data.tar.gz: b520ba98690443f89e9f683248ba321675de0ffa
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 98704f9ebff04476910ff2292e141fd8bf11f6831a27d429cdcecf16a8a2053bee071dbb2c8310b7ed600a42d71d7ddf52111b019b4cf968ef6704eda3de0113
|
7
|
+
data.tar.gz: 06eac631498041b2c523b926587afc49dd90d13c39ba92a527beb8e04ea9369449885ab2dc0a055282af090491c7b5ca3fe8a016bc2569a083628d99e815ae39
|
data/.gitignore
CHANGED
@@ -1,17 +1,11 @@
|
|
1
|
+
~*
|
2
|
+
#*
|
3
|
+
*~
|
4
|
+
[._]*.s[a-w][a-z]
|
5
|
+
.DS_Store
|
6
|
+
|
1
7
|
*.gem
|
2
|
-
*.rbc
|
3
8
|
.bundle
|
4
|
-
.config
|
5
|
-
.yardoc
|
6
9
|
Gemfile.lock
|
7
|
-
|
8
|
-
|
9
|
-
coverage
|
10
|
-
doc/
|
11
|
-
lib/bundler/man
|
12
|
-
pkg
|
13
|
-
rdoc
|
14
|
-
spec/reports
|
15
|
-
test/tmp
|
16
|
-
test/version_tmp
|
17
|
-
tmp
|
10
|
+
vendor
|
11
|
+
.ruby-version
|
data/.travis.yml
ADDED
data/Gemfile
CHANGED
data/Rakefile
CHANGED
@@ -1 +1,14 @@
|
|
1
|
-
|
1
|
+
|
2
|
+
require 'bundler'
|
3
|
+
Bundler::GemHelper.install_tasks
|
4
|
+
|
5
|
+
require 'rake/testtask'
|
6
|
+
|
7
|
+
Rake::TestTask.new(:test) do |test|
|
8
|
+
test.libs << 'lib' << 'test'
|
9
|
+
test.test_files = FileList['test/test_*.rb']
|
10
|
+
test.verbose = true
|
11
|
+
end
|
12
|
+
|
13
|
+
task :default => [:build]
|
14
|
+
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.0.6
|
@@ -1,26 +1,24 @@
|
|
1
|
-
#
|
2
|
-
|
3
|
-
#$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
1
|
+
# encoding: utf-8
|
2
|
+
$:.push File.expand_path('../lib', __FILE__)
|
4
3
|
|
5
|
-
Gem::Specification.new do |
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
4
|
+
Gem::Specification.new do |gem|
|
5
|
+
gem.name = "fluent-plugin-azurestorage"
|
6
|
+
gem.description = "Azure Storage output plugin for Fluentd event collector"
|
7
|
+
gem.license = "Apache-2.0"
|
8
|
+
gem.homepage = "https://github.com/htgc/fluent-plugin-azurestorage"
|
9
|
+
gem.summary = gem.description
|
10
|
+
gem.version = File.read("VERSION").strip
|
11
|
+
gem.authors = ["Hidemasa Togashi"]
|
12
|
+
gem.email = ["togachiro@gmail.com"]
|
13
|
+
gem.has_rdoc = false
|
14
|
+
#gem.platform = Gem::Platform::RUBY
|
15
|
+
gem.files = `git ls-files`.split("\n")
|
16
|
+
gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
17
|
+
gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
18
|
+
gem.require_paths = ['lib']
|
14
19
|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
spec.add_development_dependency "bundler", "~> 1.3"
|
21
|
-
spec.add_development_dependency "rake"
|
22
|
-
|
23
|
-
spec.add_runtime_dependency "fluentd", [">= 0.10.58", "< 2"]
|
24
|
-
spec.add_runtime_dependency "fluent-mixin-config-placeholders"
|
25
|
-
spec.add_runtime_dependency "azure", "0.6.4"
|
20
|
+
gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
|
21
|
+
gem.add_dependency "azure", "0.6.4"
|
22
|
+
gem.add_dependency "fluent-mixin-config-placeholders", ">= 0.3.0"
|
23
|
+
gem.add_development_dependency "rake", ">= 0.9.2"
|
26
24
|
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
module Fluent
|
2
|
+
class AzureStorageOutput
|
3
|
+
class GzipCommandCompressor < Compressor
|
4
|
+
AzureStorageOutput.register_compressor('gzip_command', self)
|
5
|
+
|
6
|
+
config_param :command_parameter, :string, :default => ''
|
7
|
+
|
8
|
+
def configure(conf)
|
9
|
+
super
|
10
|
+
check_command('gzip')
|
11
|
+
end
|
12
|
+
|
13
|
+
def ext
|
14
|
+
'gz'.freeze
|
15
|
+
end
|
16
|
+
|
17
|
+
def content_type
|
18
|
+
'application/x-gzip'.freeze
|
19
|
+
end
|
20
|
+
|
21
|
+
def compress(chunk, tmp)
|
22
|
+
chunk_is_file = @buffer_type == 'file'
|
23
|
+
path = if chunk_is_file
|
24
|
+
chunk.path
|
25
|
+
else
|
26
|
+
w = Tempfile.new("chunk-gzip-tmp")
|
27
|
+
chunk.write_to(w)
|
28
|
+
w.close
|
29
|
+
w.path
|
30
|
+
end
|
31
|
+
|
32
|
+
res = system "gzip #{@command_parameter} -c #{path} > #{tmp.path}"
|
33
|
+
unless res
|
34
|
+
log.warn "failed to execute gzip command. Fallback to GzipWriter. status = #{$?}"
|
35
|
+
begin
|
36
|
+
tmp.truncate(0)
|
37
|
+
gw = Zlib::GzipWriter.new(tmp)
|
38
|
+
chunk.write_to(gw)
|
39
|
+
gw.close
|
40
|
+
ensure
|
41
|
+
gw.close rescue nil
|
42
|
+
end
|
43
|
+
end
|
44
|
+
ensure
|
45
|
+
unless chunk_is_file
|
46
|
+
w.close(true) rescue nil
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
module Fluent
|
2
|
+
class AzureStorageOutput
|
3
|
+
class LZMA2Compressor < Compressor
|
4
|
+
AzureStorageOutput.register_compressor('lzma2', self)
|
5
|
+
|
6
|
+
config_param :command_parameter, :string, :default => '-qf0'
|
7
|
+
|
8
|
+
def configure(conf)
|
9
|
+
super
|
10
|
+
check_command('xz', 'LZMA2')
|
11
|
+
end
|
12
|
+
|
13
|
+
def ext
|
14
|
+
'xz'.freeze
|
15
|
+
end
|
16
|
+
|
17
|
+
def content_type
|
18
|
+
'application/x-xz'.freeze
|
19
|
+
end
|
20
|
+
|
21
|
+
def compress(chunk, tmp)
|
22
|
+
w = Tempfile.new("chunk-xz-tmp")
|
23
|
+
chunk.write_to(w)
|
24
|
+
w.close
|
25
|
+
|
26
|
+
# We don't check the return code because we can't recover lzop failure.
|
27
|
+
system "xz #{@command_parameter} -c #{w.path} > #{tmp.path}"
|
28
|
+
ensure
|
29
|
+
w.close rescue nil
|
30
|
+
w.unlink rescue nil
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
module Fluent
|
2
|
+
class AzureStorageOutput
|
3
|
+
class LZOCompressor < Compressor
|
4
|
+
AzureStorageOutput.register_compressor('lzo', self)
|
5
|
+
|
6
|
+
config_param :command_parameter, :string, :default => '-qf1'
|
7
|
+
|
8
|
+
def configure(conf)
|
9
|
+
super
|
10
|
+
check_command('lzop', 'LZO')
|
11
|
+
end
|
12
|
+
|
13
|
+
def ext
|
14
|
+
'lzo'.freeze
|
15
|
+
end
|
16
|
+
|
17
|
+
def content_type
|
18
|
+
'application/x-lzop'.freeze
|
19
|
+
end
|
20
|
+
|
21
|
+
def compress(chunk, tmp)
|
22
|
+
w = Tempfile.new("chunk-tmp")
|
23
|
+
chunk.write_to(w)
|
24
|
+
w.close
|
25
|
+
|
26
|
+
# We don't check the return code because we can't recover lzop failure.
|
27
|
+
system "lzop #{@command_parameter} -o #{tmp.path} #{w.path}"
|
28
|
+
ensure
|
29
|
+
w.close rescue nil
|
30
|
+
w.unlink rescue nil
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -10,20 +10,23 @@ module Fluent
|
|
10
10
|
require 'zlib'
|
11
11
|
require 'time'
|
12
12
|
require 'tempfile'
|
13
|
-
|
13
|
+
|
14
|
+
@compressor = nil
|
14
15
|
end
|
15
16
|
|
16
|
-
config_param :path, :string, :default =>
|
17
|
+
config_param :path, :string, :default => ""
|
17
18
|
config_param :azure_storage_account, :string, :default => nil
|
18
19
|
config_param :azure_storage_access_key, :string, :default => nil, :secret => true
|
19
20
|
config_param :azure_container, :string, :default => nil
|
20
|
-
config_param :azure_storage_type, :string, :default =>
|
21
|
+
config_param :azure_storage_type, :string, :default => "blob"
|
21
22
|
config_param :azure_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
|
22
|
-
config_param :store_as, :string, :default =>
|
23
|
+
config_param :store_as, :string, :default => "gzip"
|
23
24
|
config_param :auto_create_container, :bool, :default => true
|
24
|
-
config_param :format, :string, :default =>
|
25
|
+
config_param :format, :string, :default => "out_file"
|
25
26
|
config_param :command_parameter, :string, :default => nil
|
26
27
|
|
28
|
+
attr_reader :bs
|
29
|
+
|
27
30
|
include Fluent::Mixin::ConfigPlaceholders
|
28
31
|
|
29
32
|
def placeholders
|
@@ -33,31 +36,13 @@ module Fluent
|
|
33
36
|
def configure(conf)
|
34
37
|
super
|
35
38
|
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
when 'lzma2'
|
44
|
-
check_command('xz', 'LZMA2')
|
45
|
-
@command_paramter = '-qf0' if @command_parameter.nil?
|
46
|
-
'xz'
|
47
|
-
when 'json'
|
48
|
-
'json'
|
49
|
-
else
|
50
|
-
'txt'
|
51
|
-
end
|
52
|
-
|
53
|
-
@storage_type = case @azure_storage_type
|
54
|
-
when 'tables'
|
55
|
-
raise NotImplementedError
|
56
|
-
when 'queues'
|
57
|
-
raise NotImplementedError
|
58
|
-
else
|
59
|
-
'blob'
|
60
|
-
end
|
39
|
+
begin
|
40
|
+
@compressor = COMPRESSOR_REGISTRY.lookup(@store_as).new(:buffer_type => @buffer_type, :log => log)
|
41
|
+
rescue => e
|
42
|
+
$log.warn "#{@store_as} not found. Use 'text' instead"
|
43
|
+
@compressor = TextCompressor.new
|
44
|
+
end
|
45
|
+
@compressor.configure(conf)
|
61
46
|
|
62
47
|
@formatter = Plugin.new_formatter(@format)
|
63
48
|
@formatter.configure(conf)
|
@@ -75,6 +60,15 @@ module Fluent
|
|
75
60
|
if @azure_container.nil?
|
76
61
|
raise ConfigError, 'azure_container is needed'
|
77
62
|
end
|
63
|
+
|
64
|
+
@storage_type = case @azure_storage_type
|
65
|
+
when 'tables'
|
66
|
+
raise NotImplementedError
|
67
|
+
when 'queues'
|
68
|
+
raise NotImplementedError
|
69
|
+
else
|
70
|
+
'blob'
|
71
|
+
end
|
78
72
|
end
|
79
73
|
|
80
74
|
def start
|
@@ -102,15 +96,15 @@ module Fluent
|
|
102
96
|
begin
|
103
97
|
path = @path_slicer.call(@path)
|
104
98
|
values_for_object_key = {
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
99
|
+
"path" => path,
|
100
|
+
"time_slice" => chunk.key,
|
101
|
+
"file_extension" => @compressor.ext,
|
102
|
+
"index" => i,
|
103
|
+
"uuid_flush" => uuid_random
|
109
104
|
}
|
110
105
|
storage_path = @azure_object_key_format.gsub(%r(%{[^}]+})) { |expr|
|
111
106
|
values_for_object_key[expr[2...expr.size-1]]
|
112
107
|
}
|
113
|
-
|
114
108
|
if (i > 0) && (storage_path == previous_path)
|
115
109
|
raise "duplicated path is generated. use %{index} in azure_object_key_format: path = #{storage_path}"
|
116
110
|
end
|
@@ -118,36 +112,13 @@ module Fluent
|
|
118
112
|
i += 1
|
119
113
|
previous_path = storage_path
|
120
114
|
end while blob_exists?(@azure_container, storage_path)
|
121
|
-
|
115
|
+
|
122
116
|
tmp = Tempfile.new("azure-")
|
123
117
|
begin
|
124
|
-
|
125
|
-
|
126
|
-
w = Zlib::GzipWriter.new(tmp)
|
127
|
-
chunk.write_to(w)
|
128
|
-
w.close
|
129
|
-
when 'lzo'
|
130
|
-
w = Tempfile.new('chunk-tmp')
|
131
|
-
chunk.write_to(w)
|
132
|
-
w.close
|
133
|
-
tmp.close
|
134
|
-
system "lzop #{@command_parameter} -o #{tmp.path} #{w.path}"
|
135
|
-
when 'lzma2'
|
136
|
-
w = Tempfile.new('chunk-xz-tmp')
|
137
|
-
chunk.write_to(w)
|
138
|
-
w.close
|
139
|
-
tmp.close
|
140
|
-
system "xz #{@command_parameter} -c #{w.path} > #{tmp.path}"
|
141
|
-
else
|
142
|
-
chunk.write_to(tmp)
|
143
|
-
tmp.close
|
144
|
-
end
|
118
|
+
@compressor.compress(chunk, tmp)
|
119
|
+
tmp.close
|
145
120
|
content = File.open(tmp.path, 'rb') { |file| file.read }
|
146
121
|
@bs.create_block_blob(@azure_container, storage_path, content)
|
147
|
-
ensure
|
148
|
-
tmp.close(true) rescue nil
|
149
|
-
w.close rescue nil
|
150
|
-
w.unlink rescue nil
|
151
122
|
end
|
152
123
|
end
|
153
124
|
|
@@ -162,12 +133,97 @@ module Fluent
|
|
162
133
|
end
|
163
134
|
end
|
164
135
|
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
136
|
+
class Compressor
|
137
|
+
include Configurable
|
138
|
+
|
139
|
+
def initialize(opts = {})
|
140
|
+
super()
|
141
|
+
@buffer_type = opts[:buffer_type]
|
142
|
+
@log = opts[:log]
|
170
143
|
end
|
144
|
+
|
145
|
+
attr_reader :buffer_type, :log
|
146
|
+
|
147
|
+
def configure(conf)
|
148
|
+
super
|
149
|
+
end
|
150
|
+
|
151
|
+
def ext
|
152
|
+
end
|
153
|
+
|
154
|
+
def content_type
|
155
|
+
end
|
156
|
+
|
157
|
+
def compress(chunk, tmp)
|
158
|
+
end
|
159
|
+
|
160
|
+
private
|
161
|
+
|
162
|
+
def check_command(command, algo = nil)
|
163
|
+
require 'open3'
|
164
|
+
|
165
|
+
algo = command if algo.nil?
|
166
|
+
begin
|
167
|
+
Open3.capture3("#{command} -V")
|
168
|
+
rescue Errno::ENOENT
|
169
|
+
raise ConfigError, "'#{command}' utility must be in PATH for #{algo} compression"
|
170
|
+
end
|
171
|
+
end
|
172
|
+
end
|
173
|
+
|
174
|
+
class GzipCompressor < Compressor
|
175
|
+
def ext
|
176
|
+
'gz'.freeze
|
177
|
+
end
|
178
|
+
|
179
|
+
def content_type
|
180
|
+
'application/x-gzip'.freeze
|
181
|
+
end
|
182
|
+
|
183
|
+
def compress(chunk, tmp)
|
184
|
+
w = Zlib::GzipWriter.new(tmp)
|
185
|
+
chunk.write_to(w)
|
186
|
+
w.finish
|
187
|
+
ensure
|
188
|
+
w.finish rescue nil
|
189
|
+
end
|
190
|
+
end
|
191
|
+
|
192
|
+
class TextCompressor < Compressor
|
193
|
+
def ext
|
194
|
+
'txt'.freeze
|
195
|
+
end
|
196
|
+
|
197
|
+
def content_type
|
198
|
+
'text/plain'.freeze
|
199
|
+
end
|
200
|
+
|
201
|
+
def compress(chunk, tmp)
|
202
|
+
chunk.write_to(tmp)
|
203
|
+
end
|
204
|
+
end
|
205
|
+
|
206
|
+
class JsonCompressor < TextCompressor
|
207
|
+
def ext
|
208
|
+
'json'.freeze
|
209
|
+
end
|
210
|
+
|
211
|
+
def content_type
|
212
|
+
'application/json'.freeze
|
213
|
+
end
|
214
|
+
end
|
215
|
+
|
216
|
+
COMPRESSOR_REGISTRY = Registry.new(:azurestorage_compressor_type, 'fluent/plugin/azurestorage_compressor_')
|
217
|
+
{
|
218
|
+
'gzip' => GzipCompressor,
|
219
|
+
'json' => JsonCompressor,
|
220
|
+
'text' => TextCompressor
|
221
|
+
}.each { |name, compressor|
|
222
|
+
COMPRESSOR_REGISTRY.register(name, compressor)
|
223
|
+
}
|
224
|
+
|
225
|
+
def self.register_compressor(name, compressor)
|
226
|
+
COMPRESSOR_REGISTRY.register(name, compressor)
|
171
227
|
end
|
172
228
|
|
173
229
|
def blob_exists?(container, blob)
|
@@ -0,0 +1,224 @@
|
|
1
|
+
require 'fluent/test'
|
2
|
+
require 'fluent/plugin/out_azurestorage'
|
3
|
+
|
4
|
+
require 'test/unit/rr'
|
5
|
+
require 'zlib'
|
6
|
+
require 'fileutils'
|
7
|
+
|
8
|
+
class AzureStorageOutputTest < Test::Unit::TestCase
|
9
|
+
def setup
|
10
|
+
require 'azure'
|
11
|
+
Fluent::Test.setup
|
12
|
+
end
|
13
|
+
|
14
|
+
CONFIG = %[
|
15
|
+
azure_storage_account test_storage_account
|
16
|
+
azure_storage_access_key dGVzdF9zdG9yYWdlX2FjY2Vzc19rZXk=
|
17
|
+
azure_container test_container
|
18
|
+
path log
|
19
|
+
utc
|
20
|
+
buffer_type memory
|
21
|
+
]
|
22
|
+
|
23
|
+
def create_driver(conf = CONFIG)
|
24
|
+
Fluent::Test::BufferedOutputTestDriver.new(Fluent::AzureStorageOutput) do
|
25
|
+
def write(chunk)
|
26
|
+
chunk.read
|
27
|
+
end
|
28
|
+
|
29
|
+
private
|
30
|
+
|
31
|
+
def ensure_container
|
32
|
+
end
|
33
|
+
|
34
|
+
end.configure(conf)
|
35
|
+
end
|
36
|
+
|
37
|
+
def test_configure
|
38
|
+
d = create_driver
|
39
|
+
assert_equal 'test_storage_account', d.instance.azure_storage_account
|
40
|
+
assert_equal 'dGVzdF9zdG9yYWdlX2FjY2Vzc19rZXk=', d.instance.azure_storage_access_key
|
41
|
+
assert_equal 'test_container', d.instance.azure_container
|
42
|
+
assert_equal 'log', d.instance.path
|
43
|
+
assert_equal 'gz', d.instance.instance_variable_get(:@compressor).ext
|
44
|
+
assert_equal 'application/x-gzip', d.instance.instance_variable_get(:@compressor).content_type
|
45
|
+
end
|
46
|
+
|
47
|
+
def test_configure_with_mime_type_json
|
48
|
+
conf = CONFIG.clone
|
49
|
+
conf << "\nstore_as json\n"
|
50
|
+
d = create_driver(conf)
|
51
|
+
assert_equal 'json', d.instance.instance_variable_get(:@compressor).ext
|
52
|
+
assert_equal 'application/json', d.instance.instance_variable_get(:@compressor).content_type
|
53
|
+
end
|
54
|
+
|
55
|
+
def test_configure_with_mime_type_text
|
56
|
+
conf = CONFIG.clone
|
57
|
+
conf << "\nstore_as text\n"
|
58
|
+
d = create_driver(conf)
|
59
|
+
assert_equal 'txt', d.instance.instance_variable_get(:@compressor).ext
|
60
|
+
assert_equal 'text/plain', d.instance.instance_variable_get(:@compressor).content_type
|
61
|
+
end
|
62
|
+
|
63
|
+
def test_configure_with_mime_type_lzo
|
64
|
+
conf = CONFIG.clone
|
65
|
+
conf << "\nstore_as lzo\n"
|
66
|
+
d = create_driver(conf)
|
67
|
+
assert_equal 'lzo', d.instance.instance_variable_get(:@compressor).ext
|
68
|
+
assert_equal 'application/x-lzop', d.instance.instance_variable_get(:@compressor).content_type
|
69
|
+
rescue => e
|
70
|
+
# TODO: replace code with disable lzop command
|
71
|
+
assert(e.is_a?(Fluent::ConfigError))
|
72
|
+
end
|
73
|
+
|
74
|
+
def test_path_slicing
|
75
|
+
config = CONFIG.clone.gsub(/path\slog/, "path log/%Y/%m/%d")
|
76
|
+
d = create_driver(config)
|
77
|
+
path_slicer = d.instance.instance_variable_get(:@path_slicer)
|
78
|
+
path = d.instance.instance_variable_get(:@path)
|
79
|
+
slice = path_slicer.call(path)
|
80
|
+
assert_equal slice, Time.now.utc.strftime("log/%Y/%m/%d")
|
81
|
+
end
|
82
|
+
|
83
|
+
def test_path_slicing_utc
|
84
|
+
config = CONFIG.clone.gsub(/path\slog/, "path log/%Y/%m/%d")
|
85
|
+
config << "\nutc\n"
|
86
|
+
d = create_driver(config)
|
87
|
+
path_slicer = d.instance.instance_variable_get(:@path_slicer)
|
88
|
+
path = d.instance.instance_variable_get(:@path)
|
89
|
+
slice = path_slicer.call(path)
|
90
|
+
assert_equal slice, Time.now.utc.strftime("log/%Y/%m/%d")
|
91
|
+
end
|
92
|
+
|
93
|
+
def test_format
|
94
|
+
d = create_driver
|
95
|
+
|
96
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
97
|
+
d.emit({"a"=>1}, time)
|
98
|
+
d.emit({"a"=>2}, time)
|
99
|
+
|
100
|
+
d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n]
|
101
|
+
d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
|
102
|
+
|
103
|
+
d.run
|
104
|
+
end
|
105
|
+
|
106
|
+
def test_format_included_tag_and_time
|
107
|
+
config = [CONFIG, 'include_tag_key true', 'include_time_key true'].join("\n")
|
108
|
+
d = create_driver(config)
|
109
|
+
|
110
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
111
|
+
d.emit({"a"=>1}, time)
|
112
|
+
d.emit({"a"=>2}, time)
|
113
|
+
|
114
|
+
d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":1,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
|
115
|
+
d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":2,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
|
116
|
+
|
117
|
+
d.run
|
118
|
+
end
|
119
|
+
|
120
|
+
def test_format_with_format_ltsv
|
121
|
+
config = [CONFIG, 'format ltsv'].join("\n")
|
122
|
+
d = create_driver(config)
|
123
|
+
|
124
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
125
|
+
d.emit({"a"=>1, "b"=>1}, time)
|
126
|
+
d.emit({"a"=>2, "b"=>2}, time)
|
127
|
+
|
128
|
+
d.expect_format %[a:1\tb:1\n]
|
129
|
+
d.expect_format %[a:2\tb:2\n]
|
130
|
+
|
131
|
+
d.run
|
132
|
+
end
|
133
|
+
|
134
|
+
def test_format_with_format_json
|
135
|
+
config = [CONFIG, 'format json'].join("\n")
|
136
|
+
d = create_driver(config)
|
137
|
+
|
138
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
139
|
+
d.emit({"a"=>1}, time)
|
140
|
+
d.emit({"a"=>2}, time)
|
141
|
+
|
142
|
+
d.expect_format %[{"a":1}\n]
|
143
|
+
d.expect_format %[{"a":2}\n]
|
144
|
+
|
145
|
+
d.run
|
146
|
+
end
|
147
|
+
|
148
|
+
def test_format_with_format_json_included_tag
|
149
|
+
config = [CONFIG, 'format json', 'include_tag_key true'].join("\n")
|
150
|
+
d = create_driver(config)
|
151
|
+
|
152
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
153
|
+
d.emit({"a"=>1}, time)
|
154
|
+
d.emit({"a"=>2}, time)
|
155
|
+
|
156
|
+
d.expect_format %[{"a":1,"tag":"test"}\n]
|
157
|
+
d.expect_format %[{"a":2,"tag":"test"}\n]
|
158
|
+
|
159
|
+
d.run
|
160
|
+
end
|
161
|
+
|
162
|
+
def test_format_with_format_json_included_time
|
163
|
+
config = [CONFIG, 'format json', 'include_time_key true'].join("\n")
|
164
|
+
d = create_driver(config)
|
165
|
+
|
166
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
167
|
+
d.emit({"a"=>1}, time)
|
168
|
+
d.emit({"a"=>2}, time)
|
169
|
+
|
170
|
+
d.expect_format %[{"a":1,"time":"2011-01-02T13:14:15Z"}\n]
|
171
|
+
d.expect_format %[{"a":2,"time":"2011-01-02T13:14:15Z"}\n]
|
172
|
+
|
173
|
+
d.run
|
174
|
+
end
|
175
|
+
|
176
|
+
def test_format_with_format_json_included_tag_and_time
|
177
|
+
config = [CONFIG, 'format json', 'include_tag_key true', 'include_time_key true'].join("\n")
|
178
|
+
d = create_driver(config)
|
179
|
+
|
180
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
181
|
+
d.emit({"a"=>1}, time)
|
182
|
+
d.emit({"a"=>2}, time)
|
183
|
+
|
184
|
+
d.expect_format %[{"a":1,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
|
185
|
+
d.expect_format %[{"a":2,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
|
186
|
+
|
187
|
+
d.run
|
188
|
+
end
|
189
|
+
|
190
|
+
def test_chunk_to_write
|
191
|
+
d = create_driver
|
192
|
+
|
193
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
194
|
+
d.emit({"a"=>1}, time)
|
195
|
+
d.emit({"a"=>2}, time)
|
196
|
+
|
197
|
+
# AzureStorageOutputTest#write returns chunk.read
|
198
|
+
data = d.run
|
199
|
+
|
200
|
+
assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
|
201
|
+
%[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
|
202
|
+
data
|
203
|
+
end
|
204
|
+
|
205
|
+
CONFIG_TIME_SLICE = %[
|
206
|
+
hostname testing.node.local
|
207
|
+
azure_storage_account test_storage_account
|
208
|
+
azure_storage_access_key dGVzdF9zdG9yYWdlX2FjY2Vzc19rZXk=
|
209
|
+
azure_container test_container
|
210
|
+
azure_object_key_format %{path}/events/ts=%{time_slice}/events_%{index}-%{hostname}.%{file_extension}
|
211
|
+
time_slice_format %Y%m%d-%H
|
212
|
+
path log
|
213
|
+
utc
|
214
|
+
buffer_type memory
|
215
|
+
log_level debug
|
216
|
+
]
|
217
|
+
|
218
|
+
def create_time_sliced_driver(conf = CONFIG_TIME_SLICE)
|
219
|
+
d = Fluent::Test::TimeSlicedOutputTestDriver.new(Fluent::AzureStorageOutput) do
|
220
|
+
end.configure(conf)
|
221
|
+
d
|
222
|
+
end
|
223
|
+
|
224
|
+
end
|
metadata
CHANGED
@@ -1,43 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-azurestorage
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Hidemasa Togashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-09
|
11
|
+
date: 2015-10-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
|
-
- !ruby/object:Gem::Dependency
|
14
|
-
name: bundler
|
15
|
-
requirement: !ruby/object:Gem::Requirement
|
16
|
-
requirements:
|
17
|
-
- - ~>
|
18
|
-
- !ruby/object:Gem::Version
|
19
|
-
version: '1.3'
|
20
|
-
type: :development
|
21
|
-
prerelease: false
|
22
|
-
version_requirements: !ruby/object:Gem::Requirement
|
23
|
-
requirements:
|
24
|
-
- - ~>
|
25
|
-
- !ruby/object:Gem::Version
|
26
|
-
version: '1.3'
|
27
|
-
- !ruby/object:Gem::Dependency
|
28
|
-
name: rake
|
29
|
-
requirement: !ruby/object:Gem::Requirement
|
30
|
-
requirements:
|
31
|
-
- - '>='
|
32
|
-
- !ruby/object:Gem::Version
|
33
|
-
version: '0'
|
34
|
-
type: :development
|
35
|
-
prerelease: false
|
36
|
-
version_requirements: !ruby/object:Gem::Requirement
|
37
|
-
requirements:
|
38
|
-
- - '>='
|
39
|
-
- !ruby/object:Gem::Version
|
40
|
-
version: '0'
|
41
13
|
- !ruby/object:Gem::Dependency
|
42
14
|
name: fluentd
|
43
15
|
requirement: !ruby/object:Gem::Requirement
|
@@ -58,35 +30,49 @@ dependencies:
|
|
58
30
|
- - <
|
59
31
|
- !ruby/object:Gem::Version
|
60
32
|
version: '2'
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
name: azure
|
35
|
+
requirement: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - '='
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: 0.6.4
|
40
|
+
type: :runtime
|
41
|
+
prerelease: false
|
42
|
+
version_requirements: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - '='
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: 0.6.4
|
61
47
|
- !ruby/object:Gem::Dependency
|
62
48
|
name: fluent-mixin-config-placeholders
|
63
49
|
requirement: !ruby/object:Gem::Requirement
|
64
50
|
requirements:
|
65
51
|
- - '>='
|
66
52
|
- !ruby/object:Gem::Version
|
67
|
-
version:
|
53
|
+
version: 0.3.0
|
68
54
|
type: :runtime
|
69
55
|
prerelease: false
|
70
56
|
version_requirements: !ruby/object:Gem::Requirement
|
71
57
|
requirements:
|
72
58
|
- - '>='
|
73
59
|
- !ruby/object:Gem::Version
|
74
|
-
version:
|
60
|
+
version: 0.3.0
|
75
61
|
- !ruby/object:Gem::Dependency
|
76
|
-
name:
|
62
|
+
name: rake
|
77
63
|
requirement: !ruby/object:Gem::Requirement
|
78
64
|
requirements:
|
79
|
-
- - '
|
65
|
+
- - '>='
|
80
66
|
- !ruby/object:Gem::Version
|
81
|
-
version: 0.
|
82
|
-
type: :
|
67
|
+
version: 0.9.2
|
68
|
+
type: :development
|
83
69
|
prerelease: false
|
84
70
|
version_requirements: !ruby/object:Gem::Requirement
|
85
71
|
requirements:
|
86
|
-
- - '
|
72
|
+
- - '>='
|
87
73
|
- !ruby/object:Gem::Version
|
88
|
-
version: 0.
|
89
|
-
description:
|
74
|
+
version: 0.9.2
|
75
|
+
description: Azure Storage output plugin for Fluentd event collector
|
90
76
|
email:
|
91
77
|
- togachiro@gmail.com
|
92
78
|
executables: []
|
@@ -94,15 +80,21 @@ extensions: []
|
|
94
80
|
extra_rdoc_files: []
|
95
81
|
files:
|
96
82
|
- .gitignore
|
83
|
+
- .travis.yml
|
97
84
|
- Gemfile
|
98
85
|
- LICENSE.txt
|
99
86
|
- README.md
|
100
87
|
- Rakefile
|
88
|
+
- VERSION
|
101
89
|
- fluent-plugin-azurestorage.gemspec
|
90
|
+
- lib/fluent/plugin/azurestorage_compressor_gzip_command.rb
|
91
|
+
- lib/fluent/plugin/azurestorage_compressor_lzma2.rb
|
92
|
+
- lib/fluent/plugin/azurestorage_compressor_lzo.rb
|
102
93
|
- lib/fluent/plugin/out_azurestorage.rb
|
103
|
-
|
94
|
+
- test/test_out_azurestorage.rb
|
95
|
+
homepage: https://github.com/htgc/fluent-plugin-azurestorage
|
104
96
|
licenses:
|
105
|
-
- Apache
|
97
|
+
- Apache-2.0
|
106
98
|
metadata: {}
|
107
99
|
post_install_message:
|
108
100
|
rdoc_options: []
|
@@ -123,5 +115,6 @@ rubyforge_project:
|
|
123
115
|
rubygems_version: 2.0.14
|
124
116
|
signing_key:
|
125
117
|
specification_version: 4
|
126
|
-
summary:
|
127
|
-
test_files:
|
118
|
+
summary: Azure Storage output plugin for Fluentd event collector
|
119
|
+
test_files:
|
120
|
+
- test/test_out_azurestorage.rb
|