fluent-plugin-azurestorage 0.0.5 → 0.0.6

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: b95e30f563028d1a96e4c6a5894c77e26b35c989
4
- data.tar.gz: 2a99b05c82539e6e19250dcfc63ba91b2664522c
3
+ metadata.gz: 8bb81cd14c71e59c606798aacf0fcbf3816d65c8
4
+ data.tar.gz: b520ba98690443f89e9f683248ba321675de0ffa
5
5
  SHA512:
6
- metadata.gz: 287c7183bbddb01a3ee82c991362c1ec2468e724fd685f297731a70a74857f070746c6a04fead67128bd8deae9a15e3e5ef2c98a6e582d172de76b75a37cd7fa
7
- data.tar.gz: 0212f5a6a4758bfa01c16d2ff8c0c41e2d0ea0b4e1d2044cc24de98e13f2c71620df10b41b9f7235c986e8b3e2e7ca478dbe695ee58cfa9e9c58557952590b72
6
+ metadata.gz: 98704f9ebff04476910ff2292e141fd8bf11f6831a27d429cdcecf16a8a2053bee071dbb2c8310b7ed600a42d71d7ddf52111b019b4cf968ef6704eda3de0113
7
+ data.tar.gz: 06eac631498041b2c523b926587afc49dd90d13c39ba92a527beb8e04ea9369449885ab2dc0a055282af090491c7b5ca3fe8a016bc2569a083628d99e815ae39
data/.gitignore CHANGED
@@ -1,17 +1,11 @@
1
+ ~*
2
+ #*
3
+ *~
4
+ [._]*.s[a-w][a-z]
5
+ .DS_Store
6
+
1
7
  *.gem
2
- *.rbc
3
8
  .bundle
4
- .config
5
- .yardoc
6
9
  Gemfile.lock
7
- InstalledFiles
8
- _yardoc
9
- coverage
10
- doc/
11
- lib/bundler/man
12
- pkg
13
- rdoc
14
- spec/reports
15
- test/tmp
16
- test/version_tmp
17
- tmp
10
+ vendor
11
+ .ruby-version
data/.travis.yml ADDED
@@ -0,0 +1,21 @@
1
+ language: ruby
2
+
3
+ rvm:
4
+ - 1.9.3
5
+ - 2.0
6
+ - 2.1
7
+ - 2.2.0
8
+ - rbx
9
+
10
+ gemfile:
11
+ - Gemfile
12
+
13
+ branches:
14
+ only:
15
+ - master
16
+
17
+ script: bundle exec rake test
18
+
19
+ matrix:
20
+ allow_failures:
21
+ - rvm: rbx
data/Gemfile CHANGED
@@ -1,4 +1,3 @@
1
- source 'https://rubygems.org'
1
+ source "http://rubygems.org"
2
2
 
3
- # Specify your gem's dependencies in fluent-plugin-azurestorage.gemspec
4
3
  gemspec
data/Rakefile CHANGED
@@ -1 +1,14 @@
1
- require "bundler/gem_tasks"
1
+
2
+ require 'bundler'
3
+ Bundler::GemHelper.install_tasks
4
+
5
+ require 'rake/testtask'
6
+
7
+ Rake::TestTask.new(:test) do |test|
8
+ test.libs << 'lib' << 'test'
9
+ test.test_files = FileList['test/test_*.rb']
10
+ test.verbose = true
11
+ end
12
+
13
+ task :default => [:build]
14
+
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 0.0.6
@@ -1,26 +1,24 @@
1
- # coding: utf-8
2
- #lib = File.expand_path('../lib', __FILE__)
3
- #$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
1
+ # encoding: utf-8
2
+ $:.push File.expand_path('../lib', __FILE__)
4
3
 
5
- Gem::Specification.new do |spec|
6
- spec.name = "fluent-plugin-azurestorage"
7
- spec.version = "0.0.5"
8
- spec.authors = ["Hidemasa Togashi"]
9
- spec.email = ["togachiro@gmail.com"]
10
- spec.description = %q{Fluent plugin for store to Azure Storage}
11
- spec.summary = %q{Fluent plugin for store to Azure Storage}
12
- spec.homepage = ""
13
- spec.license = "Apache License, version 2."
4
+ Gem::Specification.new do |gem|
5
+ gem.name = "fluent-plugin-azurestorage"
6
+ gem.description = "Azure Storage output plugin for Fluentd event collector"
7
+ gem.license = "Apache-2.0"
8
+ gem.homepage = "https://github.com/htgc/fluent-plugin-azurestorage"
9
+ gem.summary = gem.description
10
+ gem.version = File.read("VERSION").strip
11
+ gem.authors = ["Hidemasa Togashi"]
12
+ gem.email = ["togachiro@gmail.com"]
13
+ gem.has_rdoc = false
14
+ #gem.platform = Gem::Platform::RUBY
15
+ gem.files = `git ls-files`.split("\n")
16
+ gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
17
+ gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
18
+ gem.require_paths = ['lib']
14
19
 
15
- spec.files = `git ls-files -z`.split("\x0")
16
- spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
17
- spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
18
- spec.require_paths = ["lib"]
19
-
20
- spec.add_development_dependency "bundler", "~> 1.3"
21
- spec.add_development_dependency "rake"
22
-
23
- spec.add_runtime_dependency "fluentd", [">= 0.10.58", "< 2"]
24
- spec.add_runtime_dependency "fluent-mixin-config-placeholders"
25
- spec.add_runtime_dependency "azure", "0.6.4"
20
+ gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
21
+ gem.add_dependency "azure", "0.6.4"
22
+ gem.add_dependency "fluent-mixin-config-placeholders", ">= 0.3.0"
23
+ gem.add_development_dependency "rake", ">= 0.9.2"
26
24
  end
@@ -0,0 +1,51 @@
1
+ module Fluent
2
+ class AzureStorageOutput
3
+ class GzipCommandCompressor < Compressor
4
+ AzureStorageOutput.register_compressor('gzip_command', self)
5
+
6
+ config_param :command_parameter, :string, :default => ''
7
+
8
+ def configure(conf)
9
+ super
10
+ check_command('gzip')
11
+ end
12
+
13
+ def ext
14
+ 'gz'.freeze
15
+ end
16
+
17
+ def content_type
18
+ 'application/x-gzip'.freeze
19
+ end
20
+
21
+ def compress(chunk, tmp)
22
+ chunk_is_file = @buffer_type == 'file'
23
+ path = if chunk_is_file
24
+ chunk.path
25
+ else
26
+ w = Tempfile.new("chunk-gzip-tmp")
27
+ chunk.write_to(w)
28
+ w.close
29
+ w.path
30
+ end
31
+
32
+ res = system "gzip #{@command_parameter} -c #{path} > #{tmp.path}"
33
+ unless res
34
+ log.warn "failed to execute gzip command. Fallback to GzipWriter. status = #{$?}"
35
+ begin
36
+ tmp.truncate(0)
37
+ gw = Zlib::GzipWriter.new(tmp)
38
+ chunk.write_to(gw)
39
+ gw.close
40
+ ensure
41
+ gw.close rescue nil
42
+ end
43
+ end
44
+ ensure
45
+ unless chunk_is_file
46
+ w.close(true) rescue nil
47
+ end
48
+ end
49
+ end
50
+ end
51
+ end
@@ -0,0 +1,34 @@
1
+ module Fluent
2
+ class AzureStorageOutput
3
+ class LZMA2Compressor < Compressor
4
+ AzureStorageOutput.register_compressor('lzma2', self)
5
+
6
+ config_param :command_parameter, :string, :default => '-qf0'
7
+
8
+ def configure(conf)
9
+ super
10
+ check_command('xz', 'LZMA2')
11
+ end
12
+
13
+ def ext
14
+ 'xz'.freeze
15
+ end
16
+
17
+ def content_type
18
+ 'application/x-xz'.freeze
19
+ end
20
+
21
+ def compress(chunk, tmp)
22
+ w = Tempfile.new("chunk-xz-tmp")
23
+ chunk.write_to(w)
24
+ w.close
25
+
26
+ # We don't check the return code because we can't recover lzop failure.
27
+ system "xz #{@command_parameter} -c #{w.path} > #{tmp.path}"
28
+ ensure
29
+ w.close rescue nil
30
+ w.unlink rescue nil
31
+ end
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,34 @@
1
+ module Fluent
2
+ class AzureStorageOutput
3
+ class LZOCompressor < Compressor
4
+ AzureStorageOutput.register_compressor('lzo', self)
5
+
6
+ config_param :command_parameter, :string, :default => '-qf1'
7
+
8
+ def configure(conf)
9
+ super
10
+ check_command('lzop', 'LZO')
11
+ end
12
+
13
+ def ext
14
+ 'lzo'.freeze
15
+ end
16
+
17
+ def content_type
18
+ 'application/x-lzop'.freeze
19
+ end
20
+
21
+ def compress(chunk, tmp)
22
+ w = Tempfile.new("chunk-tmp")
23
+ chunk.write_to(w)
24
+ w.close
25
+
26
+ # We don't check the return code because we can't recover lzop failure.
27
+ system "lzop #{@command_parameter} -o #{tmp.path} #{w.path}"
28
+ ensure
29
+ w.close rescue nil
30
+ w.unlink rescue nil
31
+ end
32
+ end
33
+ end
34
+ end
@@ -10,20 +10,23 @@ module Fluent
10
10
  require 'zlib'
11
11
  require 'time'
12
12
  require 'tempfile'
13
- require 'open3'
13
+
14
+ @compressor = nil
14
15
  end
15
16
 
16
- config_param :path, :string, :default => ''
17
+ config_param :path, :string, :default => ""
17
18
  config_param :azure_storage_account, :string, :default => nil
18
19
  config_param :azure_storage_access_key, :string, :default => nil, :secret => true
19
20
  config_param :azure_container, :string, :default => nil
20
- config_param :azure_storage_type, :string, :default => 'blob'
21
+ config_param :azure_storage_type, :string, :default => "blob"
21
22
  config_param :azure_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
22
- config_param :store_as, :string, :default => 'gzip'
23
+ config_param :store_as, :string, :default => "gzip"
23
24
  config_param :auto_create_container, :bool, :default => true
24
- config_param :format, :string, :default => 'out_file'
25
+ config_param :format, :string, :default => "out_file"
25
26
  config_param :command_parameter, :string, :default => nil
26
27
 
28
+ attr_reader :bs
29
+
27
30
  include Fluent::Mixin::ConfigPlaceholders
28
31
 
29
32
  def placeholders
@@ -33,31 +36,13 @@ module Fluent
33
36
  def configure(conf)
34
37
  super
35
38
 
36
- @ext = case @store_as
37
- when 'gzip'
38
- 'gz'
39
- when 'lzo'
40
- check_command('lzop', 'LZO')
41
- @command_parameter = '-qf1' if @command_parameter.nil?
42
- 'lzo'
43
- when 'lzma2'
44
- check_command('xz', 'LZMA2')
45
- @command_paramter = '-qf0' if @command_parameter.nil?
46
- 'xz'
47
- when 'json'
48
- 'json'
49
- else
50
- 'txt'
51
- end
52
-
53
- @storage_type = case @azure_storage_type
54
- when 'tables'
55
- raise NotImplementedError
56
- when 'queues'
57
- raise NotImplementedError
58
- else
59
- 'blob'
60
- end
39
+ begin
40
+ @compressor = COMPRESSOR_REGISTRY.lookup(@store_as).new(:buffer_type => @buffer_type, :log => log)
41
+ rescue => e
42
+ $log.warn "#{@store_as} not found. Use 'text' instead"
43
+ @compressor = TextCompressor.new
44
+ end
45
+ @compressor.configure(conf)
61
46
 
62
47
  @formatter = Plugin.new_formatter(@format)
63
48
  @formatter.configure(conf)
@@ -75,6 +60,15 @@ module Fluent
75
60
  if @azure_container.nil?
76
61
  raise ConfigError, 'azure_container is needed'
77
62
  end
63
+
64
+ @storage_type = case @azure_storage_type
65
+ when 'tables'
66
+ raise NotImplementedError
67
+ when 'queues'
68
+ raise NotImplementedError
69
+ else
70
+ 'blob'
71
+ end
78
72
  end
79
73
 
80
74
  def start
@@ -102,15 +96,15 @@ module Fluent
102
96
  begin
103
97
  path = @path_slicer.call(@path)
104
98
  values_for_object_key = {
105
- 'path' => path,
106
- 'time_slice' => chunk.key,
107
- 'file_extension' => @ext,
108
- 'index' => i
99
+ "path" => path,
100
+ "time_slice" => chunk.key,
101
+ "file_extension" => @compressor.ext,
102
+ "index" => i,
103
+ "uuid_flush" => uuid_random
109
104
  }
110
105
  storage_path = @azure_object_key_format.gsub(%r(%{[^}]+})) { |expr|
111
106
  values_for_object_key[expr[2...expr.size-1]]
112
107
  }
113
-
114
108
  if (i > 0) && (storage_path == previous_path)
115
109
  raise "duplicated path is generated. use %{index} in azure_object_key_format: path = #{storage_path}"
116
110
  end
@@ -118,36 +112,13 @@ module Fluent
118
112
  i += 1
119
113
  previous_path = storage_path
120
114
  end while blob_exists?(@azure_container, storage_path)
121
-
115
+
122
116
  tmp = Tempfile.new("azure-")
123
117
  begin
124
- case @store_as
125
- when 'gzip'
126
- w = Zlib::GzipWriter.new(tmp)
127
- chunk.write_to(w)
128
- w.close
129
- when 'lzo'
130
- w = Tempfile.new('chunk-tmp')
131
- chunk.write_to(w)
132
- w.close
133
- tmp.close
134
- system "lzop #{@command_parameter} -o #{tmp.path} #{w.path}"
135
- when 'lzma2'
136
- w = Tempfile.new('chunk-xz-tmp')
137
- chunk.write_to(w)
138
- w.close
139
- tmp.close
140
- system "xz #{@command_parameter} -c #{w.path} > #{tmp.path}"
141
- else
142
- chunk.write_to(tmp)
143
- tmp.close
144
- end
118
+ @compressor.compress(chunk, tmp)
119
+ tmp.close
145
120
  content = File.open(tmp.path, 'rb') { |file| file.read }
146
121
  @bs.create_block_blob(@azure_container, storage_path, content)
147
- ensure
148
- tmp.close(true) rescue nil
149
- w.close rescue nil
150
- w.unlink rescue nil
151
122
  end
152
123
  end
153
124
 
@@ -162,12 +133,97 @@ module Fluent
162
133
  end
163
134
  end
164
135
 
165
- def check_command(command, algo)
166
- begin
167
- Open3.capture3("#{command} -V")
168
- rescue Errno::ENOENT
169
- raise ConfigError, "'#{command}' utility must be in PATH for #{algo} compression"
136
+ class Compressor
137
+ include Configurable
138
+
139
+ def initialize(opts = {})
140
+ super()
141
+ @buffer_type = opts[:buffer_type]
142
+ @log = opts[:log]
170
143
  end
144
+
145
+ attr_reader :buffer_type, :log
146
+
147
+ def configure(conf)
148
+ super
149
+ end
150
+
151
+ def ext
152
+ end
153
+
154
+ def content_type
155
+ end
156
+
157
+ def compress(chunk, tmp)
158
+ end
159
+
160
+ private
161
+
162
+ def check_command(command, algo = nil)
163
+ require 'open3'
164
+
165
+ algo = command if algo.nil?
166
+ begin
167
+ Open3.capture3("#{command} -V")
168
+ rescue Errno::ENOENT
169
+ raise ConfigError, "'#{command}' utility must be in PATH for #{algo} compression"
170
+ end
171
+ end
172
+ end
173
+
174
+ class GzipCompressor < Compressor
175
+ def ext
176
+ 'gz'.freeze
177
+ end
178
+
179
+ def content_type
180
+ 'application/x-gzip'.freeze
181
+ end
182
+
183
+ def compress(chunk, tmp)
184
+ w = Zlib::GzipWriter.new(tmp)
185
+ chunk.write_to(w)
186
+ w.finish
187
+ ensure
188
+ w.finish rescue nil
189
+ end
190
+ end
191
+
192
+ class TextCompressor < Compressor
193
+ def ext
194
+ 'txt'.freeze
195
+ end
196
+
197
+ def content_type
198
+ 'text/plain'.freeze
199
+ end
200
+
201
+ def compress(chunk, tmp)
202
+ chunk.write_to(tmp)
203
+ end
204
+ end
205
+
206
+ class JsonCompressor < TextCompressor
207
+ def ext
208
+ 'json'.freeze
209
+ end
210
+
211
+ def content_type
212
+ 'application/json'.freeze
213
+ end
214
+ end
215
+
216
+ COMPRESSOR_REGISTRY = Registry.new(:azurestorage_compressor_type, 'fluent/plugin/azurestorage_compressor_')
217
+ {
218
+ 'gzip' => GzipCompressor,
219
+ 'json' => JsonCompressor,
220
+ 'text' => TextCompressor
221
+ }.each { |name, compressor|
222
+ COMPRESSOR_REGISTRY.register(name, compressor)
223
+ }
224
+
225
+ def self.register_compressor(name, compressor)
226
+ COMPRESSOR_REGISTRY.register(name, compressor)
171
227
  end
172
228
 
173
229
  def blob_exists?(container, blob)
@@ -0,0 +1,224 @@
1
+ require 'fluent/test'
2
+ require 'fluent/plugin/out_azurestorage'
3
+
4
+ require 'test/unit/rr'
5
+ require 'zlib'
6
+ require 'fileutils'
7
+
8
+ class AzureStorageOutputTest < Test::Unit::TestCase
9
+ def setup
10
+ require 'azure'
11
+ Fluent::Test.setup
12
+ end
13
+
14
+ CONFIG = %[
15
+ azure_storage_account test_storage_account
16
+ azure_storage_access_key dGVzdF9zdG9yYWdlX2FjY2Vzc19rZXk=
17
+ azure_container test_container
18
+ path log
19
+ utc
20
+ buffer_type memory
21
+ ]
22
+
23
+ def create_driver(conf = CONFIG)
24
+ Fluent::Test::BufferedOutputTestDriver.new(Fluent::AzureStorageOutput) do
25
+ def write(chunk)
26
+ chunk.read
27
+ end
28
+
29
+ private
30
+
31
+ def ensure_container
32
+ end
33
+
34
+ end.configure(conf)
35
+ end
36
+
37
+ def test_configure
38
+ d = create_driver
39
+ assert_equal 'test_storage_account', d.instance.azure_storage_account
40
+ assert_equal 'dGVzdF9zdG9yYWdlX2FjY2Vzc19rZXk=', d.instance.azure_storage_access_key
41
+ assert_equal 'test_container', d.instance.azure_container
42
+ assert_equal 'log', d.instance.path
43
+ assert_equal 'gz', d.instance.instance_variable_get(:@compressor).ext
44
+ assert_equal 'application/x-gzip', d.instance.instance_variable_get(:@compressor).content_type
45
+ end
46
+
47
+ def test_configure_with_mime_type_json
48
+ conf = CONFIG.clone
49
+ conf << "\nstore_as json\n"
50
+ d = create_driver(conf)
51
+ assert_equal 'json', d.instance.instance_variable_get(:@compressor).ext
52
+ assert_equal 'application/json', d.instance.instance_variable_get(:@compressor).content_type
53
+ end
54
+
55
+ def test_configure_with_mime_type_text
56
+ conf = CONFIG.clone
57
+ conf << "\nstore_as text\n"
58
+ d = create_driver(conf)
59
+ assert_equal 'txt', d.instance.instance_variable_get(:@compressor).ext
60
+ assert_equal 'text/plain', d.instance.instance_variable_get(:@compressor).content_type
61
+ end
62
+
63
+ def test_configure_with_mime_type_lzo
64
+ conf = CONFIG.clone
65
+ conf << "\nstore_as lzo\n"
66
+ d = create_driver(conf)
67
+ assert_equal 'lzo', d.instance.instance_variable_get(:@compressor).ext
68
+ assert_equal 'application/x-lzop', d.instance.instance_variable_get(:@compressor).content_type
69
+ rescue => e
70
+ # TODO: replace code with disable lzop command
71
+ assert(e.is_a?(Fluent::ConfigError))
72
+ end
73
+
74
+ def test_path_slicing
75
+ config = CONFIG.clone.gsub(/path\slog/, "path log/%Y/%m/%d")
76
+ d = create_driver(config)
77
+ path_slicer = d.instance.instance_variable_get(:@path_slicer)
78
+ path = d.instance.instance_variable_get(:@path)
79
+ slice = path_slicer.call(path)
80
+ assert_equal slice, Time.now.utc.strftime("log/%Y/%m/%d")
81
+ end
82
+
83
+ def test_path_slicing_utc
84
+ config = CONFIG.clone.gsub(/path\slog/, "path log/%Y/%m/%d")
85
+ config << "\nutc\n"
86
+ d = create_driver(config)
87
+ path_slicer = d.instance.instance_variable_get(:@path_slicer)
88
+ path = d.instance.instance_variable_get(:@path)
89
+ slice = path_slicer.call(path)
90
+ assert_equal slice, Time.now.utc.strftime("log/%Y/%m/%d")
91
+ end
92
+
93
+ def test_format
94
+ d = create_driver
95
+
96
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
97
+ d.emit({"a"=>1}, time)
98
+ d.emit({"a"=>2}, time)
99
+
100
+ d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n]
101
+ d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
102
+
103
+ d.run
104
+ end
105
+
106
+ def test_format_included_tag_and_time
107
+ config = [CONFIG, 'include_tag_key true', 'include_time_key true'].join("\n")
108
+ d = create_driver(config)
109
+
110
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
111
+ d.emit({"a"=>1}, time)
112
+ d.emit({"a"=>2}, time)
113
+
114
+ d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":1,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
115
+ d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":2,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
116
+
117
+ d.run
118
+ end
119
+
120
+ def test_format_with_format_ltsv
121
+ config = [CONFIG, 'format ltsv'].join("\n")
122
+ d = create_driver(config)
123
+
124
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
125
+ d.emit({"a"=>1, "b"=>1}, time)
126
+ d.emit({"a"=>2, "b"=>2}, time)
127
+
128
+ d.expect_format %[a:1\tb:1\n]
129
+ d.expect_format %[a:2\tb:2\n]
130
+
131
+ d.run
132
+ end
133
+
134
+ def test_format_with_format_json
135
+ config = [CONFIG, 'format json'].join("\n")
136
+ d = create_driver(config)
137
+
138
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
139
+ d.emit({"a"=>1}, time)
140
+ d.emit({"a"=>2}, time)
141
+
142
+ d.expect_format %[{"a":1}\n]
143
+ d.expect_format %[{"a":2}\n]
144
+
145
+ d.run
146
+ end
147
+
148
+ def test_format_with_format_json_included_tag
149
+ config = [CONFIG, 'format json', 'include_tag_key true'].join("\n")
150
+ d = create_driver(config)
151
+
152
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
153
+ d.emit({"a"=>1}, time)
154
+ d.emit({"a"=>2}, time)
155
+
156
+ d.expect_format %[{"a":1,"tag":"test"}\n]
157
+ d.expect_format %[{"a":2,"tag":"test"}\n]
158
+
159
+ d.run
160
+ end
161
+
162
+ def test_format_with_format_json_included_time
163
+ config = [CONFIG, 'format json', 'include_time_key true'].join("\n")
164
+ d = create_driver(config)
165
+
166
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
167
+ d.emit({"a"=>1}, time)
168
+ d.emit({"a"=>2}, time)
169
+
170
+ d.expect_format %[{"a":1,"time":"2011-01-02T13:14:15Z"}\n]
171
+ d.expect_format %[{"a":2,"time":"2011-01-02T13:14:15Z"}\n]
172
+
173
+ d.run
174
+ end
175
+
176
+ def test_format_with_format_json_included_tag_and_time
177
+ config = [CONFIG, 'format json', 'include_tag_key true', 'include_time_key true'].join("\n")
178
+ d = create_driver(config)
179
+
180
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
181
+ d.emit({"a"=>1}, time)
182
+ d.emit({"a"=>2}, time)
183
+
184
+ d.expect_format %[{"a":1,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
185
+ d.expect_format %[{"a":2,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
186
+
187
+ d.run
188
+ end
189
+
190
+ def test_chunk_to_write
191
+ d = create_driver
192
+
193
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
194
+ d.emit({"a"=>1}, time)
195
+ d.emit({"a"=>2}, time)
196
+
197
+ # AzureStorageOutputTest#write returns chunk.read
198
+ data = d.run
199
+
200
+ assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
201
+ %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
202
+ data
203
+ end
204
+
205
+ CONFIG_TIME_SLICE = %[
206
+ hostname testing.node.local
207
+ azure_storage_account test_storage_account
208
+ azure_storage_access_key dGVzdF9zdG9yYWdlX2FjY2Vzc19rZXk=
209
+ azure_container test_container
210
+ azure_object_key_format %{path}/events/ts=%{time_slice}/events_%{index}-%{hostname}.%{file_extension}
211
+ time_slice_format %Y%m%d-%H
212
+ path log
213
+ utc
214
+ buffer_type memory
215
+ log_level debug
216
+ ]
217
+
218
+ def create_time_sliced_driver(conf = CONFIG_TIME_SLICE)
219
+ d = Fluent::Test::TimeSlicedOutputTestDriver.new(Fluent::AzureStorageOutput) do
220
+ end.configure(conf)
221
+ d
222
+ end
223
+
224
+ end
metadata CHANGED
@@ -1,43 +1,15 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-azurestorage
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.5
4
+ version: 0.0.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - Hidemasa Togashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-09-11 00:00:00.000000000 Z
11
+ date: 2015-10-09 00:00:00.000000000 Z
12
12
  dependencies:
13
- - !ruby/object:Gem::Dependency
14
- name: bundler
15
- requirement: !ruby/object:Gem::Requirement
16
- requirements:
17
- - - ~>
18
- - !ruby/object:Gem::Version
19
- version: '1.3'
20
- type: :development
21
- prerelease: false
22
- version_requirements: !ruby/object:Gem::Requirement
23
- requirements:
24
- - - ~>
25
- - !ruby/object:Gem::Version
26
- version: '1.3'
27
- - !ruby/object:Gem::Dependency
28
- name: rake
29
- requirement: !ruby/object:Gem::Requirement
30
- requirements:
31
- - - '>='
32
- - !ruby/object:Gem::Version
33
- version: '0'
34
- type: :development
35
- prerelease: false
36
- version_requirements: !ruby/object:Gem::Requirement
37
- requirements:
38
- - - '>='
39
- - !ruby/object:Gem::Version
40
- version: '0'
41
13
  - !ruby/object:Gem::Dependency
42
14
  name: fluentd
43
15
  requirement: !ruby/object:Gem::Requirement
@@ -58,35 +30,49 @@ dependencies:
58
30
  - - <
59
31
  - !ruby/object:Gem::Version
60
32
  version: '2'
33
+ - !ruby/object:Gem::Dependency
34
+ name: azure
35
+ requirement: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - '='
38
+ - !ruby/object:Gem::Version
39
+ version: 0.6.4
40
+ type: :runtime
41
+ prerelease: false
42
+ version_requirements: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - '='
45
+ - !ruby/object:Gem::Version
46
+ version: 0.6.4
61
47
  - !ruby/object:Gem::Dependency
62
48
  name: fluent-mixin-config-placeholders
63
49
  requirement: !ruby/object:Gem::Requirement
64
50
  requirements:
65
51
  - - '>='
66
52
  - !ruby/object:Gem::Version
67
- version: '0'
53
+ version: 0.3.0
68
54
  type: :runtime
69
55
  prerelease: false
70
56
  version_requirements: !ruby/object:Gem::Requirement
71
57
  requirements:
72
58
  - - '>='
73
59
  - !ruby/object:Gem::Version
74
- version: '0'
60
+ version: 0.3.0
75
61
  - !ruby/object:Gem::Dependency
76
- name: azure
62
+ name: rake
77
63
  requirement: !ruby/object:Gem::Requirement
78
64
  requirements:
79
- - - '='
65
+ - - '>='
80
66
  - !ruby/object:Gem::Version
81
- version: 0.6.4
82
- type: :runtime
67
+ version: 0.9.2
68
+ type: :development
83
69
  prerelease: false
84
70
  version_requirements: !ruby/object:Gem::Requirement
85
71
  requirements:
86
- - - '='
72
+ - - '>='
87
73
  - !ruby/object:Gem::Version
88
- version: 0.6.4
89
- description: Fluent plugin for store to Azure Storage
74
+ version: 0.9.2
75
+ description: Azure Storage output plugin for Fluentd event collector
90
76
  email:
91
77
  - togachiro@gmail.com
92
78
  executables: []
@@ -94,15 +80,21 @@ extensions: []
94
80
  extra_rdoc_files: []
95
81
  files:
96
82
  - .gitignore
83
+ - .travis.yml
97
84
  - Gemfile
98
85
  - LICENSE.txt
99
86
  - README.md
100
87
  - Rakefile
88
+ - VERSION
101
89
  - fluent-plugin-azurestorage.gemspec
90
+ - lib/fluent/plugin/azurestorage_compressor_gzip_command.rb
91
+ - lib/fluent/plugin/azurestorage_compressor_lzma2.rb
92
+ - lib/fluent/plugin/azurestorage_compressor_lzo.rb
102
93
  - lib/fluent/plugin/out_azurestorage.rb
103
- homepage: ''
94
+ - test/test_out_azurestorage.rb
95
+ homepage: https://github.com/htgc/fluent-plugin-azurestorage
104
96
  licenses:
105
- - Apache License, version 2.
97
+ - Apache-2.0
106
98
  metadata: {}
107
99
  post_install_message:
108
100
  rdoc_options: []
@@ -123,5 +115,6 @@ rubyforge_project:
123
115
  rubygems_version: 2.0.14
124
116
  signing_key:
125
117
  specification_version: 4
126
- summary: Fluent plugin for store to Azure Storage
127
- test_files: []
118
+ summary: Azure Storage output plugin for Fluentd event collector
119
+ test_files:
120
+ - test/test_out_azurestorage.rb