fluent-plugin-azurestorage 0.0.5 → 0.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: b95e30f563028d1a96e4c6a5894c77e26b35c989
4
- data.tar.gz: 2a99b05c82539e6e19250dcfc63ba91b2664522c
3
+ metadata.gz: 8bb81cd14c71e59c606798aacf0fcbf3816d65c8
4
+ data.tar.gz: b520ba98690443f89e9f683248ba321675de0ffa
5
5
  SHA512:
6
- metadata.gz: 287c7183bbddb01a3ee82c991362c1ec2468e724fd685f297731a70a74857f070746c6a04fead67128bd8deae9a15e3e5ef2c98a6e582d172de76b75a37cd7fa
7
- data.tar.gz: 0212f5a6a4758bfa01c16d2ff8c0c41e2d0ea0b4e1d2044cc24de98e13f2c71620df10b41b9f7235c986e8b3e2e7ca478dbe695ee58cfa9e9c58557952590b72
6
+ metadata.gz: 98704f9ebff04476910ff2292e141fd8bf11f6831a27d429cdcecf16a8a2053bee071dbb2c8310b7ed600a42d71d7ddf52111b019b4cf968ef6704eda3de0113
7
+ data.tar.gz: 06eac631498041b2c523b926587afc49dd90d13c39ba92a527beb8e04ea9369449885ab2dc0a055282af090491c7b5ca3fe8a016bc2569a083628d99e815ae39
data/.gitignore CHANGED
@@ -1,17 +1,11 @@
1
+ ~*
2
+ #*
3
+ *~
4
+ [._]*.s[a-w][a-z]
5
+ .DS_Store
6
+
1
7
  *.gem
2
- *.rbc
3
8
  .bundle
4
- .config
5
- .yardoc
6
9
  Gemfile.lock
7
- InstalledFiles
8
- _yardoc
9
- coverage
10
- doc/
11
- lib/bundler/man
12
- pkg
13
- rdoc
14
- spec/reports
15
- test/tmp
16
- test/version_tmp
17
- tmp
10
+ vendor
11
+ .ruby-version
data/.travis.yml ADDED
@@ -0,0 +1,21 @@
1
+ language: ruby
2
+
3
+ rvm:
4
+ - 1.9.3
5
+ - 2.0
6
+ - 2.1
7
+ - 2.2.0
8
+ - rbx
9
+
10
+ gemfile:
11
+ - Gemfile
12
+
13
+ branches:
14
+ only:
15
+ - master
16
+
17
+ script: bundle exec rake test
18
+
19
+ matrix:
20
+ allow_failures:
21
+ - rvm: rbx
data/Gemfile CHANGED
@@ -1,4 +1,3 @@
1
- source 'https://rubygems.org'
1
+ source "http://rubygems.org"
2
2
 
3
- # Specify your gem's dependencies in fluent-plugin-azurestorage.gemspec
4
3
  gemspec
data/Rakefile CHANGED
@@ -1 +1,14 @@
1
- require "bundler/gem_tasks"
1
+
2
+ require 'bundler'
3
+ Bundler::GemHelper.install_tasks
4
+
5
+ require 'rake/testtask'
6
+
7
+ Rake::TestTask.new(:test) do |test|
8
+ test.libs << 'lib' << 'test'
9
+ test.test_files = FileList['test/test_*.rb']
10
+ test.verbose = true
11
+ end
12
+
13
+ task :default => [:build]
14
+
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 0.0.6
@@ -1,26 +1,24 @@
1
- # coding: utf-8
2
- #lib = File.expand_path('../lib', __FILE__)
3
- #$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
1
+ # encoding: utf-8
2
+ $:.push File.expand_path('../lib', __FILE__)
4
3
 
5
- Gem::Specification.new do |spec|
6
- spec.name = "fluent-plugin-azurestorage"
7
- spec.version = "0.0.5"
8
- spec.authors = ["Hidemasa Togashi"]
9
- spec.email = ["togachiro@gmail.com"]
10
- spec.description = %q{Fluent plugin for store to Azure Storage}
11
- spec.summary = %q{Fluent plugin for store to Azure Storage}
12
- spec.homepage = ""
13
- spec.license = "Apache License, version 2."
4
+ Gem::Specification.new do |gem|
5
+ gem.name = "fluent-plugin-azurestorage"
6
+ gem.description = "Azure Storage output plugin for Fluentd event collector"
7
+ gem.license = "Apache-2.0"
8
+ gem.homepage = "https://github.com/htgc/fluent-plugin-azurestorage"
9
+ gem.summary = gem.description
10
+ gem.version = File.read("VERSION").strip
11
+ gem.authors = ["Hidemasa Togashi"]
12
+ gem.email = ["togachiro@gmail.com"]
13
+ gem.has_rdoc = false
14
+ #gem.platform = Gem::Platform::RUBY
15
+ gem.files = `git ls-files`.split("\n")
16
+ gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
17
+ gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
18
+ gem.require_paths = ['lib']
14
19
 
15
- spec.files = `git ls-files -z`.split("\x0")
16
- spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
17
- spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
18
- spec.require_paths = ["lib"]
19
-
20
- spec.add_development_dependency "bundler", "~> 1.3"
21
- spec.add_development_dependency "rake"
22
-
23
- spec.add_runtime_dependency "fluentd", [">= 0.10.58", "< 2"]
24
- spec.add_runtime_dependency "fluent-mixin-config-placeholders"
25
- spec.add_runtime_dependency "azure", "0.6.4"
20
+ gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
21
+ gem.add_dependency "azure", "0.6.4"
22
+ gem.add_dependency "fluent-mixin-config-placeholders", ">= 0.3.0"
23
+ gem.add_development_dependency "rake", ">= 0.9.2"
26
24
  end
@@ -0,0 +1,51 @@
1
+ module Fluent
2
+ class AzureStorageOutput
3
+ class GzipCommandCompressor < Compressor
4
+ AzureStorageOutput.register_compressor('gzip_command', self)
5
+
6
+ config_param :command_parameter, :string, :default => ''
7
+
8
+ def configure(conf)
9
+ super
10
+ check_command('gzip')
11
+ end
12
+
13
+ def ext
14
+ 'gz'.freeze
15
+ end
16
+
17
+ def content_type
18
+ 'application/x-gzip'.freeze
19
+ end
20
+
21
+ def compress(chunk, tmp)
22
+ chunk_is_file = @buffer_type == 'file'
23
+ path = if chunk_is_file
24
+ chunk.path
25
+ else
26
+ w = Tempfile.new("chunk-gzip-tmp")
27
+ chunk.write_to(w)
28
+ w.close
29
+ w.path
30
+ end
31
+
32
+ res = system "gzip #{@command_parameter} -c #{path} > #{tmp.path}"
33
+ unless res
34
+ log.warn "failed to execute gzip command. Fallback to GzipWriter. status = #{$?}"
35
+ begin
36
+ tmp.truncate(0)
37
+ gw = Zlib::GzipWriter.new(tmp)
38
+ chunk.write_to(gw)
39
+ gw.close
40
+ ensure
41
+ gw.close rescue nil
42
+ end
43
+ end
44
+ ensure
45
+ unless chunk_is_file
46
+ w.close(true) rescue nil
47
+ end
48
+ end
49
+ end
50
+ end
51
+ end
@@ -0,0 +1,34 @@
1
+ module Fluent
2
+ class AzureStorageOutput
3
+ class LZMA2Compressor < Compressor
4
+ AzureStorageOutput.register_compressor('lzma2', self)
5
+
6
+ config_param :command_parameter, :string, :default => '-qf0'
7
+
8
+ def configure(conf)
9
+ super
10
+ check_command('xz', 'LZMA2')
11
+ end
12
+
13
+ def ext
14
+ 'xz'.freeze
15
+ end
16
+
17
+ def content_type
18
+ 'application/x-xz'.freeze
19
+ end
20
+
21
+ def compress(chunk, tmp)
22
+ w = Tempfile.new("chunk-xz-tmp")
23
+ chunk.write_to(w)
24
+ w.close
25
+
26
+ # We don't check the return code because we can't recover lzop failure.
27
+ system "xz #{@command_parameter} -c #{w.path} > #{tmp.path}"
28
+ ensure
29
+ w.close rescue nil
30
+ w.unlink rescue nil
31
+ end
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,34 @@
1
+ module Fluent
2
+ class AzureStorageOutput
3
+ class LZOCompressor < Compressor
4
+ AzureStorageOutput.register_compressor('lzo', self)
5
+
6
+ config_param :command_parameter, :string, :default => '-qf1'
7
+
8
+ def configure(conf)
9
+ super
10
+ check_command('lzop', 'LZO')
11
+ end
12
+
13
+ def ext
14
+ 'lzo'.freeze
15
+ end
16
+
17
+ def content_type
18
+ 'application/x-lzop'.freeze
19
+ end
20
+
21
+ def compress(chunk, tmp)
22
+ w = Tempfile.new("chunk-tmp")
23
+ chunk.write_to(w)
24
+ w.close
25
+
26
+ # We don't check the return code because we can't recover lzop failure.
27
+ system "lzop #{@command_parameter} -o #{tmp.path} #{w.path}"
28
+ ensure
29
+ w.close rescue nil
30
+ w.unlink rescue nil
31
+ end
32
+ end
33
+ end
34
+ end
@@ -10,20 +10,23 @@ module Fluent
10
10
  require 'zlib'
11
11
  require 'time'
12
12
  require 'tempfile'
13
- require 'open3'
13
+
14
+ @compressor = nil
14
15
  end
15
16
 
16
- config_param :path, :string, :default => ''
17
+ config_param :path, :string, :default => ""
17
18
  config_param :azure_storage_account, :string, :default => nil
18
19
  config_param :azure_storage_access_key, :string, :default => nil, :secret => true
19
20
  config_param :azure_container, :string, :default => nil
20
- config_param :azure_storage_type, :string, :default => 'blob'
21
+ config_param :azure_storage_type, :string, :default => "blob"
21
22
  config_param :azure_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
22
- config_param :store_as, :string, :default => 'gzip'
23
+ config_param :store_as, :string, :default => "gzip"
23
24
  config_param :auto_create_container, :bool, :default => true
24
- config_param :format, :string, :default => 'out_file'
25
+ config_param :format, :string, :default => "out_file"
25
26
  config_param :command_parameter, :string, :default => nil
26
27
 
28
+ attr_reader :bs
29
+
27
30
  include Fluent::Mixin::ConfigPlaceholders
28
31
 
29
32
  def placeholders
@@ -33,31 +36,13 @@ module Fluent
33
36
  def configure(conf)
34
37
  super
35
38
 
36
- @ext = case @store_as
37
- when 'gzip'
38
- 'gz'
39
- when 'lzo'
40
- check_command('lzop', 'LZO')
41
- @command_parameter = '-qf1' if @command_parameter.nil?
42
- 'lzo'
43
- when 'lzma2'
44
- check_command('xz', 'LZMA2')
45
- @command_paramter = '-qf0' if @command_parameter.nil?
46
- 'xz'
47
- when 'json'
48
- 'json'
49
- else
50
- 'txt'
51
- end
52
-
53
- @storage_type = case @azure_storage_type
54
- when 'tables'
55
- raise NotImplementedError
56
- when 'queues'
57
- raise NotImplementedError
58
- else
59
- 'blob'
60
- end
39
+ begin
40
+ @compressor = COMPRESSOR_REGISTRY.lookup(@store_as).new(:buffer_type => @buffer_type, :log => log)
41
+ rescue => e
42
+ $log.warn "#{@store_as} not found. Use 'text' instead"
43
+ @compressor = TextCompressor.new
44
+ end
45
+ @compressor.configure(conf)
61
46
 
62
47
  @formatter = Plugin.new_formatter(@format)
63
48
  @formatter.configure(conf)
@@ -75,6 +60,15 @@ module Fluent
75
60
  if @azure_container.nil?
76
61
  raise ConfigError, 'azure_container is needed'
77
62
  end
63
+
64
+ @storage_type = case @azure_storage_type
65
+ when 'tables'
66
+ raise NotImplementedError
67
+ when 'queues'
68
+ raise NotImplementedError
69
+ else
70
+ 'blob'
71
+ end
78
72
  end
79
73
 
80
74
  def start
@@ -102,15 +96,15 @@ module Fluent
102
96
  begin
103
97
  path = @path_slicer.call(@path)
104
98
  values_for_object_key = {
105
- 'path' => path,
106
- 'time_slice' => chunk.key,
107
- 'file_extension' => @ext,
108
- 'index' => i
99
+ "path" => path,
100
+ "time_slice" => chunk.key,
101
+ "file_extension" => @compressor.ext,
102
+ "index" => i,
103
+ "uuid_flush" => uuid_random
109
104
  }
110
105
  storage_path = @azure_object_key_format.gsub(%r(%{[^}]+})) { |expr|
111
106
  values_for_object_key[expr[2...expr.size-1]]
112
107
  }
113
-
114
108
  if (i > 0) && (storage_path == previous_path)
115
109
  raise "duplicated path is generated. use %{index} in azure_object_key_format: path = #{storage_path}"
116
110
  end
@@ -118,36 +112,13 @@ module Fluent
118
112
  i += 1
119
113
  previous_path = storage_path
120
114
  end while blob_exists?(@azure_container, storage_path)
121
-
115
+
122
116
  tmp = Tempfile.new("azure-")
123
117
  begin
124
- case @store_as
125
- when 'gzip'
126
- w = Zlib::GzipWriter.new(tmp)
127
- chunk.write_to(w)
128
- w.close
129
- when 'lzo'
130
- w = Tempfile.new('chunk-tmp')
131
- chunk.write_to(w)
132
- w.close
133
- tmp.close
134
- system "lzop #{@command_parameter} -o #{tmp.path} #{w.path}"
135
- when 'lzma2'
136
- w = Tempfile.new('chunk-xz-tmp')
137
- chunk.write_to(w)
138
- w.close
139
- tmp.close
140
- system "xz #{@command_parameter} -c #{w.path} > #{tmp.path}"
141
- else
142
- chunk.write_to(tmp)
143
- tmp.close
144
- end
118
+ @compressor.compress(chunk, tmp)
119
+ tmp.close
145
120
  content = File.open(tmp.path, 'rb') { |file| file.read }
146
121
  @bs.create_block_blob(@azure_container, storage_path, content)
147
- ensure
148
- tmp.close(true) rescue nil
149
- w.close rescue nil
150
- w.unlink rescue nil
151
122
  end
152
123
  end
153
124
 
@@ -162,12 +133,97 @@ module Fluent
162
133
  end
163
134
  end
164
135
 
165
- def check_command(command, algo)
166
- begin
167
- Open3.capture3("#{command} -V")
168
- rescue Errno::ENOENT
169
- raise ConfigError, "'#{command}' utility must be in PATH for #{algo} compression"
136
+ class Compressor
137
+ include Configurable
138
+
139
+ def initialize(opts = {})
140
+ super()
141
+ @buffer_type = opts[:buffer_type]
142
+ @log = opts[:log]
170
143
  end
144
+
145
+ attr_reader :buffer_type, :log
146
+
147
+ def configure(conf)
148
+ super
149
+ end
150
+
151
+ def ext
152
+ end
153
+
154
+ def content_type
155
+ end
156
+
157
+ def compress(chunk, tmp)
158
+ end
159
+
160
+ private
161
+
162
+ def check_command(command, algo = nil)
163
+ require 'open3'
164
+
165
+ algo = command if algo.nil?
166
+ begin
167
+ Open3.capture3("#{command} -V")
168
+ rescue Errno::ENOENT
169
+ raise ConfigError, "'#{command}' utility must be in PATH for #{algo} compression"
170
+ end
171
+ end
172
+ end
173
+
174
+ class GzipCompressor < Compressor
175
+ def ext
176
+ 'gz'.freeze
177
+ end
178
+
179
+ def content_type
180
+ 'application/x-gzip'.freeze
181
+ end
182
+
183
+ def compress(chunk, tmp)
184
+ w = Zlib::GzipWriter.new(tmp)
185
+ chunk.write_to(w)
186
+ w.finish
187
+ ensure
188
+ w.finish rescue nil
189
+ end
190
+ end
191
+
192
+ class TextCompressor < Compressor
193
+ def ext
194
+ 'txt'.freeze
195
+ end
196
+
197
+ def content_type
198
+ 'text/plain'.freeze
199
+ end
200
+
201
+ def compress(chunk, tmp)
202
+ chunk.write_to(tmp)
203
+ end
204
+ end
205
+
206
+ class JsonCompressor < TextCompressor
207
+ def ext
208
+ 'json'.freeze
209
+ end
210
+
211
+ def content_type
212
+ 'application/json'.freeze
213
+ end
214
+ end
215
+
216
+ COMPRESSOR_REGISTRY = Registry.new(:azurestorage_compressor_type, 'fluent/plugin/azurestorage_compressor_')
217
+ {
218
+ 'gzip' => GzipCompressor,
219
+ 'json' => JsonCompressor,
220
+ 'text' => TextCompressor
221
+ }.each { |name, compressor|
222
+ COMPRESSOR_REGISTRY.register(name, compressor)
223
+ }
224
+
225
+ def self.register_compressor(name, compressor)
226
+ COMPRESSOR_REGISTRY.register(name, compressor)
171
227
  end
172
228
 
173
229
  def blob_exists?(container, blob)
@@ -0,0 +1,224 @@
1
+ require 'fluent/test'
2
+ require 'fluent/plugin/out_azurestorage'
3
+
4
+ require 'test/unit/rr'
5
+ require 'zlib'
6
+ require 'fileutils'
7
+
8
+ class AzureStorageOutputTest < Test::Unit::TestCase
9
+ def setup
10
+ require 'azure'
11
+ Fluent::Test.setup
12
+ end
13
+
14
+ CONFIG = %[
15
+ azure_storage_account test_storage_account
16
+ azure_storage_access_key dGVzdF9zdG9yYWdlX2FjY2Vzc19rZXk=
17
+ azure_container test_container
18
+ path log
19
+ utc
20
+ buffer_type memory
21
+ ]
22
+
23
+ def create_driver(conf = CONFIG)
24
+ Fluent::Test::BufferedOutputTestDriver.new(Fluent::AzureStorageOutput) do
25
+ def write(chunk)
26
+ chunk.read
27
+ end
28
+
29
+ private
30
+
31
+ def ensure_container
32
+ end
33
+
34
+ end.configure(conf)
35
+ end
36
+
37
+ def test_configure
38
+ d = create_driver
39
+ assert_equal 'test_storage_account', d.instance.azure_storage_account
40
+ assert_equal 'dGVzdF9zdG9yYWdlX2FjY2Vzc19rZXk=', d.instance.azure_storage_access_key
41
+ assert_equal 'test_container', d.instance.azure_container
42
+ assert_equal 'log', d.instance.path
43
+ assert_equal 'gz', d.instance.instance_variable_get(:@compressor).ext
44
+ assert_equal 'application/x-gzip', d.instance.instance_variable_get(:@compressor).content_type
45
+ end
46
+
47
+ def test_configure_with_mime_type_json
48
+ conf = CONFIG.clone
49
+ conf << "\nstore_as json\n"
50
+ d = create_driver(conf)
51
+ assert_equal 'json', d.instance.instance_variable_get(:@compressor).ext
52
+ assert_equal 'application/json', d.instance.instance_variable_get(:@compressor).content_type
53
+ end
54
+
55
+ def test_configure_with_mime_type_text
56
+ conf = CONFIG.clone
57
+ conf << "\nstore_as text\n"
58
+ d = create_driver(conf)
59
+ assert_equal 'txt', d.instance.instance_variable_get(:@compressor).ext
60
+ assert_equal 'text/plain', d.instance.instance_variable_get(:@compressor).content_type
61
+ end
62
+
63
+ def test_configure_with_mime_type_lzo
64
+ conf = CONFIG.clone
65
+ conf << "\nstore_as lzo\n"
66
+ d = create_driver(conf)
67
+ assert_equal 'lzo', d.instance.instance_variable_get(:@compressor).ext
68
+ assert_equal 'application/x-lzop', d.instance.instance_variable_get(:@compressor).content_type
69
+ rescue => e
70
+ # TODO: replace code with disable lzop command
71
+ assert(e.is_a?(Fluent::ConfigError))
72
+ end
73
+
74
+ def test_path_slicing
75
+ config = CONFIG.clone.gsub(/path\slog/, "path log/%Y/%m/%d")
76
+ d = create_driver(config)
77
+ path_slicer = d.instance.instance_variable_get(:@path_slicer)
78
+ path = d.instance.instance_variable_get(:@path)
79
+ slice = path_slicer.call(path)
80
+ assert_equal slice, Time.now.utc.strftime("log/%Y/%m/%d")
81
+ end
82
+
83
+ def test_path_slicing_utc
84
+ config = CONFIG.clone.gsub(/path\slog/, "path log/%Y/%m/%d")
85
+ config << "\nutc\n"
86
+ d = create_driver(config)
87
+ path_slicer = d.instance.instance_variable_get(:@path_slicer)
88
+ path = d.instance.instance_variable_get(:@path)
89
+ slice = path_slicer.call(path)
90
+ assert_equal slice, Time.now.utc.strftime("log/%Y/%m/%d")
91
+ end
92
+
93
+ def test_format
94
+ d = create_driver
95
+
96
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
97
+ d.emit({"a"=>1}, time)
98
+ d.emit({"a"=>2}, time)
99
+
100
+ d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n]
101
+ d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
102
+
103
+ d.run
104
+ end
105
+
106
+ def test_format_included_tag_and_time
107
+ config = [CONFIG, 'include_tag_key true', 'include_time_key true'].join("\n")
108
+ d = create_driver(config)
109
+
110
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
111
+ d.emit({"a"=>1}, time)
112
+ d.emit({"a"=>2}, time)
113
+
114
+ d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":1,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
115
+ d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":2,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
116
+
117
+ d.run
118
+ end
119
+
120
+ def test_format_with_format_ltsv
121
+ config = [CONFIG, 'format ltsv'].join("\n")
122
+ d = create_driver(config)
123
+
124
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
125
+ d.emit({"a"=>1, "b"=>1}, time)
126
+ d.emit({"a"=>2, "b"=>2}, time)
127
+
128
+ d.expect_format %[a:1\tb:1\n]
129
+ d.expect_format %[a:2\tb:2\n]
130
+
131
+ d.run
132
+ end
133
+
134
+ def test_format_with_format_json
135
+ config = [CONFIG, 'format json'].join("\n")
136
+ d = create_driver(config)
137
+
138
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
139
+ d.emit({"a"=>1}, time)
140
+ d.emit({"a"=>2}, time)
141
+
142
+ d.expect_format %[{"a":1}\n]
143
+ d.expect_format %[{"a":2}\n]
144
+
145
+ d.run
146
+ end
147
+
148
+ def test_format_with_format_json_included_tag
149
+ config = [CONFIG, 'format json', 'include_tag_key true'].join("\n")
150
+ d = create_driver(config)
151
+
152
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
153
+ d.emit({"a"=>1}, time)
154
+ d.emit({"a"=>2}, time)
155
+
156
+ d.expect_format %[{"a":1,"tag":"test"}\n]
157
+ d.expect_format %[{"a":2,"tag":"test"}\n]
158
+
159
+ d.run
160
+ end
161
+
162
+ def test_format_with_format_json_included_time
163
+ config = [CONFIG, 'format json', 'include_time_key true'].join("\n")
164
+ d = create_driver(config)
165
+
166
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
167
+ d.emit({"a"=>1}, time)
168
+ d.emit({"a"=>2}, time)
169
+
170
+ d.expect_format %[{"a":1,"time":"2011-01-02T13:14:15Z"}\n]
171
+ d.expect_format %[{"a":2,"time":"2011-01-02T13:14:15Z"}\n]
172
+
173
+ d.run
174
+ end
175
+
176
+ def test_format_with_format_json_included_tag_and_time
177
+ config = [CONFIG, 'format json', 'include_tag_key true', 'include_time_key true'].join("\n")
178
+ d = create_driver(config)
179
+
180
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
181
+ d.emit({"a"=>1}, time)
182
+ d.emit({"a"=>2}, time)
183
+
184
+ d.expect_format %[{"a":1,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
185
+ d.expect_format %[{"a":2,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
186
+
187
+ d.run
188
+ end
189
+
190
+ def test_chunk_to_write
191
+ d = create_driver
192
+
193
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
194
+ d.emit({"a"=>1}, time)
195
+ d.emit({"a"=>2}, time)
196
+
197
+ # AzureStorageOutputTest#write returns chunk.read
198
+ data = d.run
199
+
200
+ assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
201
+ %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
202
+ data
203
+ end
204
+
205
+ CONFIG_TIME_SLICE = %[
206
+ hostname testing.node.local
207
+ azure_storage_account test_storage_account
208
+ azure_storage_access_key dGVzdF9zdG9yYWdlX2FjY2Vzc19rZXk=
209
+ azure_container test_container
210
+ azure_object_key_format %{path}/events/ts=%{time_slice}/events_%{index}-%{hostname}.%{file_extension}
211
+ time_slice_format %Y%m%d-%H
212
+ path log
213
+ utc
214
+ buffer_type memory
215
+ log_level debug
216
+ ]
217
+
218
+ def create_time_sliced_driver(conf = CONFIG_TIME_SLICE)
219
+ d = Fluent::Test::TimeSlicedOutputTestDriver.new(Fluent::AzureStorageOutput) do
220
+ end.configure(conf)
221
+ d
222
+ end
223
+
224
+ end
metadata CHANGED
@@ -1,43 +1,15 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-azurestorage
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.5
4
+ version: 0.0.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - Hidemasa Togashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-09-11 00:00:00.000000000 Z
11
+ date: 2015-10-09 00:00:00.000000000 Z
12
12
  dependencies:
13
- - !ruby/object:Gem::Dependency
14
- name: bundler
15
- requirement: !ruby/object:Gem::Requirement
16
- requirements:
17
- - - ~>
18
- - !ruby/object:Gem::Version
19
- version: '1.3'
20
- type: :development
21
- prerelease: false
22
- version_requirements: !ruby/object:Gem::Requirement
23
- requirements:
24
- - - ~>
25
- - !ruby/object:Gem::Version
26
- version: '1.3'
27
- - !ruby/object:Gem::Dependency
28
- name: rake
29
- requirement: !ruby/object:Gem::Requirement
30
- requirements:
31
- - - '>='
32
- - !ruby/object:Gem::Version
33
- version: '0'
34
- type: :development
35
- prerelease: false
36
- version_requirements: !ruby/object:Gem::Requirement
37
- requirements:
38
- - - '>='
39
- - !ruby/object:Gem::Version
40
- version: '0'
41
13
  - !ruby/object:Gem::Dependency
42
14
  name: fluentd
43
15
  requirement: !ruby/object:Gem::Requirement
@@ -58,35 +30,49 @@ dependencies:
58
30
  - - <
59
31
  - !ruby/object:Gem::Version
60
32
  version: '2'
33
+ - !ruby/object:Gem::Dependency
34
+ name: azure
35
+ requirement: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - '='
38
+ - !ruby/object:Gem::Version
39
+ version: 0.6.4
40
+ type: :runtime
41
+ prerelease: false
42
+ version_requirements: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - '='
45
+ - !ruby/object:Gem::Version
46
+ version: 0.6.4
61
47
  - !ruby/object:Gem::Dependency
62
48
  name: fluent-mixin-config-placeholders
63
49
  requirement: !ruby/object:Gem::Requirement
64
50
  requirements:
65
51
  - - '>='
66
52
  - !ruby/object:Gem::Version
67
- version: '0'
53
+ version: 0.3.0
68
54
  type: :runtime
69
55
  prerelease: false
70
56
  version_requirements: !ruby/object:Gem::Requirement
71
57
  requirements:
72
58
  - - '>='
73
59
  - !ruby/object:Gem::Version
74
- version: '0'
60
+ version: 0.3.0
75
61
  - !ruby/object:Gem::Dependency
76
- name: azure
62
+ name: rake
77
63
  requirement: !ruby/object:Gem::Requirement
78
64
  requirements:
79
- - - '='
65
+ - - '>='
80
66
  - !ruby/object:Gem::Version
81
- version: 0.6.4
82
- type: :runtime
67
+ version: 0.9.2
68
+ type: :development
83
69
  prerelease: false
84
70
  version_requirements: !ruby/object:Gem::Requirement
85
71
  requirements:
86
- - - '='
72
+ - - '>='
87
73
  - !ruby/object:Gem::Version
88
- version: 0.6.4
89
- description: Fluent plugin for store to Azure Storage
74
+ version: 0.9.2
75
+ description: Azure Storage output plugin for Fluentd event collector
90
76
  email:
91
77
  - togachiro@gmail.com
92
78
  executables: []
@@ -94,15 +80,21 @@ extensions: []
94
80
  extra_rdoc_files: []
95
81
  files:
96
82
  - .gitignore
83
+ - .travis.yml
97
84
  - Gemfile
98
85
  - LICENSE.txt
99
86
  - README.md
100
87
  - Rakefile
88
+ - VERSION
101
89
  - fluent-plugin-azurestorage.gemspec
90
+ - lib/fluent/plugin/azurestorage_compressor_gzip_command.rb
91
+ - lib/fluent/plugin/azurestorage_compressor_lzma2.rb
92
+ - lib/fluent/plugin/azurestorage_compressor_lzo.rb
102
93
  - lib/fluent/plugin/out_azurestorage.rb
103
- homepage: ''
94
+ - test/test_out_azurestorage.rb
95
+ homepage: https://github.com/htgc/fluent-plugin-azurestorage
104
96
  licenses:
105
- - Apache License, version 2.
97
+ - Apache-2.0
106
98
  metadata: {}
107
99
  post_install_message:
108
100
  rdoc_options: []
@@ -123,5 +115,6 @@ rubyforge_project:
123
115
  rubygems_version: 2.0.14
124
116
  signing_key:
125
117
  specification_version: 4
126
- summary: Fluent plugin for store to Azure Storage
127
- test_files: []
118
+ summary: Azure Storage output plugin for Fluentd event collector
119
+ test_files:
120
+ - test/test_out_azurestorage.rb