logstash-output-qingstor 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,26 @@
1
+ # encoding: utf-8
2
+ module LogStash
3
+ module Outputs
4
+ class Qingstor
5
+ class TimeRotationPolicy
6
+ attr_reader :time_file
7
+
8
+ def initialize(time_file)
9
+ if time_file <= 0
10
+ raise LogStash::ConfigurationError, "'time_file' need to be greater then 0"
11
+ end
12
+
13
+ @time_file = time_file * 60
14
+ end
15
+
16
+ def rotate?(file)
17
+ file.size > 0 && (Time.now - file.ctime) >= time_file
18
+ end
19
+
20
+ def needs_periodic?
21
+ true
22
+ end
23
+ end
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,64 @@
1
+ # encoding: utf-8
2
+ require "qingstor/sdk"
3
+ require "concurrent"
4
+ require "digest/md5"
5
+ require "base64"
6
+ module LogStash
7
+ module Outputs
8
+ class Qingstor
9
+ class Uploader
10
+ TIME_BEFORE_RETRYING_SECONDS = 1
11
+ DEFAULT_THREADPOOL = Concurrent::ThreadPoolExecutor.new({ :min_thread => 1,
12
+ :max_thread => 8,
13
+ :max_queue => 1,
14
+ :fallback_policy => :caller_runs
15
+ })
16
+ attr_reader :bucket, :upload_options, :logger
17
+
18
+ def initialize(bucket, logger, threadpool = DEFAULT_THREADPOOL)
19
+ @bucket = bucket
20
+ @logger = logger
21
+ @workers_pool = threadpool
22
+ end
23
+
24
+ def upload_async(file, options = {})
25
+ @workers_pool.post do
26
+ upload(file, options)
27
+ end
28
+ end
29
+
30
+ def upload(file, options = {})
31
+ upload_options = options.fetch(:upload_options, {})
32
+
33
+ file_md5 = Digest::MD5.file(file.path).to_s
34
+
35
+ upload_headers = {
36
+ "content_md5" => file_md5,
37
+ "body" => ::File.open(file.path)
38
+ }
39
+
40
+ if !upload_options[:server_side_encryption_algorithm].nil?
41
+ base64_key = Base64.strict_encode64 upload_options[:customer_key]
42
+ key_md5 = Digest::MD5.hexdigest upload_options[:customer_key]
43
+ base64_key_md5 = Base64.strict_encode64 key_md5
44
+ upload_headers.merge!({
45
+ "x_qs_encryption_customer_algorithm" => upload_options[:server_side_encryption_algorithm],
46
+ "x_qs_encryption_customer_key" => base64_key,
47
+ "x_qs_encryption_customer_key_md5" => base64_key_md5,
48
+ })
49
+ end
50
+ @logger.debug("uploading file", :file => file.key)
51
+ bucket.put_object(file.key, upload_headers)
52
+
53
+ options[:on_complete].call(file) unless options[:on_complete].nil?
54
+ end
55
+
56
+ def stop
57
+ @workers_pool.shutdown
58
+ @workers_pool.wait_for_termination(nil)
59
+ end
60
+ end
61
+ end
62
+ end
63
+ end
64
+
@@ -0,0 +1,253 @@
1
+ # encoding: utf-8
2
+ require "logstash-core"
3
+ require "logstash/outputs/base"
4
+ require "logstash/namespace"
5
+ require "tmpdir"
6
+ require "qingstor/sdk"
7
+ require "concurrent"
8
+
9
+ class LogStash::Outputs::Qingstor < LogStash::Outputs::Base
10
+ require "logstash/outputs/qingstor/temporary_file"
11
+ require "logstash/outputs/qingstor/temporary_file_factory"
12
+ require "logstash/outputs/qingstor/file_repository"
13
+ require "logstash/outputs/qingstor/size_rotation_policy"
14
+ require "logstash/outputs/qingstor/time_rotation_policy"
15
+ require "logstash/outputs/qingstor/size_and_time_rotation_policy"
16
+ require "logstash/outputs/qingstor/uploader"
17
+ require "logstash/outputs/qingstor/qingstor_validator"
18
+
19
+ PERIODIC_CHECK_INTERVAL_IN_SECONDS = 15
20
+ CRASH_RECOVERY_THREADPOOL = Concurrent::ThreadPoolExecutor.new({
21
+ :min_threads => 1,
22
+ :max_threads => 2,
23
+ :fallback_policy => :caller_runs
24
+ })
25
+ config_name "qingstor"
26
+
27
+ # When configured as :single a single instance of the Output will be shared among the
28
+ # pipeline worker threads. Access to the `#multi_receive/#multi_receive_encoded/#receive` method will be synchronized
29
+ # i.e. only one thread will be active at a time making threadsafety much simpler.
30
+ #
31
+ # You can set this to :shared if your output is threadsafe. This will maximize
32
+ # concurrency but you will need to make appropriate uses of mutexes in `#multi_receive/#receive`.
33
+ #
34
+ # Only the `#multi_receive/#multi_receive_encoded` methods need to actually be threadsafe, the other methods
35
+ # will only be executed in a single thread
36
+ concurrency :shared
37
+
38
+ # The key id to access your QingStor
39
+ config :access_key_id, :validate => :string, :required => true
40
+
41
+ # The key to access your QingStor
42
+ config :secret_access_key, :validate => :string, :required => true
43
+
44
+ # The name of the qingstor bucket
45
+ config :bucket, :validate => :string, :required => true
46
+
47
+ # The region of the QingStor bucket
48
+ config :region, :validate => ["pek3a", "sh1a"], :default => "pek3a"
49
+
50
+ # The prefix of filenames which will work as directory in qingstor
51
+ config :prefix, :validate => :string, :default => ''
52
+
53
+ # Set the directory where logstash store the tmp files before
54
+ # sending it to qingstor, default directory in linux /tmp/logstash2qingstor
55
+ config :tmpdir, :validate => :string, :default => File.join(Dir.tmpdir, "logstash2qingstor")
56
+
57
+ # Define tags to append to the file on the qingstor bucket
58
+ config :tags, :validate => :array, :default => []
59
+
60
+ # Specify the content encoding. Supports ("gzip"), defaults to "none"
61
+ config :encoding, :validate => ["gzip", "none"], :default => "none"
62
+
63
+ # Define the strategy to use to decide when we need to rotate the file and push it to S3,
64
+ # The default strategy is to check for both size and time, the first one to match will rotate the file.
65
+ config :rotation_strategy, :validate => ["size_and_time", "size", "time"], :default => "size_and_time"
66
+
67
+ # Define the size requirement for each file to upload to qingstor. In byte.
68
+ config :size_file, :validate => :number, :default => 1024 * 1024 * 5
69
+
70
+ # Define the time interval for each file to upload to qingstor. In minutes.
71
+ config :time_file, :validate => :number, :default => 15
72
+
73
+ # Specify maximum number of workers to use to upload the files to Qingstor
74
+ config :upload_workers_count, :validate => :number, :default => (Concurrent.processor_count * 0.5).ceil
75
+
76
+ # Number of items we can keep in the local queue before uploading them
77
+ config :upload_queue_size, :validate => :number, :default => 2 * (Concurrent.processor_count * 0.25).ceil
78
+
79
+ # Specifies what type of encryption to use when SSE is enabled.
80
+ config :server_side_encryption_algorithm, :validate => ["AES256", "none"], :default => "none"
81
+
82
+ # Specifies the encryption customer key that would be used in server side
83
+ config :customer_key, :validate => :string
84
+
85
+ # Specifies if set to true, it would upload existing file in targeting folder at the beginning.
86
+ config :restore, :validate => :boolean, :default => false
87
+
88
+ public
89
+ def register
90
+ QingstorValidator.prefix_valid?(@prefix) unless @prefix.nil?
91
+
92
+ if !directory_valid?(@tmpdir)
93
+ raise LogStash::ConfigurationError, "Logstash must have the permissions to write to the temporary directory: #{@tmpdir}"
94
+ end
95
+
96
+ @file_repository = FileRepository.new(@tags, @encoding, @tmpdir)
97
+
98
+ @rotation = rotation_strategy
99
+
100
+ executor = Concurrent::ThreadPoolExecutor.new({
101
+ :min_threads => 1,
102
+ :max_threads => @upload_workers_count,
103
+ :max_queue => @upload_queue_size,
104
+ :fallback_policy => :caller_runs
105
+ })
106
+
107
+ @qs_bucket = get_bucket
108
+ QingstorValidator.bucket_valid?(@qs_bucket)
109
+
110
+ @uploader = Uploader.new(@qs_bucket, @logger, executor)
111
+
112
+ start_periodic_check if @rotation.needs_periodic?
113
+
114
+ restore_from_crash if @restore
115
+ end # def register
116
+
117
+ public
118
+ def multi_receive_encoded(events_and_encoded)
119
+ prefix_written_to = Set.new
120
+
121
+ events_and_encoded.each do |event, encoded|
122
+ prefix_key = event.sprintf(@prefix)
123
+ prefix_written_to << prefix_key
124
+
125
+ begin
126
+ @file_repository.get_file(prefix_key) { |file| file.write(encoded) }
127
+ rescue Errno::ENOSPC => e
128
+ @logger.error("QingStor: Nospace left in temporary directory", :tmpdir => @tmpdir)
129
+ raise e
130
+ end
131
+ end # end of each method
132
+
133
+ # check the file after file writing
134
+ # Groups IO calls to optimize fstat checks
135
+ rotate_if_needed(prefix_written_to)
136
+ end # def multi_receive_encoded
137
+
138
+ def rotation_strategy
139
+ case @rotation_strategy
140
+ when "size"
141
+ SizeRotationPolicy.new(@size_file)
142
+ when "time"
143
+ TimeRotationPolicy.new(@time_file)
144
+ when "size_and_time"
145
+ SizeAndTimeRotationPolicy.new(@size_file, @time_file)
146
+ end
147
+ end
148
+
149
+ def rotate_if_needed(prefixs)
150
+ prefixs.each do |prefix|
151
+ @file_repository.get_factory(prefix) do |factory|
152
+ tmp_file = factory.current
153
+
154
+ if @rotation.rotate?(tmp_file)
155
+ @logger.debug("Rotate file",
156
+ :strategy => tmp_file.key,
157
+ :path => tmp_file.path)
158
+ upload_file(tmp_file)
159
+ factory.rotate!
160
+ end
161
+ end
162
+ end
163
+ end # def rotate_if_needed
164
+
165
+ def upload_file(file)
166
+ @logger.debug("Add file to uploading queue", :key => file.key)
167
+ file.close
168
+ @logger.debug("upload options", :upload_options => upload_options)
169
+ @uploader.upload_async(file,
170
+ :on_complete => method(:clean_temporary_file),
171
+ :upload_options => upload_options)
172
+ end
173
+
174
+ def get_bucket
175
+ @qs_config = QingStor::SDK::Config.init @access_key_id, @secret_access_key
176
+ @qs_service = QingStor::SDK::Service.new @qs_config
177
+ @qs_service.bucket @bucket, @region
178
+ end
179
+
180
+ def close
181
+ stop_periodic_check if @rotation.needs_periodic?
182
+
183
+ @logger.debug("uploading current workspace")
184
+ @file_repository.each_files do |file|
185
+ upload_file(file)
186
+ end
187
+
188
+ @file_repository.shutdown
189
+
190
+ @uploader.stop
191
+
192
+ @crash_uploader.stop if @restore
193
+ end
194
+
195
+ def upload_options
196
+ options = {
197
+ :content_encoding => @encoding == "gzip" ? "gzip" : nil
198
+ }
199
+
200
+ if @server_side_encryption_algorithm == "AES256" && !@customer_key.nil?
201
+ options.merge!({
202
+ :server_side_encryption_algorithm => @server_side_encryption_algorithm,
203
+ :customer_key => @customer_key
204
+ })
205
+ end
206
+
207
+ options
208
+ end
209
+
210
+ def clean_temporary_file(file)
211
+ @logger.debug("Removing temporary file", :file => file.path)
212
+ file.delete!
213
+ end
214
+
215
+ def start_periodic_check
216
+ @logger.debug("Start periodic rotation check")
217
+
218
+ @periodic_check = Concurrent::TimerTask.new(:execution_interval => PERIODIC_CHECK_INTERVAL_IN_SECONDS) do
219
+ @logger.debug("Periodic check for stale files")
220
+
221
+ rotate_if_needed(@file_repository.keys)
222
+ end
223
+
224
+ @periodic_check.execute
225
+ end
226
+
227
+ def stop_periodic_check
228
+ @periodic_check.shutdown
229
+ end
230
+
231
+ def directory_valid?(path)
232
+ begin
233
+ FileUtils.mkdir_p(path) unless Dir.exist?(path)
234
+ ::File.writable?(path)
235
+ rescue
236
+ false
237
+ end
238
+ end
239
+
240
+ def restore_from_crash
241
+ @crash_uploader = Uploader.new(@qs_bucket, @logger, CRASH_RECOVERY_THREADPOOL)
242
+
243
+ temp_folder_path = Pathname.new(@tmpdir)
244
+ Dir.glob(::File.join(@tmpdir, "**/*"))
245
+ .select { |file| ::File.file?(file) }
246
+ .each do |file|
247
+ temp_file = TemporaryFile.create_from_existing_file(file, temp_folder_path)
248
+ @logger.debug("Recoving from crash and uploading", :file => temp_file.path)
249
+ @crash_uploader.upload_async(temp_file, :on_complete => method(:clean_temporary_file),
250
+ :upload_options => upload_options)
251
+ end
252
+ end
253
+ end # class LogStash::Outputs::Qingstor
@@ -0,0 +1,28 @@
1
+ Gem::Specification.new do |s|
2
+ s.name = 'logstash-output-qingstor'
3
+ s.version = '0.1.0'
4
+ s.licenses = ['Apache License (2.0)']
5
+ s.summary = 'logstash output plugin for qingstor'
6
+ s.description = 'Put the outcomes of logstash into qingstor'
7
+ s.homepage = 'https://github.com/tacinight/logstash-output-qingstor'
8
+ s.authors = ['Evan Zhao']
9
+ s.email = 'tacingiht@gmail.com'
10
+ s.require_paths = ['lib']
11
+
12
+ # Files
13
+ s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
14
+ # Tests
15
+ s.test_files = s.files.grep(%r{^(test|spec|features)/})
16
+
17
+ # Special flag to let us know this is actually a logstash plugin
18
+ s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
19
+
20
+ # Gem dependencies
21
+ s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
22
+ s.add_runtime_dependency "logstash-codec-plain"
23
+ s.add_runtime_dependency "qingstor-sdk", "=1.9.2"
24
+ s.add_runtime_dependency "concurrent-ruby"
25
+
26
+ s.add_development_dependency "stud", "~> 0.0.22"
27
+ s.add_development_dependency "logstash-devutils"
28
+ end
@@ -0,0 +1,26 @@
1
+ require "logstash/devutils/rspec/spec_helper"
2
+ require "logstash/outputs/qingstor/temporary_file"
3
+ require "logstash/outputs/qingstor/temporary_file_factory"
4
+ require "logstash/outputs/qingstor/file_repository"
5
+ require "tmpdir"
6
+
7
+ describe LogStash::Outputs::Qingstor::FileRepository do
8
+ let(:tags) { ["tag1", "tag2", "tag3"]}
9
+ let(:encoding) { "none" }
10
+ let(:tmpdir) { File.join(Dir.tmpdir, "lg2qs") }
11
+ let(:prefix) { "aprefix" }
12
+
13
+ subject { described_class.new(tags, encoding, tmpdir) }
14
+
15
+ it "can get current file io" do
16
+ subject.get_file(prefix) do |file|
17
+ expect(file).to be_kind_of(LogStash::Outputs::Qingstor::TemporaryFile)
18
+ end
19
+ end
20
+
21
+ it "can get current file factory" do
22
+ subject.get_factory(prefix) do |factory|
23
+ expect(factory).to be_kind_of(LogStash::Outputs::Qingstor::TemporaryFileFactory)
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,21 @@
1
+ # encoding: utf-8
2
+ require "logstash/devutils/rspec/spec_helper"
3
+ require "logstash/outputs/qingstor/qingstor_validator"
4
+ require_relative "../qs_access_helper"
5
+
6
+ describe LogStash::Outputs::Qingstor::QingstorValidator do
7
+ let(:normal_prefix) { "super/bucket" }
8
+ let(:wrong_prefix1) { "/wrong/prefix" }
9
+ let(:wrong_prefix2) { normal_prefix * 100 }
10
+ let(:bucket) { qs_bucket_init }
11
+
12
+ it "raise error if the prefix is not valid" do
13
+ expect{ described_class.prefix_valid?(wrong_prefix1) }.to raise_error(LogStash::ConfigurationError)
14
+ expect{ described_class.prefix_valid?(wrong_prefix2) }.to raise_error(LogStash::ConfigurationError)
15
+ end
16
+
17
+ it "return true if the prefix is valid" do
18
+ expect(described_class.prefix_valid?(normal_prefix)).to be_truthy
19
+ end
20
+
21
+ end
@@ -0,0 +1,39 @@
1
+ # encoding: utf-8
2
+ require "logstash/devutils/rspec/spec_helper"
3
+ require "logstash/outputs/qingstor/temporary_file"
4
+ require "logstash/outputs/qingstor/size_and_time_rotation_policy"
5
+
6
+ describe LogStash::Outputs::Qingstor::SizeAndTimeRotationPolicy do
7
+ let(:size_file) { 1024 * 2 }
8
+ let(:time_file) { 2 }
9
+ let(:name) { "foobar" }
10
+ let(:tmp_file) { Stud::Temporary.file }
11
+ let(:tmp_dir) { tmp_file.path }
12
+ let(:file) { LogStash::Outputs::Qingstor::TemporaryFile.new(name, tmp_file, tmp_dir) }
13
+ let(:content) { "May the code be with you" * 100 }
14
+ subject { described_class.new(size_file, time_file) }
15
+
16
+ it "raise error if time_file is no grater then 0" do
17
+ expect{ described_class.new(0, 0) }.to raise_error(LogStash::ConfigurationError)
18
+ expect{ described_class.new(-1, 0) }.to raise_error(LogStash::ConfigurationError)
19
+ expect{ described_class.new(0, -1) }.to raise_error(LogStash::ConfigurationError)
20
+ expect{ described_class.new(-1, -1) }.to raise_error(LogStash::ConfigurationError)
21
+
22
+ end
23
+
24
+ it "return false if the file is not old enough" do
25
+ expect(subject.rotate?(file)).to be_falsey
26
+ end
27
+
28
+ it "return false if the file is old enough with file size 0" do
29
+ allow(file).to receive(:ctime).and_return(Time.now - (time_file * 2 * 60))
30
+ expect(subject.rotate?(file)).to be_falsey
31
+ end
32
+
33
+ it "return truth if the file is old enough and non-empty" do
34
+ file.write(content)
35
+ file.fsync
36
+ allow(file).to receive(:ctime).and_return(Time.now - (time_file * 2 * 60))
37
+ expect(subject.rotate?(file)).to be_truthy
38
+ end
39
+ end
@@ -0,0 +1,25 @@
1
+ # encoding: utf-8
2
+ require "logstash/devutils/rspec/spec_helper"
3
+ require "logstash/outputs/qingstor/temporary_file"
4
+ require "logstash/outputs/qingstor/size_rotation_policy"
5
+
6
+ describe LogStash::Outputs::Qingstor::SizeRotationPolicy do
7
+ let(:size_file) { 1024 * 2 }
8
+ let(:name) { "foobar" }
9
+ let(:tmp_file) { Stud::Temporary.file }
10
+ let(:tmp_dir) { tmp_file.path }
11
+ let(:file) { LogStash::Outputs::Qingstor::TemporaryFile.new(name, tmp_file, tmp_dir) }
12
+ let(:content) { "May the code be with you" * 100 }
13
+ subject { described_class.new(size_file) }
14
+
15
+ it "raise error if size_file is no grater then 0" do
16
+ expect{described_class.new(0)}.to raise_error(LogStash::ConfigurationError)
17
+ expect{described_class.new(-1)}.to raise_error(LogStash::ConfigurationError)
18
+ end
19
+
20
+ it "return true if the file has a bigger size value then 'size_file'" do
21
+ file.write(content)
22
+ file.fsync
23
+ expect(subject.rotate?(file)).to be_truthy
24
+ end
25
+ end
@@ -0,0 +1,82 @@
1
+ # encoding: utf-8
2
+ require "logstash/devutils/rspec/spec_helper"
3
+ require "logstash/outputs/qingstor/temporary_file"
4
+ require "logstash/outputs/qingstor/temporary_file_factory"
5
+ require "fileutils"
6
+ require "tmpdir"
7
+
8
+ describe LogStash::Outputs::Qingstor::TemporaryFileFactory do
9
+ let(:prefix) { "lg2qs" }
10
+ let(:tags) { [ ] }
11
+ let(:tmpdir) { File.join(Dir.tmpdir, "logstash-qs") }
12
+
13
+ subject { described_class.new(prefix, tags, encoding, tmpdir) }
14
+
15
+ shared_examples "file factory" do
16
+ it "creates the file on disk" do
17
+ expect(File.exist?(subject.current.path)).to be_truthy
18
+ end
19
+
20
+ it "create a temporary file when initialized" do
21
+ expect(subject.current).to be_kind_of(LogStash::Outputs::Qingstor::TemporaryFile)
22
+ end
23
+
24
+ it "create a file in the right format" do
25
+ expect(subject.current.path).to match(extension)
26
+ end
27
+
28
+ it "allow to rotate the file" do
29
+ file_path = subject.current.path
30
+ expect(subject.rotate!.path).not_to eq(file_path)
31
+ end
32
+
33
+ it "increments the part name on rotation" do
34
+ expect(subject.current.path).to match(/part0/)
35
+ expect(subject.rotate!.path).to match(/part1/)
36
+ end
37
+
38
+ it "includes the date" do
39
+ n = Time.now
40
+ expect(subject.current.path).to include(n.strftime("%Y-%m-%dT"))
41
+ end
42
+
43
+ it "include the file key in the path" do
44
+ file = subject.current
45
+ expect(file.path).to match(/#{file.key}/)
46
+ end
47
+
48
+ it "create a unique directory in the temporary directory for each file" do
49
+ uuid = "hola"
50
+ expect(SecureRandom).to receive(:uuid).and_return(uuid).twice
51
+ expect(subject.current.path).to include(uuid)
52
+ end
53
+
54
+ context "with tags supplied" do
55
+ let(:tags) { ["secret", "service"] }
56
+
57
+ it "adds tags to the filename" do
58
+ expect(subject.current.path).to match(/tag_#{tags.join('.')}.part/)
59
+ end
60
+ end
61
+
62
+ context "without tags" do
63
+ it "doesn't add tags to the filename" do
64
+ expect(subject.current.path).not_to match(/tag_/)
65
+ end
66
+ end
67
+ end
68
+
69
+ context "when gzip" do
70
+ let(:encoding) { "gzip" }
71
+ let(:extension) { /\.log.gz$/ }
72
+
73
+ include_examples "file factory"
74
+ end
75
+
76
+ context "when encoding set to `none`" do
77
+ let(:encoding) { "none" }
78
+ let(:extension) { /\.log$/ }
79
+
80
+ include_examples "file factory"
81
+ end
82
+ end
@@ -0,0 +1,55 @@
1
+ # encoding: utf-8
2
+ require "logstash/devutils/rspec/spec_helper"
3
+ require "logstash/outputs/qingstor/temporary_file"
4
+ require "fileutils"
5
+ require "tmpdir"
6
+
7
+ describe LogStash::Outputs::Qingstor::TemporaryFile do
8
+ let(:key) { "foo.log" }
9
+ let(:content) { "May the code be with you!" }
10
+ let(:tmp_path) { File.join(Dir.tmpdir, key) }
11
+ let(:file_mode) { "w+" }
12
+ let(:io){ ::File.open(tmp_path, file_mode)}
13
+
14
+ after(:all) do
15
+ FileUtils.rm_r("/tmp/foo.log") if File.exist?("/tmp/foo.log")
16
+ end
17
+
18
+ subject { described_class.new(key, io, tmp_path) }
19
+
20
+ it "return the key of the file" do
21
+ expect(subject.key).to eq(key)
22
+ end
23
+
24
+ it "is writable and readable" do
25
+ subject.write(content)
26
+ subject.close
27
+ expect(File.read(subject.path).strip).to eq(content)
28
+ end
29
+
30
+ it "can return the correct file size" do
31
+ subject.write(content)
32
+ subject.close
33
+ expect(subject.size).to eq(File.size(tmp_path))
34
+ end
35
+
36
+ it "return the tmp_path of the file" do
37
+ expect(subject.tmp_path).to eq(tmp_path)
38
+ end
39
+
40
+ it "return the creation time" do
41
+ expect(subject.ctime).to be < Time.now + 1.0
42
+ end
43
+
44
+ it "can delete file correctly" do
45
+ expect(File.exist?(subject.tmp_path)).to be_truthy
46
+ subject.delete!
47
+ expect(File.exist?(subject.tmp_path)).to be_falsey
48
+ end
49
+
50
+ it "return if the file is empty" do
51
+ expect(subject.empty?).to be_truthy
52
+ subject.write(content)
53
+ expect(subject.empty?).to be_falsey
54
+ end
55
+ end
@@ -0,0 +1,35 @@
1
+ # encoding: utf-8
2
+ require "logstash/devutils/rspec/spec_helper"
3
+ require "logstash/outputs/qingstor/temporary_file"
4
+ require "logstash/outputs/qingstor/time_rotation_policy"
5
+
6
+ describe LogStash::Outputs::Qingstor::TimeRotationPolicy do
7
+ let(:time_file) { 2 }
8
+ let(:name) { "foobar" }
9
+ let(:tmp_file) { Stud::Temporary.file }
10
+ let(:tmp_dir) { tmp_file.path }
11
+ let(:file) { LogStash::Outputs::Qingstor::TemporaryFile.new(name, tmp_file, tmp_dir) }
12
+ let(:content) { "May the code be with you" * 100 }
13
+ subject { described_class.new(time_file) }
14
+
15
+ it "raise error if time_file is no grater then 0" do
16
+ expect{described_class.new(0)}.to raise_error(LogStash::ConfigurationError)
17
+ expect{described_class.new(-1)}.to raise_error(LogStash::ConfigurationError)
18
+ end
19
+
20
+ it "return false if the file is not old enough" do
21
+ expect(subject.rotate?(file)).to be_falsey
22
+ end
23
+
24
+ it "return false if the file is old enough with file size 0" do
25
+ allow(file).to receive(:ctime).and_return(Time.now - (time_file * 2 * 60))
26
+ expect(subject.rotate?(file)).to be_falsey
27
+ end
28
+
29
+ it "return truth if the file is old enough and non-empty" do
30
+ file.write(content)
31
+ file.fsync
32
+ allow(file).to receive(:ctime).and_return(Time.now - (time_file * 2 * 60))
33
+ expect(subject.rotate?(file)).to be_truthy
34
+ end
35
+ end