logstash-output-qingstor 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,26 @@
1
+ # encoding: utf-8
2
+ module LogStash
3
+ module Outputs
4
+ class Qingstor
5
+ class TimeRotationPolicy
6
+ attr_reader :time_file
7
+
8
+ def initialize(time_file)
9
+ if time_file <= 0
10
+ raise LogStash::ConfigurationError, "'time_file' need to be greater then 0"
11
+ end
12
+
13
+ @time_file = time_file * 60
14
+ end
15
+
16
+ def rotate?(file)
17
+ file.size > 0 && (Time.now - file.ctime) >= time_file
18
+ end
19
+
20
+ def needs_periodic?
21
+ true
22
+ end
23
+ end
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,64 @@
1
+ # encoding: utf-8
2
+ require "qingstor/sdk"
3
+ require "concurrent"
4
+ require "digest/md5"
5
+ require "base64"
6
+ module LogStash
7
+ module Outputs
8
+ class Qingstor
9
+ class Uploader
10
+ TIME_BEFORE_RETRYING_SECONDS = 1
11
+ DEFAULT_THREADPOOL = Concurrent::ThreadPoolExecutor.new({ :min_thread => 1,
12
+ :max_thread => 8,
13
+ :max_queue => 1,
14
+ :fallback_policy => :caller_runs
15
+ })
16
+ attr_reader :bucket, :upload_options, :logger
17
+
18
+ def initialize(bucket, logger, threadpool = DEFAULT_THREADPOOL)
19
+ @bucket = bucket
20
+ @logger = logger
21
+ @workers_pool = threadpool
22
+ end
23
+
24
+ def upload_async(file, options = {})
25
+ @workers_pool.post do
26
+ upload(file, options)
27
+ end
28
+ end
29
+
30
+ def upload(file, options = {})
31
+ upload_options = options.fetch(:upload_options, {})
32
+
33
+ file_md5 = Digest::MD5.file(file.path).to_s
34
+
35
+ upload_headers = {
36
+ "content_md5" => file_md5,
37
+ "body" => ::File.open(file.path)
38
+ }
39
+
40
+ if !upload_options[:server_side_encryption_algorithm].nil?
41
+ base64_key = Base64.strict_encode64 upload_options[:customer_key]
42
+ key_md5 = Digest::MD5.hexdigest upload_options[:customer_key]
43
+ base64_key_md5 = Base64.strict_encode64 key_md5
44
+ upload_headers.merge!({
45
+ "x_qs_encryption_customer_algorithm" => upload_options[:server_side_encryption_algorithm],
46
+ "x_qs_encryption_customer_key" => base64_key,
47
+ "x_qs_encryption_customer_key_md5" => base64_key_md5,
48
+ })
49
+ end
50
+ @logger.debug("uploading file", :file => file.key)
51
+ bucket.put_object(file.key, upload_headers)
52
+
53
+ options[:on_complete].call(file) unless options[:on_complete].nil?
54
+ end
55
+
56
+ def stop
57
+ @workers_pool.shutdown
58
+ @workers_pool.wait_for_termination(nil)
59
+ end
60
+ end
61
+ end
62
+ end
63
+ end
64
+
@@ -0,0 +1,253 @@
1
+ # encoding: utf-8
2
+ require "logstash-core"
3
+ require "logstash/outputs/base"
4
+ require "logstash/namespace"
5
+ require "tmpdir"
6
+ require "qingstor/sdk"
7
+ require "concurrent"
8
+
9
+ class LogStash::Outputs::Qingstor < LogStash::Outputs::Base
10
+ require "logstash/outputs/qingstor/temporary_file"
11
+ require "logstash/outputs/qingstor/temporary_file_factory"
12
+ require "logstash/outputs/qingstor/file_repository"
13
+ require "logstash/outputs/qingstor/size_rotation_policy"
14
+ require "logstash/outputs/qingstor/time_rotation_policy"
15
+ require "logstash/outputs/qingstor/size_and_time_rotation_policy"
16
+ require "logstash/outputs/qingstor/uploader"
17
+ require "logstash/outputs/qingstor/qingstor_validator"
18
+
19
+ PERIODIC_CHECK_INTERVAL_IN_SECONDS = 15
20
+ CRASH_RECOVERY_THREADPOOL = Concurrent::ThreadPoolExecutor.new({
21
+ :min_threads => 1,
22
+ :max_threads => 2,
23
+ :fallback_policy => :caller_runs
24
+ })
25
+ config_name "qingstor"
26
+
27
+ # When configured as :single a single instance of the Output will be shared among the
28
+ # pipeline worker threads. Access to the `#multi_receive/#multi_receive_encoded/#receive` method will be synchronized
29
+ # i.e. only one thread will be active at a time making threadsafety much simpler.
30
+ #
31
+ # You can set this to :shared if your output is threadsafe. This will maximize
32
+ # concurrency but you will need to make appropriate uses of mutexes in `#multi_receive/#receive`.
33
+ #
34
+ # Only the `#multi_receive/#multi_receive_encoded` methods need to actually be threadsafe, the other methods
35
+ # will only be executed in a single thread
36
+ concurrency :shared
37
+
38
+ # The key id to access your QingStor
39
+ config :access_key_id, :validate => :string, :required => true
40
+
41
+ # The key to access your QingStor
42
+ config :secret_access_key, :validate => :string, :required => true
43
+
44
+ # The name of the qingstor bucket
45
+ config :bucket, :validate => :string, :required => true
46
+
47
+ # The region of the QingStor bucket
48
+ config :region, :validate => ["pek3a", "sh1a"], :default => "pek3a"
49
+
50
+ # The prefix of filenames which will work as directory in qingstor
51
+ config :prefix, :validate => :string, :default => ''
52
+
53
+ # Set the directory where logstash store the tmp files before
54
+ # sending it to qingstor, default directory in linux /tmp/logstash2qingstor
55
+ config :tmpdir, :validate => :string, :default => File.join(Dir.tmpdir, "logstash2qingstor")
56
+
57
+ # Define tags to append to the file on the qingstor bucket
58
+ config :tags, :validate => :array, :default => []
59
+
60
+ # Specify the content encoding. Supports ("gzip"), defaults to "none"
61
+ config :encoding, :validate => ["gzip", "none"], :default => "none"
62
+
63
+ # Define the strategy to use to decide when we need to rotate the file and push it to S3,
64
+ # The default strategy is to check for both size and time, the first one to match will rotate the file.
65
+ config :rotation_strategy, :validate => ["size_and_time", "size", "time"], :default => "size_and_time"
66
+
67
+ # Define the size requirement for each file to upload to qingstor. In byte.
68
+ config :size_file, :validate => :number, :default => 1024 * 1024 * 5
69
+
70
+ # Define the time interval for each file to upload to qingstor. In minutes.
71
+ config :time_file, :validate => :number, :default => 15
72
+
73
+ # Specify maximum number of workers to use to upload the files to Qingstor
74
+ config :upload_workers_count, :validate => :number, :default => (Concurrent.processor_count * 0.5).ceil
75
+
76
+ # Number of items we can keep in the local queue before uploading them
77
+ config :upload_queue_size, :validate => :number, :default => 2 * (Concurrent.processor_count * 0.25).ceil
78
+
79
+ # Specifies what type of encryption to use when SSE is enabled.
80
+ config :server_side_encryption_algorithm, :validate => ["AES256", "none"], :default => "none"
81
+
82
+ # Specifies the encryption customer key that would be used in server side
83
+ config :customer_key, :validate => :string
84
+
85
+ # Specifies if set to true, it would upload existing file in targeting folder at the beginning.
86
+ config :restore, :validate => :boolean, :default => false
87
+
88
+ public
89
+ def register
90
+ QingstorValidator.prefix_valid?(@prefix) unless @prefix.nil?
91
+
92
+ if !directory_valid?(@tmpdir)
93
+ raise LogStash::ConfigurationError, "Logstash must have the permissions to write to the temporary directory: #{@tmpdir}"
94
+ end
95
+
96
+ @file_repository = FileRepository.new(@tags, @encoding, @tmpdir)
97
+
98
+ @rotation = rotation_strategy
99
+
100
+ executor = Concurrent::ThreadPoolExecutor.new({
101
+ :min_threads => 1,
102
+ :max_threads => @upload_workers_count,
103
+ :max_queue => @upload_queue_size,
104
+ :fallback_policy => :caller_runs
105
+ })
106
+
107
+ @qs_bucket = get_bucket
108
+ QingstorValidator.bucket_valid?(@qs_bucket)
109
+
110
+ @uploader = Uploader.new(@qs_bucket, @logger, executor)
111
+
112
+ start_periodic_check if @rotation.needs_periodic?
113
+
114
+ restore_from_crash if @restore
115
+ end # def register
116
+
117
+ public
118
+ def multi_receive_encoded(events_and_encoded)
119
+ prefix_written_to = Set.new
120
+
121
+ events_and_encoded.each do |event, encoded|
122
+ prefix_key = event.sprintf(@prefix)
123
+ prefix_written_to << prefix_key
124
+
125
+ begin
126
+ @file_repository.get_file(prefix_key) { |file| file.write(encoded) }
127
+ rescue Errno::ENOSPC => e
128
+ @logger.error("QingStor: Nospace left in temporary directory", :tmpdir => @tmpdir)
129
+ raise e
130
+ end
131
+ end # end of each method
132
+
133
+ # check the file after file writing
134
+ # Groups IO calls to optimize fstat checks
135
+ rotate_if_needed(prefix_written_to)
136
+ end # def multi_receive_encoded
137
+
138
+ def rotation_strategy
139
+ case @rotation_strategy
140
+ when "size"
141
+ SizeRotationPolicy.new(@size_file)
142
+ when "time"
143
+ TimeRotationPolicy.new(@time_file)
144
+ when "size_and_time"
145
+ SizeAndTimeRotationPolicy.new(@size_file, @time_file)
146
+ end
147
+ end
148
+
149
+ def rotate_if_needed(prefixs)
150
+ prefixs.each do |prefix|
151
+ @file_repository.get_factory(prefix) do |factory|
152
+ tmp_file = factory.current
153
+
154
+ if @rotation.rotate?(tmp_file)
155
+ @logger.debug("Rotate file",
156
+ :strategy => tmp_file.key,
157
+ :path => tmp_file.path)
158
+ upload_file(tmp_file)
159
+ factory.rotate!
160
+ end
161
+ end
162
+ end
163
+ end # def rotate_if_needed
164
+
165
+ def upload_file(file)
166
+ @logger.debug("Add file to uploading queue", :key => file.key)
167
+ file.close
168
+ @logger.debug("upload options", :upload_options => upload_options)
169
+ @uploader.upload_async(file,
170
+ :on_complete => method(:clean_temporary_file),
171
+ :upload_options => upload_options)
172
+ end
173
+
174
+ def get_bucket
175
+ @qs_config = QingStor::SDK::Config.init @access_key_id, @secret_access_key
176
+ @qs_service = QingStor::SDK::Service.new @qs_config
177
+ @qs_service.bucket @bucket, @region
178
+ end
179
+
180
+ def close
181
+ stop_periodic_check if @rotation.needs_periodic?
182
+
183
+ @logger.debug("uploading current workspace")
184
+ @file_repository.each_files do |file|
185
+ upload_file(file)
186
+ end
187
+
188
+ @file_repository.shutdown
189
+
190
+ @uploader.stop
191
+
192
+ @crash_uploader.stop if @restore
193
+ end
194
+
195
+ def upload_options
196
+ options = {
197
+ :content_encoding => @encoding == "gzip" ? "gzip" : nil
198
+ }
199
+
200
+ if @server_side_encryption_algorithm == "AES256" && !@customer_key.nil?
201
+ options.merge!({
202
+ :server_side_encryption_algorithm => @server_side_encryption_algorithm,
203
+ :customer_key => @customer_key
204
+ })
205
+ end
206
+
207
+ options
208
+ end
209
+
210
+ def clean_temporary_file(file)
211
+ @logger.debug("Removing temporary file", :file => file.path)
212
+ file.delete!
213
+ end
214
+
215
+ def start_periodic_check
216
+ @logger.debug("Start periodic rotation check")
217
+
218
+ @periodic_check = Concurrent::TimerTask.new(:execution_interval => PERIODIC_CHECK_INTERVAL_IN_SECONDS) do
219
+ @logger.debug("Periodic check for stale files")
220
+
221
+ rotate_if_needed(@file_repository.keys)
222
+ end
223
+
224
+ @periodic_check.execute
225
+ end
226
+
227
+ def stop_periodic_check
228
+ @periodic_check.shutdown
229
+ end
230
+
231
+ def directory_valid?(path)
232
+ begin
233
+ FileUtils.mkdir_p(path) unless Dir.exist?(path)
234
+ ::File.writable?(path)
235
+ rescue
236
+ false
237
+ end
238
+ end
239
+
240
+ def restore_from_crash
241
+ @crash_uploader = Uploader.new(@qs_bucket, @logger, CRASH_RECOVERY_THREADPOOL)
242
+
243
+ temp_folder_path = Pathname.new(@tmpdir)
244
+ Dir.glob(::File.join(@tmpdir, "**/*"))
245
+ .select { |file| ::File.file?(file) }
246
+ .each do |file|
247
+ temp_file = TemporaryFile.create_from_existing_file(file, temp_folder_path)
248
+ @logger.debug("Recoving from crash and uploading", :file => temp_file.path)
249
+ @crash_uploader.upload_async(temp_file, :on_complete => method(:clean_temporary_file),
250
+ :upload_options => upload_options)
251
+ end
252
+ end
253
+ end # class LogStash::Outputs::Qingstor
@@ -0,0 +1,28 @@
1
+ Gem::Specification.new do |s|
2
+ s.name = 'logstash-output-qingstor'
3
+ s.version = '0.1.0'
4
+ s.licenses = ['Apache License (2.0)']
5
+ s.summary = 'logstash output plugin for qingstor'
6
+ s.description = 'Put the outcomes of logstash into qingstor'
7
+ s.homepage = 'https://github.com/tacinight/logstash-output-qingstor'
8
+ s.authors = ['Evan Zhao']
9
+ s.email = 'tacingiht@gmail.com'
10
+ s.require_paths = ['lib']
11
+
12
+ # Files
13
+ s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
14
+ # Tests
15
+ s.test_files = s.files.grep(%r{^(test|spec|features)/})
16
+
17
+ # Special flag to let us know this is actually a logstash plugin
18
+ s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
19
+
20
+ # Gem dependencies
21
+ s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
22
+ s.add_runtime_dependency "logstash-codec-plain"
23
+ s.add_runtime_dependency "qingstor-sdk", "=1.9.2"
24
+ s.add_runtime_dependency "concurrent-ruby"
25
+
26
+ s.add_development_dependency "stud", "~> 0.0.22"
27
+ s.add_development_dependency "logstash-devutils"
28
+ end
@@ -0,0 +1,26 @@
1
+ require "logstash/devutils/rspec/spec_helper"
2
+ require "logstash/outputs/qingstor/temporary_file"
3
+ require "logstash/outputs/qingstor/temporary_file_factory"
4
+ require "logstash/outputs/qingstor/file_repository"
5
+ require "tmpdir"
6
+
7
+ describe LogStash::Outputs::Qingstor::FileRepository do
8
+ let(:tags) { ["tag1", "tag2", "tag3"]}
9
+ let(:encoding) { "none" }
10
+ let(:tmpdir) { File.join(Dir.tmpdir, "lg2qs") }
11
+ let(:prefix) { "aprefix" }
12
+
13
+ subject { described_class.new(tags, encoding, tmpdir) }
14
+
15
+ it "can get current file io" do
16
+ subject.get_file(prefix) do |file|
17
+ expect(file).to be_kind_of(LogStash::Outputs::Qingstor::TemporaryFile)
18
+ end
19
+ end
20
+
21
+ it "can get current file factory" do
22
+ subject.get_factory(prefix) do |factory|
23
+ expect(factory).to be_kind_of(LogStash::Outputs::Qingstor::TemporaryFileFactory)
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,21 @@
1
+ # encoding: utf-8
2
+ require "logstash/devutils/rspec/spec_helper"
3
+ require "logstash/outputs/qingstor/qingstor_validator"
4
+ require_relative "../qs_access_helper"
5
+
6
+ describe LogStash::Outputs::Qingstor::QingstorValidator do
7
+ let(:normal_prefix) { "super/bucket" }
8
+ let(:wrong_prefix1) { "/wrong/prefix" }
9
+ let(:wrong_prefix2) { normal_prefix * 100 }
10
+ let(:bucket) { qs_bucket_init }
11
+
12
+ it "raise error if the prefix is not valid" do
13
+ expect{ described_class.prefix_valid?(wrong_prefix1) }.to raise_error(LogStash::ConfigurationError)
14
+ expect{ described_class.prefix_valid?(wrong_prefix2) }.to raise_error(LogStash::ConfigurationError)
15
+ end
16
+
17
+ it "return true if the prefix is valid" do
18
+ expect(described_class.prefix_valid?(normal_prefix)).to be_truthy
19
+ end
20
+
21
+ end
@@ -0,0 +1,39 @@
1
+ # encoding: utf-8
2
+ require "logstash/devutils/rspec/spec_helper"
3
+ require "logstash/outputs/qingstor/temporary_file"
4
+ require "logstash/outputs/qingstor/size_and_time_rotation_policy"
5
+
6
+ describe LogStash::Outputs::Qingstor::SizeAndTimeRotationPolicy do
7
+ let(:size_file) { 1024 * 2 }
8
+ let(:time_file) { 2 }
9
+ let(:name) { "foobar" }
10
+ let(:tmp_file) { Stud::Temporary.file }
11
+ let(:tmp_dir) { tmp_file.path }
12
+ let(:file) { LogStash::Outputs::Qingstor::TemporaryFile.new(name, tmp_file, tmp_dir) }
13
+ let(:content) { "May the code be with you" * 100 }
14
+ subject { described_class.new(size_file, time_file) }
15
+
16
+ it "raise error if time_file is no grater then 0" do
17
+ expect{ described_class.new(0, 0) }.to raise_error(LogStash::ConfigurationError)
18
+ expect{ described_class.new(-1, 0) }.to raise_error(LogStash::ConfigurationError)
19
+ expect{ described_class.new(0, -1) }.to raise_error(LogStash::ConfigurationError)
20
+ expect{ described_class.new(-1, -1) }.to raise_error(LogStash::ConfigurationError)
21
+
22
+ end
23
+
24
+ it "return false if the file is not old enough" do
25
+ expect(subject.rotate?(file)).to be_falsey
26
+ end
27
+
28
+ it "return false if the file is old enough with file size 0" do
29
+ allow(file).to receive(:ctime).and_return(Time.now - (time_file * 2 * 60))
30
+ expect(subject.rotate?(file)).to be_falsey
31
+ end
32
+
33
+ it "return truth if the file is old enough and non-empty" do
34
+ file.write(content)
35
+ file.fsync
36
+ allow(file).to receive(:ctime).and_return(Time.now - (time_file * 2 * 60))
37
+ expect(subject.rotate?(file)).to be_truthy
38
+ end
39
+ end
@@ -0,0 +1,25 @@
1
+ # encoding: utf-8
2
+ require "logstash/devutils/rspec/spec_helper"
3
+ require "logstash/outputs/qingstor/temporary_file"
4
+ require "logstash/outputs/qingstor/size_rotation_policy"
5
+
6
+ describe LogStash::Outputs::Qingstor::SizeRotationPolicy do
7
+ let(:size_file) { 1024 * 2 }
8
+ let(:name) { "foobar" }
9
+ let(:tmp_file) { Stud::Temporary.file }
10
+ let(:tmp_dir) { tmp_file.path }
11
+ let(:file) { LogStash::Outputs::Qingstor::TemporaryFile.new(name, tmp_file, tmp_dir) }
12
+ let(:content) { "May the code be with you" * 100 }
13
+ subject { described_class.new(size_file) }
14
+
15
+ it "raise error if size_file is no grater then 0" do
16
+ expect{described_class.new(0)}.to raise_error(LogStash::ConfigurationError)
17
+ expect{described_class.new(-1)}.to raise_error(LogStash::ConfigurationError)
18
+ end
19
+
20
+ it "return true if the file has a bigger size value then 'size_file'" do
21
+ file.write(content)
22
+ file.fsync
23
+ expect(subject.rotate?(file)).to be_truthy
24
+ end
25
+ end
@@ -0,0 +1,82 @@
1
+ # encoding: utf-8
2
+ require "logstash/devutils/rspec/spec_helper"
3
+ require "logstash/outputs/qingstor/temporary_file"
4
+ require "logstash/outputs/qingstor/temporary_file_factory"
5
+ require "fileutils"
6
+ require "tmpdir"
7
+
8
+ describe LogStash::Outputs::Qingstor::TemporaryFileFactory do
9
+ let(:prefix) { "lg2qs" }
10
+ let(:tags) { [ ] }
11
+ let(:tmpdir) { File.join(Dir.tmpdir, "logstash-qs") }
12
+
13
+ subject { described_class.new(prefix, tags, encoding, tmpdir) }
14
+
15
+ shared_examples "file factory" do
16
+ it "creates the file on disk" do
17
+ expect(File.exist?(subject.current.path)).to be_truthy
18
+ end
19
+
20
+ it "create a temporary file when initialized" do
21
+ expect(subject.current).to be_kind_of(LogStash::Outputs::Qingstor::TemporaryFile)
22
+ end
23
+
24
+ it "create a file in the right format" do
25
+ expect(subject.current.path).to match(extension)
26
+ end
27
+
28
+ it "allow to rotate the file" do
29
+ file_path = subject.current.path
30
+ expect(subject.rotate!.path).not_to eq(file_path)
31
+ end
32
+
33
+ it "increments the part name on rotation" do
34
+ expect(subject.current.path).to match(/part0/)
35
+ expect(subject.rotate!.path).to match(/part1/)
36
+ end
37
+
38
+ it "includes the date" do
39
+ n = Time.now
40
+ expect(subject.current.path).to include(n.strftime("%Y-%m-%dT"))
41
+ end
42
+
43
+ it "include the file key in the path" do
44
+ file = subject.current
45
+ expect(file.path).to match(/#{file.key}/)
46
+ end
47
+
48
+ it "create a unique directory in the temporary directory for each file" do
49
+ uuid = "hola"
50
+ expect(SecureRandom).to receive(:uuid).and_return(uuid).twice
51
+ expect(subject.current.path).to include(uuid)
52
+ end
53
+
54
+ context "with tags supplied" do
55
+ let(:tags) { ["secret", "service"] }
56
+
57
+ it "adds tags to the filename" do
58
+ expect(subject.current.path).to match(/tag_#{tags.join('.')}.part/)
59
+ end
60
+ end
61
+
62
+ context "without tags" do
63
+ it "doesn't add tags to the filename" do
64
+ expect(subject.current.path).not_to match(/tag_/)
65
+ end
66
+ end
67
+ end
68
+
69
+ context "when gzip" do
70
+ let(:encoding) { "gzip" }
71
+ let(:extension) { /\.log.gz$/ }
72
+
73
+ include_examples "file factory"
74
+ end
75
+
76
+ context "when encoding set to `none`" do
77
+ let(:encoding) { "none" }
78
+ let(:extension) { /\.log$/ }
79
+
80
+ include_examples "file factory"
81
+ end
82
+ end
@@ -0,0 +1,55 @@
1
+ # encoding: utf-8
2
+ require "logstash/devutils/rspec/spec_helper"
3
+ require "logstash/outputs/qingstor/temporary_file"
4
+ require "fileutils"
5
+ require "tmpdir"
6
+
7
+ describe LogStash::Outputs::Qingstor::TemporaryFile do
8
+ let(:key) { "foo.log" }
9
+ let(:content) { "May the code be with you!" }
10
+ let(:tmp_path) { File.join(Dir.tmpdir, key) }
11
+ let(:file_mode) { "w+" }
12
+ let(:io){ ::File.open(tmp_path, file_mode)}
13
+
14
+ after(:all) do
15
+ FileUtils.rm_r("/tmp/foo.log") if File.exist?("/tmp/foo.log")
16
+ end
17
+
18
+ subject { described_class.new(key, io, tmp_path) }
19
+
20
+ it "return the key of the file" do
21
+ expect(subject.key).to eq(key)
22
+ end
23
+
24
+ it "is writable and readable" do
25
+ subject.write(content)
26
+ subject.close
27
+ expect(File.read(subject.path).strip).to eq(content)
28
+ end
29
+
30
+ it "can return the correct file size" do
31
+ subject.write(content)
32
+ subject.close
33
+ expect(subject.size).to eq(File.size(tmp_path))
34
+ end
35
+
36
+ it "return the tmp_path of the file" do
37
+ expect(subject.tmp_path).to eq(tmp_path)
38
+ end
39
+
40
+ it "return the creation time" do
41
+ expect(subject.ctime).to be < Time.now + 1.0
42
+ end
43
+
44
+ it "can delete file correctly" do
45
+ expect(File.exist?(subject.tmp_path)).to be_truthy
46
+ subject.delete!
47
+ expect(File.exist?(subject.tmp_path)).to be_falsey
48
+ end
49
+
50
+ it "return if the file is empty" do
51
+ expect(subject.empty?).to be_truthy
52
+ subject.write(content)
53
+ expect(subject.empty?).to be_falsey
54
+ end
55
+ end
@@ -0,0 +1,35 @@
1
+ # encoding: utf-8
2
+ require "logstash/devutils/rspec/spec_helper"
3
+ require "logstash/outputs/qingstor/temporary_file"
4
+ require "logstash/outputs/qingstor/time_rotation_policy"
5
+
6
+ describe LogStash::Outputs::Qingstor::TimeRotationPolicy do
7
+ let(:time_file) { 2 }
8
+ let(:name) { "foobar" }
9
+ let(:tmp_file) { Stud::Temporary.file }
10
+ let(:tmp_dir) { tmp_file.path }
11
+ let(:file) { LogStash::Outputs::Qingstor::TemporaryFile.new(name, tmp_file, tmp_dir) }
12
+ let(:content) { "May the code be with you" * 100 }
13
+ subject { described_class.new(time_file) }
14
+
15
+ it "raise error if time_file is no grater then 0" do
16
+ expect{described_class.new(0)}.to raise_error(LogStash::ConfigurationError)
17
+ expect{described_class.new(-1)}.to raise_error(LogStash::ConfigurationError)
18
+ end
19
+
20
+ it "return false if the file is not old enough" do
21
+ expect(subject.rotate?(file)).to be_falsey
22
+ end
23
+
24
+ it "return false if the file is old enough with file size 0" do
25
+ allow(file).to receive(:ctime).and_return(Time.now - (time_file * 2 * 60))
26
+ expect(subject.rotate?(file)).to be_falsey
27
+ end
28
+
29
+ it "return truth if the file is old enough and non-empty" do
30
+ file.write(content)
31
+ file.fsync
32
+ allow(file).to receive(:ctime).and_return(Time.now - (time_file * 2 * 60))
33
+ expect(subject.rotate?(file)).to be_truthy
34
+ end
35
+ end