logstash-output-qingstor 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +2 -0
- data/CODE_OF_CONDUCT.md +44 -0
- data/CONTRIBUTING.md +43 -0
- data/CONTRIBUTORS +10 -0
- data/DEVELOPER.md +2 -0
- data/Gemfile +3 -0
- data/LICENSE +11 -0
- data/README.md +181 -0
- data/lib/logstash/outputs/qingstor/file_repository.rb +112 -0
- data/lib/logstash/outputs/qingstor/qingstor_validator.rb +30 -0
- data/lib/logstash/outputs/qingstor/size_and_time_rotation_policy.rb +24 -0
- data/lib/logstash/outputs/qingstor/size_rotation_policy.rb +26 -0
- data/lib/logstash/outputs/qingstor/temporary_file.rb +63 -0
- data/lib/logstash/outputs/qingstor/temporary_file_factory.rb +110 -0
- data/lib/logstash/outputs/qingstor/time_rotation_policy.rb +26 -0
- data/lib/logstash/outputs/qingstor/uploader.rb +64 -0
- data/lib/logstash/outputs/qingstor.rb +253 -0
- data/logstash-output-qingstor.gemspec +28 -0
- data/spec/outputs/qingstor/file_repository_spec.rb +26 -0
- data/spec/outputs/qingstor/qingstor_validator_spec.rb +21 -0
- data/spec/outputs/qingstor/size_and_time_rotation_policy_spec.rb +39 -0
- data/spec/outputs/qingstor/size_rotation_policy_spec.rb +25 -0
- data/spec/outputs/qingstor/temporary_file_factory_spec.rb +82 -0
- data/spec/outputs/qingstor/temporary_file_spec.rb +55 -0
- data/spec/outputs/qingstor/time_rotation_policy_spec.rb +35 -0
- data/spec/outputs/qingstor/uploader_spec.rb +49 -0
- data/spec/outputs/qingstor_spec.rb +50 -0
- data/spec/outputs/qs_access_helper.rb +38 -0
- data/spec/outputs/spec_helper.rb +4 -0
- metadata +169 -0
@@ -0,0 +1,26 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
module LogStash
|
3
|
+
module Outputs
|
4
|
+
class Qingstor
|
5
|
+
class TimeRotationPolicy
|
6
|
+
attr_reader :time_file
|
7
|
+
|
8
|
+
def initialize(time_file)
|
9
|
+
if time_file <= 0
|
10
|
+
raise LogStash::ConfigurationError, "'time_file' need to be greater then 0"
|
11
|
+
end
|
12
|
+
|
13
|
+
@time_file = time_file * 60
|
14
|
+
end
|
15
|
+
|
16
|
+
def rotate?(file)
|
17
|
+
file.size > 0 && (Time.now - file.ctime) >= time_file
|
18
|
+
end
|
19
|
+
|
20
|
+
def needs_periodic?
|
21
|
+
true
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,64 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "qingstor/sdk"
|
3
|
+
require "concurrent"
|
4
|
+
require "digest/md5"
|
5
|
+
require "base64"
|
6
|
+
module LogStash
|
7
|
+
module Outputs
|
8
|
+
class Qingstor
|
9
|
+
class Uploader
|
10
|
+
TIME_BEFORE_RETRYING_SECONDS = 1
|
11
|
+
DEFAULT_THREADPOOL = Concurrent::ThreadPoolExecutor.new({ :min_thread => 1,
|
12
|
+
:max_thread => 8,
|
13
|
+
:max_queue => 1,
|
14
|
+
:fallback_policy => :caller_runs
|
15
|
+
})
|
16
|
+
attr_reader :bucket, :upload_options, :logger
|
17
|
+
|
18
|
+
def initialize(bucket, logger, threadpool = DEFAULT_THREADPOOL)
|
19
|
+
@bucket = bucket
|
20
|
+
@logger = logger
|
21
|
+
@workers_pool = threadpool
|
22
|
+
end
|
23
|
+
|
24
|
+
def upload_async(file, options = {})
|
25
|
+
@workers_pool.post do
|
26
|
+
upload(file, options)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def upload(file, options = {})
|
31
|
+
upload_options = options.fetch(:upload_options, {})
|
32
|
+
|
33
|
+
file_md5 = Digest::MD5.file(file.path).to_s
|
34
|
+
|
35
|
+
upload_headers = {
|
36
|
+
"content_md5" => file_md5,
|
37
|
+
"body" => ::File.open(file.path)
|
38
|
+
}
|
39
|
+
|
40
|
+
if !upload_options[:server_side_encryption_algorithm].nil?
|
41
|
+
base64_key = Base64.strict_encode64 upload_options[:customer_key]
|
42
|
+
key_md5 = Digest::MD5.hexdigest upload_options[:customer_key]
|
43
|
+
base64_key_md5 = Base64.strict_encode64 key_md5
|
44
|
+
upload_headers.merge!({
|
45
|
+
"x_qs_encryption_customer_algorithm" => upload_options[:server_side_encryption_algorithm],
|
46
|
+
"x_qs_encryption_customer_key" => base64_key,
|
47
|
+
"x_qs_encryption_customer_key_md5" => base64_key_md5,
|
48
|
+
})
|
49
|
+
end
|
50
|
+
@logger.debug("uploading file", :file => file.key)
|
51
|
+
bucket.put_object(file.key, upload_headers)
|
52
|
+
|
53
|
+
options[:on_complete].call(file) unless options[:on_complete].nil?
|
54
|
+
end
|
55
|
+
|
56
|
+
def stop
|
57
|
+
@workers_pool.shutdown
|
58
|
+
@workers_pool.wait_for_termination(nil)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
@@ -0,0 +1,253 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash-core"
|
3
|
+
require "logstash/outputs/base"
|
4
|
+
require "logstash/namespace"
|
5
|
+
require "tmpdir"
|
6
|
+
require "qingstor/sdk"
|
7
|
+
require "concurrent"
|
8
|
+
|
9
|
+
class LogStash::Outputs::Qingstor < LogStash::Outputs::Base
|
10
|
+
require "logstash/outputs/qingstor/temporary_file"
|
11
|
+
require "logstash/outputs/qingstor/temporary_file_factory"
|
12
|
+
require "logstash/outputs/qingstor/file_repository"
|
13
|
+
require "logstash/outputs/qingstor/size_rotation_policy"
|
14
|
+
require "logstash/outputs/qingstor/time_rotation_policy"
|
15
|
+
require "logstash/outputs/qingstor/size_and_time_rotation_policy"
|
16
|
+
require "logstash/outputs/qingstor/uploader"
|
17
|
+
require "logstash/outputs/qingstor/qingstor_validator"
|
18
|
+
|
19
|
+
PERIODIC_CHECK_INTERVAL_IN_SECONDS = 15
|
20
|
+
CRASH_RECOVERY_THREADPOOL = Concurrent::ThreadPoolExecutor.new({
|
21
|
+
:min_threads => 1,
|
22
|
+
:max_threads => 2,
|
23
|
+
:fallback_policy => :caller_runs
|
24
|
+
})
|
25
|
+
config_name "qingstor"
|
26
|
+
|
27
|
+
# When configured as :single a single instance of the Output will be shared among the
|
28
|
+
# pipeline worker threads. Access to the `#multi_receive/#multi_receive_encoded/#receive` method will be synchronized
|
29
|
+
# i.e. only one thread will be active at a time making threadsafety much simpler.
|
30
|
+
#
|
31
|
+
# You can set this to :shared if your output is threadsafe. This will maximize
|
32
|
+
# concurrency but you will need to make appropriate uses of mutexes in `#multi_receive/#receive`.
|
33
|
+
#
|
34
|
+
# Only the `#multi_receive/#multi_receive_encoded` methods need to actually be threadsafe, the other methods
|
35
|
+
# will only be executed in a single thread
|
36
|
+
concurrency :shared
|
37
|
+
|
38
|
+
# The key id to access your QingStor
|
39
|
+
config :access_key_id, :validate => :string, :required => true
|
40
|
+
|
41
|
+
# The key to access your QingStor
|
42
|
+
config :secret_access_key, :validate => :string, :required => true
|
43
|
+
|
44
|
+
# The name of the qingstor bucket
|
45
|
+
config :bucket, :validate => :string, :required => true
|
46
|
+
|
47
|
+
# The region of the QingStor bucket
|
48
|
+
config :region, :validate => ["pek3a", "sh1a"], :default => "pek3a"
|
49
|
+
|
50
|
+
# The prefix of filenames which will work as directory in qingstor
|
51
|
+
config :prefix, :validate => :string, :default => ''
|
52
|
+
|
53
|
+
# Set the directory where logstash store the tmp files before
|
54
|
+
# sending it to qingstor, default directory in linux /tmp/logstash2qingstor
|
55
|
+
config :tmpdir, :validate => :string, :default => File.join(Dir.tmpdir, "logstash2qingstor")
|
56
|
+
|
57
|
+
# Define tags to append to the file on the qingstor bucket
|
58
|
+
config :tags, :validate => :array, :default => []
|
59
|
+
|
60
|
+
# Specify the content encoding. Supports ("gzip"), defaults to "none"
|
61
|
+
config :encoding, :validate => ["gzip", "none"], :default => "none"
|
62
|
+
|
63
|
+
# Define the strategy to use to decide when we need to rotate the file and push it to S3,
|
64
|
+
# The default strategy is to check for both size and time, the first one to match will rotate the file.
|
65
|
+
config :rotation_strategy, :validate => ["size_and_time", "size", "time"], :default => "size_and_time"
|
66
|
+
|
67
|
+
# Define the size requirement for each file to upload to qingstor. In byte.
|
68
|
+
config :size_file, :validate => :number, :default => 1024 * 1024 * 5
|
69
|
+
|
70
|
+
# Define the time interval for each file to upload to qingstor. In minutes.
|
71
|
+
config :time_file, :validate => :number, :default => 15
|
72
|
+
|
73
|
+
# Specify maximum number of workers to use to upload the files to Qingstor
|
74
|
+
config :upload_workers_count, :validate => :number, :default => (Concurrent.processor_count * 0.5).ceil
|
75
|
+
|
76
|
+
# Number of items we can keep in the local queue before uploading them
|
77
|
+
config :upload_queue_size, :validate => :number, :default => 2 * (Concurrent.processor_count * 0.25).ceil
|
78
|
+
|
79
|
+
# Specifies what type of encryption to use when SSE is enabled.
|
80
|
+
config :server_side_encryption_algorithm, :validate => ["AES256", "none"], :default => "none"
|
81
|
+
|
82
|
+
# Specifies the encryption customer key that would be used in server side
|
83
|
+
config :customer_key, :validate => :string
|
84
|
+
|
85
|
+
# Specifies if set to true, it would upload existing file in targeting folder at the beginning.
|
86
|
+
config :restore, :validate => :boolean, :default => false
|
87
|
+
|
88
|
+
public
|
89
|
+
def register
|
90
|
+
QingstorValidator.prefix_valid?(@prefix) unless @prefix.nil?
|
91
|
+
|
92
|
+
if !directory_valid?(@tmpdir)
|
93
|
+
raise LogStash::ConfigurationError, "Logstash must have the permissions to write to the temporary directory: #{@tmpdir}"
|
94
|
+
end
|
95
|
+
|
96
|
+
@file_repository = FileRepository.new(@tags, @encoding, @tmpdir)
|
97
|
+
|
98
|
+
@rotation = rotation_strategy
|
99
|
+
|
100
|
+
executor = Concurrent::ThreadPoolExecutor.new({
|
101
|
+
:min_threads => 1,
|
102
|
+
:max_threads => @upload_workers_count,
|
103
|
+
:max_queue => @upload_queue_size,
|
104
|
+
:fallback_policy => :caller_runs
|
105
|
+
})
|
106
|
+
|
107
|
+
@qs_bucket = get_bucket
|
108
|
+
QingstorValidator.bucket_valid?(@qs_bucket)
|
109
|
+
|
110
|
+
@uploader = Uploader.new(@qs_bucket, @logger, executor)
|
111
|
+
|
112
|
+
start_periodic_check if @rotation.needs_periodic?
|
113
|
+
|
114
|
+
restore_from_crash if @restore
|
115
|
+
end # def register
|
116
|
+
|
117
|
+
public
|
118
|
+
def multi_receive_encoded(events_and_encoded)
|
119
|
+
prefix_written_to = Set.new
|
120
|
+
|
121
|
+
events_and_encoded.each do |event, encoded|
|
122
|
+
prefix_key = event.sprintf(@prefix)
|
123
|
+
prefix_written_to << prefix_key
|
124
|
+
|
125
|
+
begin
|
126
|
+
@file_repository.get_file(prefix_key) { |file| file.write(encoded) }
|
127
|
+
rescue Errno::ENOSPC => e
|
128
|
+
@logger.error("QingStor: Nospace left in temporary directory", :tmpdir => @tmpdir)
|
129
|
+
raise e
|
130
|
+
end
|
131
|
+
end # end of each method
|
132
|
+
|
133
|
+
# check the file after file writing
|
134
|
+
# Groups IO calls to optimize fstat checks
|
135
|
+
rotate_if_needed(prefix_written_to)
|
136
|
+
end # def multi_receive_encoded
|
137
|
+
|
138
|
+
def rotation_strategy
|
139
|
+
case @rotation_strategy
|
140
|
+
when "size"
|
141
|
+
SizeRotationPolicy.new(@size_file)
|
142
|
+
when "time"
|
143
|
+
TimeRotationPolicy.new(@time_file)
|
144
|
+
when "size_and_time"
|
145
|
+
SizeAndTimeRotationPolicy.new(@size_file, @time_file)
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
149
|
+
def rotate_if_needed(prefixs)
|
150
|
+
prefixs.each do |prefix|
|
151
|
+
@file_repository.get_factory(prefix) do |factory|
|
152
|
+
tmp_file = factory.current
|
153
|
+
|
154
|
+
if @rotation.rotate?(tmp_file)
|
155
|
+
@logger.debug("Rotate file",
|
156
|
+
:strategy => tmp_file.key,
|
157
|
+
:path => tmp_file.path)
|
158
|
+
upload_file(tmp_file)
|
159
|
+
factory.rotate!
|
160
|
+
end
|
161
|
+
end
|
162
|
+
end
|
163
|
+
end # def rotate_if_needed
|
164
|
+
|
165
|
+
def upload_file(file)
|
166
|
+
@logger.debug("Add file to uploading queue", :key => file.key)
|
167
|
+
file.close
|
168
|
+
@logger.debug("upload options", :upload_options => upload_options)
|
169
|
+
@uploader.upload_async(file,
|
170
|
+
:on_complete => method(:clean_temporary_file),
|
171
|
+
:upload_options => upload_options)
|
172
|
+
end
|
173
|
+
|
174
|
+
def get_bucket
|
175
|
+
@qs_config = QingStor::SDK::Config.init @access_key_id, @secret_access_key
|
176
|
+
@qs_service = QingStor::SDK::Service.new @qs_config
|
177
|
+
@qs_service.bucket @bucket, @region
|
178
|
+
end
|
179
|
+
|
180
|
+
def close
|
181
|
+
stop_periodic_check if @rotation.needs_periodic?
|
182
|
+
|
183
|
+
@logger.debug("uploading current workspace")
|
184
|
+
@file_repository.each_files do |file|
|
185
|
+
upload_file(file)
|
186
|
+
end
|
187
|
+
|
188
|
+
@file_repository.shutdown
|
189
|
+
|
190
|
+
@uploader.stop
|
191
|
+
|
192
|
+
@crash_uploader.stop if @restore
|
193
|
+
end
|
194
|
+
|
195
|
+
def upload_options
|
196
|
+
options = {
|
197
|
+
:content_encoding => @encoding == "gzip" ? "gzip" : nil
|
198
|
+
}
|
199
|
+
|
200
|
+
if @server_side_encryption_algorithm == "AES256" && !@customer_key.nil?
|
201
|
+
options.merge!({
|
202
|
+
:server_side_encryption_algorithm => @server_side_encryption_algorithm,
|
203
|
+
:customer_key => @customer_key
|
204
|
+
})
|
205
|
+
end
|
206
|
+
|
207
|
+
options
|
208
|
+
end
|
209
|
+
|
210
|
+
def clean_temporary_file(file)
|
211
|
+
@logger.debug("Removing temporary file", :file => file.path)
|
212
|
+
file.delete!
|
213
|
+
end
|
214
|
+
|
215
|
+
def start_periodic_check
|
216
|
+
@logger.debug("Start periodic rotation check")
|
217
|
+
|
218
|
+
@periodic_check = Concurrent::TimerTask.new(:execution_interval => PERIODIC_CHECK_INTERVAL_IN_SECONDS) do
|
219
|
+
@logger.debug("Periodic check for stale files")
|
220
|
+
|
221
|
+
rotate_if_needed(@file_repository.keys)
|
222
|
+
end
|
223
|
+
|
224
|
+
@periodic_check.execute
|
225
|
+
end
|
226
|
+
|
227
|
+
def stop_periodic_check
|
228
|
+
@periodic_check.shutdown
|
229
|
+
end
|
230
|
+
|
231
|
+
def directory_valid?(path)
|
232
|
+
begin
|
233
|
+
FileUtils.mkdir_p(path) unless Dir.exist?(path)
|
234
|
+
::File.writable?(path)
|
235
|
+
rescue
|
236
|
+
false
|
237
|
+
end
|
238
|
+
end
|
239
|
+
|
240
|
+
def restore_from_crash
|
241
|
+
@crash_uploader = Uploader.new(@qs_bucket, @logger, CRASH_RECOVERY_THREADPOOL)
|
242
|
+
|
243
|
+
temp_folder_path = Pathname.new(@tmpdir)
|
244
|
+
Dir.glob(::File.join(@tmpdir, "**/*"))
|
245
|
+
.select { |file| ::File.file?(file) }
|
246
|
+
.each do |file|
|
247
|
+
temp_file = TemporaryFile.create_from_existing_file(file, temp_folder_path)
|
248
|
+
@logger.debug("Recoving from crash and uploading", :file => temp_file.path)
|
249
|
+
@crash_uploader.upload_async(temp_file, :on_complete => method(:clean_temporary_file),
|
250
|
+
:upload_options => upload_options)
|
251
|
+
end
|
252
|
+
end
|
253
|
+
end # class LogStash::Outputs::Qingstor
|
@@ -0,0 +1,28 @@
|
|
1
|
+
Gem::Specification.new do |s|
|
2
|
+
s.name = 'logstash-output-qingstor'
|
3
|
+
s.version = '0.1.0'
|
4
|
+
s.licenses = ['Apache License (2.0)']
|
5
|
+
s.summary = 'logstash output plugin for qingstor'
|
6
|
+
s.description = 'Put the outcomes of logstash into qingstor'
|
7
|
+
s.homepage = 'https://github.com/tacinight/logstash-output-qingstor'
|
8
|
+
s.authors = ['Evan Zhao']
|
9
|
+
s.email = 'tacingiht@gmail.com'
|
10
|
+
s.require_paths = ['lib']
|
11
|
+
|
12
|
+
# Files
|
13
|
+
s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
|
14
|
+
# Tests
|
15
|
+
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
16
|
+
|
17
|
+
# Special flag to let us know this is actually a logstash plugin
|
18
|
+
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
|
19
|
+
|
20
|
+
# Gem dependencies
|
21
|
+
s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
|
22
|
+
s.add_runtime_dependency "logstash-codec-plain"
|
23
|
+
s.add_runtime_dependency "qingstor-sdk", "=1.9.2"
|
24
|
+
s.add_runtime_dependency "concurrent-ruby"
|
25
|
+
|
26
|
+
s.add_development_dependency "stud", "~> 0.0.22"
|
27
|
+
s.add_development_dependency "logstash-devutils"
|
28
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require "logstash/devutils/rspec/spec_helper"
|
2
|
+
require "logstash/outputs/qingstor/temporary_file"
|
3
|
+
require "logstash/outputs/qingstor/temporary_file_factory"
|
4
|
+
require "logstash/outputs/qingstor/file_repository"
|
5
|
+
require "tmpdir"
|
6
|
+
|
7
|
+
describe LogStash::Outputs::Qingstor::FileRepository do
|
8
|
+
let(:tags) { ["tag1", "tag2", "tag3"]}
|
9
|
+
let(:encoding) { "none" }
|
10
|
+
let(:tmpdir) { File.join(Dir.tmpdir, "lg2qs") }
|
11
|
+
let(:prefix) { "aprefix" }
|
12
|
+
|
13
|
+
subject { described_class.new(tags, encoding, tmpdir) }
|
14
|
+
|
15
|
+
it "can get current file io" do
|
16
|
+
subject.get_file(prefix) do |file|
|
17
|
+
expect(file).to be_kind_of(LogStash::Outputs::Qingstor::TemporaryFile)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
it "can get current file factory" do
|
22
|
+
subject.get_factory(prefix) do |factory|
|
23
|
+
expect(factory).to be_kind_of(LogStash::Outputs::Qingstor::TemporaryFileFactory)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/devutils/rspec/spec_helper"
|
3
|
+
require "logstash/outputs/qingstor/qingstor_validator"
|
4
|
+
require_relative "../qs_access_helper"
|
5
|
+
|
6
|
+
describe LogStash::Outputs::Qingstor::QingstorValidator do
|
7
|
+
let(:normal_prefix) { "super/bucket" }
|
8
|
+
let(:wrong_prefix1) { "/wrong/prefix" }
|
9
|
+
let(:wrong_prefix2) { normal_prefix * 100 }
|
10
|
+
let(:bucket) { qs_bucket_init }
|
11
|
+
|
12
|
+
it "raise error if the prefix is not valid" do
|
13
|
+
expect{ described_class.prefix_valid?(wrong_prefix1) }.to raise_error(LogStash::ConfigurationError)
|
14
|
+
expect{ described_class.prefix_valid?(wrong_prefix2) }.to raise_error(LogStash::ConfigurationError)
|
15
|
+
end
|
16
|
+
|
17
|
+
it "return true if the prefix is valid" do
|
18
|
+
expect(described_class.prefix_valid?(normal_prefix)).to be_truthy
|
19
|
+
end
|
20
|
+
|
21
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/devutils/rspec/spec_helper"
|
3
|
+
require "logstash/outputs/qingstor/temporary_file"
|
4
|
+
require "logstash/outputs/qingstor/size_and_time_rotation_policy"
|
5
|
+
|
6
|
+
describe LogStash::Outputs::Qingstor::SizeAndTimeRotationPolicy do
|
7
|
+
let(:size_file) { 1024 * 2 }
|
8
|
+
let(:time_file) { 2 }
|
9
|
+
let(:name) { "foobar" }
|
10
|
+
let(:tmp_file) { Stud::Temporary.file }
|
11
|
+
let(:tmp_dir) { tmp_file.path }
|
12
|
+
let(:file) { LogStash::Outputs::Qingstor::TemporaryFile.new(name, tmp_file, tmp_dir) }
|
13
|
+
let(:content) { "May the code be with you" * 100 }
|
14
|
+
subject { described_class.new(size_file, time_file) }
|
15
|
+
|
16
|
+
it "raise error if time_file is no grater then 0" do
|
17
|
+
expect{ described_class.new(0, 0) }.to raise_error(LogStash::ConfigurationError)
|
18
|
+
expect{ described_class.new(-1, 0) }.to raise_error(LogStash::ConfigurationError)
|
19
|
+
expect{ described_class.new(0, -1) }.to raise_error(LogStash::ConfigurationError)
|
20
|
+
expect{ described_class.new(-1, -1) }.to raise_error(LogStash::ConfigurationError)
|
21
|
+
|
22
|
+
end
|
23
|
+
|
24
|
+
it "return false if the file is not old enough" do
|
25
|
+
expect(subject.rotate?(file)).to be_falsey
|
26
|
+
end
|
27
|
+
|
28
|
+
it "return false if the file is old enough with file size 0" do
|
29
|
+
allow(file).to receive(:ctime).and_return(Time.now - (time_file * 2 * 60))
|
30
|
+
expect(subject.rotate?(file)).to be_falsey
|
31
|
+
end
|
32
|
+
|
33
|
+
it "return truth if the file is old enough and non-empty" do
|
34
|
+
file.write(content)
|
35
|
+
file.fsync
|
36
|
+
allow(file).to receive(:ctime).and_return(Time.now - (time_file * 2 * 60))
|
37
|
+
expect(subject.rotate?(file)).to be_truthy
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/devutils/rspec/spec_helper"
|
3
|
+
require "logstash/outputs/qingstor/temporary_file"
|
4
|
+
require "logstash/outputs/qingstor/size_rotation_policy"
|
5
|
+
|
6
|
+
describe LogStash::Outputs::Qingstor::SizeRotationPolicy do
|
7
|
+
let(:size_file) { 1024 * 2 }
|
8
|
+
let(:name) { "foobar" }
|
9
|
+
let(:tmp_file) { Stud::Temporary.file }
|
10
|
+
let(:tmp_dir) { tmp_file.path }
|
11
|
+
let(:file) { LogStash::Outputs::Qingstor::TemporaryFile.new(name, tmp_file, tmp_dir) }
|
12
|
+
let(:content) { "May the code be with you" * 100 }
|
13
|
+
subject { described_class.new(size_file) }
|
14
|
+
|
15
|
+
it "raise error if size_file is no grater then 0" do
|
16
|
+
expect{described_class.new(0)}.to raise_error(LogStash::ConfigurationError)
|
17
|
+
expect{described_class.new(-1)}.to raise_error(LogStash::ConfigurationError)
|
18
|
+
end
|
19
|
+
|
20
|
+
it "return true if the file has a bigger size value then 'size_file'" do
|
21
|
+
file.write(content)
|
22
|
+
file.fsync
|
23
|
+
expect(subject.rotate?(file)).to be_truthy
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,82 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/devutils/rspec/spec_helper"
|
3
|
+
require "logstash/outputs/qingstor/temporary_file"
|
4
|
+
require "logstash/outputs/qingstor/temporary_file_factory"
|
5
|
+
require "fileutils"
|
6
|
+
require "tmpdir"
|
7
|
+
|
8
|
+
describe LogStash::Outputs::Qingstor::TemporaryFileFactory do
|
9
|
+
let(:prefix) { "lg2qs" }
|
10
|
+
let(:tags) { [ ] }
|
11
|
+
let(:tmpdir) { File.join(Dir.tmpdir, "logstash-qs") }
|
12
|
+
|
13
|
+
subject { described_class.new(prefix, tags, encoding, tmpdir) }
|
14
|
+
|
15
|
+
shared_examples "file factory" do
|
16
|
+
it "creates the file on disk" do
|
17
|
+
expect(File.exist?(subject.current.path)).to be_truthy
|
18
|
+
end
|
19
|
+
|
20
|
+
it "create a temporary file when initialized" do
|
21
|
+
expect(subject.current).to be_kind_of(LogStash::Outputs::Qingstor::TemporaryFile)
|
22
|
+
end
|
23
|
+
|
24
|
+
it "create a file in the right format" do
|
25
|
+
expect(subject.current.path).to match(extension)
|
26
|
+
end
|
27
|
+
|
28
|
+
it "allow to rotate the file" do
|
29
|
+
file_path = subject.current.path
|
30
|
+
expect(subject.rotate!.path).not_to eq(file_path)
|
31
|
+
end
|
32
|
+
|
33
|
+
it "increments the part name on rotation" do
|
34
|
+
expect(subject.current.path).to match(/part0/)
|
35
|
+
expect(subject.rotate!.path).to match(/part1/)
|
36
|
+
end
|
37
|
+
|
38
|
+
it "includes the date" do
|
39
|
+
n = Time.now
|
40
|
+
expect(subject.current.path).to include(n.strftime("%Y-%m-%dT"))
|
41
|
+
end
|
42
|
+
|
43
|
+
it "include the file key in the path" do
|
44
|
+
file = subject.current
|
45
|
+
expect(file.path).to match(/#{file.key}/)
|
46
|
+
end
|
47
|
+
|
48
|
+
it "create a unique directory in the temporary directory for each file" do
|
49
|
+
uuid = "hola"
|
50
|
+
expect(SecureRandom).to receive(:uuid).and_return(uuid).twice
|
51
|
+
expect(subject.current.path).to include(uuid)
|
52
|
+
end
|
53
|
+
|
54
|
+
context "with tags supplied" do
|
55
|
+
let(:tags) { ["secret", "service"] }
|
56
|
+
|
57
|
+
it "adds tags to the filename" do
|
58
|
+
expect(subject.current.path).to match(/tag_#{tags.join('.')}.part/)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
context "without tags" do
|
63
|
+
it "doesn't add tags to the filename" do
|
64
|
+
expect(subject.current.path).not_to match(/tag_/)
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
context "when gzip" do
|
70
|
+
let(:encoding) { "gzip" }
|
71
|
+
let(:extension) { /\.log.gz$/ }
|
72
|
+
|
73
|
+
include_examples "file factory"
|
74
|
+
end
|
75
|
+
|
76
|
+
context "when encoding set to `none`" do
|
77
|
+
let(:encoding) { "none" }
|
78
|
+
let(:extension) { /\.log$/ }
|
79
|
+
|
80
|
+
include_examples "file factory"
|
81
|
+
end
|
82
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/devutils/rspec/spec_helper"
|
3
|
+
require "logstash/outputs/qingstor/temporary_file"
|
4
|
+
require "fileutils"
|
5
|
+
require "tmpdir"
|
6
|
+
|
7
|
+
describe LogStash::Outputs::Qingstor::TemporaryFile do
|
8
|
+
let(:key) { "foo.log" }
|
9
|
+
let(:content) { "May the code be with you!" }
|
10
|
+
let(:tmp_path) { File.join(Dir.tmpdir, key) }
|
11
|
+
let(:file_mode) { "w+" }
|
12
|
+
let(:io){ ::File.open(tmp_path, file_mode)}
|
13
|
+
|
14
|
+
after(:all) do
|
15
|
+
FileUtils.rm_r("/tmp/foo.log") if File.exist?("/tmp/foo.log")
|
16
|
+
end
|
17
|
+
|
18
|
+
subject { described_class.new(key, io, tmp_path) }
|
19
|
+
|
20
|
+
it "return the key of the file" do
|
21
|
+
expect(subject.key).to eq(key)
|
22
|
+
end
|
23
|
+
|
24
|
+
it "is writable and readable" do
|
25
|
+
subject.write(content)
|
26
|
+
subject.close
|
27
|
+
expect(File.read(subject.path).strip).to eq(content)
|
28
|
+
end
|
29
|
+
|
30
|
+
it "can return the correct file size" do
|
31
|
+
subject.write(content)
|
32
|
+
subject.close
|
33
|
+
expect(subject.size).to eq(File.size(tmp_path))
|
34
|
+
end
|
35
|
+
|
36
|
+
it "return the tmp_path of the file" do
|
37
|
+
expect(subject.tmp_path).to eq(tmp_path)
|
38
|
+
end
|
39
|
+
|
40
|
+
it "return the creation time" do
|
41
|
+
expect(subject.ctime).to be < Time.now + 1.0
|
42
|
+
end
|
43
|
+
|
44
|
+
it "can delete file correctly" do
|
45
|
+
expect(File.exist?(subject.tmp_path)).to be_truthy
|
46
|
+
subject.delete!
|
47
|
+
expect(File.exist?(subject.tmp_path)).to be_falsey
|
48
|
+
end
|
49
|
+
|
50
|
+
it "return if the file is empty" do
|
51
|
+
expect(subject.empty?).to be_truthy
|
52
|
+
subject.write(content)
|
53
|
+
expect(subject.empty?).to be_falsey
|
54
|
+
end
|
55
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/devutils/rspec/spec_helper"
|
3
|
+
require "logstash/outputs/qingstor/temporary_file"
|
4
|
+
require "logstash/outputs/qingstor/time_rotation_policy"
|
5
|
+
|
6
|
+
describe LogStash::Outputs::Qingstor::TimeRotationPolicy do
|
7
|
+
let(:time_file) { 2 }
|
8
|
+
let(:name) { "foobar" }
|
9
|
+
let(:tmp_file) { Stud::Temporary.file }
|
10
|
+
let(:tmp_dir) { tmp_file.path }
|
11
|
+
let(:file) { LogStash::Outputs::Qingstor::TemporaryFile.new(name, tmp_file, tmp_dir) }
|
12
|
+
let(:content) { "May the code be with you" * 100 }
|
13
|
+
subject { described_class.new(time_file) }
|
14
|
+
|
15
|
+
it "raise error if time_file is no grater then 0" do
|
16
|
+
expect{described_class.new(0)}.to raise_error(LogStash::ConfigurationError)
|
17
|
+
expect{described_class.new(-1)}.to raise_error(LogStash::ConfigurationError)
|
18
|
+
end
|
19
|
+
|
20
|
+
it "return false if the file is not old enough" do
|
21
|
+
expect(subject.rotate?(file)).to be_falsey
|
22
|
+
end
|
23
|
+
|
24
|
+
it "return false if the file is old enough with file size 0" do
|
25
|
+
allow(file).to receive(:ctime).and_return(Time.now - (time_file * 2 * 60))
|
26
|
+
expect(subject.rotate?(file)).to be_falsey
|
27
|
+
end
|
28
|
+
|
29
|
+
it "return truth if the file is old enough and non-empty" do
|
30
|
+
file.write(content)
|
31
|
+
file.fsync
|
32
|
+
allow(file).to receive(:ctime).and_return(Time.now - (time_file * 2 * 60))
|
33
|
+
expect(subject.rotate?(file)).to be_truthy
|
34
|
+
end
|
35
|
+
end
|