logstash-output-qingstor 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/CHANGELOG.md +2 -0
- data/CODE_OF_CONDUCT.md +44 -0
- data/CONTRIBUTING.md +43 -0
- data/CONTRIBUTORS +10 -0
- data/DEVELOPER.md +2 -0
- data/Gemfile +3 -0
- data/LICENSE +11 -0
- data/README.md +181 -0
- data/lib/logstash/outputs/qingstor/file_repository.rb +112 -0
- data/lib/logstash/outputs/qingstor/qingstor_validator.rb +30 -0
- data/lib/logstash/outputs/qingstor/size_and_time_rotation_policy.rb +24 -0
- data/lib/logstash/outputs/qingstor/size_rotation_policy.rb +26 -0
- data/lib/logstash/outputs/qingstor/temporary_file.rb +63 -0
- data/lib/logstash/outputs/qingstor/temporary_file_factory.rb +110 -0
- data/lib/logstash/outputs/qingstor/time_rotation_policy.rb +26 -0
- data/lib/logstash/outputs/qingstor/uploader.rb +64 -0
- data/lib/logstash/outputs/qingstor.rb +253 -0
- data/logstash-output-qingstor.gemspec +28 -0
- data/spec/outputs/qingstor/file_repository_spec.rb +26 -0
- data/spec/outputs/qingstor/qingstor_validator_spec.rb +21 -0
- data/spec/outputs/qingstor/size_and_time_rotation_policy_spec.rb +39 -0
- data/spec/outputs/qingstor/size_rotation_policy_spec.rb +25 -0
- data/spec/outputs/qingstor/temporary_file_factory_spec.rb +82 -0
- data/spec/outputs/qingstor/temporary_file_spec.rb +55 -0
- data/spec/outputs/qingstor/time_rotation_policy_spec.rb +35 -0
- data/spec/outputs/qingstor/uploader_spec.rb +49 -0
- data/spec/outputs/qingstor_spec.rb +50 -0
- data/spec/outputs/qs_access_helper.rb +38 -0
- data/spec/outputs/spec_helper.rb +4 -0
- metadata +169 -0
@@ -0,0 +1,26 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
module LogStash
|
3
|
+
module Outputs
|
4
|
+
class Qingstor
|
5
|
+
class TimeRotationPolicy
|
6
|
+
attr_reader :time_file
|
7
|
+
|
8
|
+
def initialize(time_file)
|
9
|
+
if time_file <= 0
|
10
|
+
raise LogStash::ConfigurationError, "'time_file' need to be greater then 0"
|
11
|
+
end
|
12
|
+
|
13
|
+
@time_file = time_file * 60
|
14
|
+
end
|
15
|
+
|
16
|
+
def rotate?(file)
|
17
|
+
file.size > 0 && (Time.now - file.ctime) >= time_file
|
18
|
+
end
|
19
|
+
|
20
|
+
def needs_periodic?
|
21
|
+
true
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,64 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "qingstor/sdk"
|
3
|
+
require "concurrent"
|
4
|
+
require "digest/md5"
|
5
|
+
require "base64"
|
6
|
+
module LogStash
|
7
|
+
module Outputs
|
8
|
+
class Qingstor
|
9
|
+
class Uploader
|
10
|
+
TIME_BEFORE_RETRYING_SECONDS = 1
|
11
|
+
DEFAULT_THREADPOOL = Concurrent::ThreadPoolExecutor.new({ :min_thread => 1,
|
12
|
+
:max_thread => 8,
|
13
|
+
:max_queue => 1,
|
14
|
+
:fallback_policy => :caller_runs
|
15
|
+
})
|
16
|
+
attr_reader :bucket, :upload_options, :logger
|
17
|
+
|
18
|
+
def initialize(bucket, logger, threadpool = DEFAULT_THREADPOOL)
|
19
|
+
@bucket = bucket
|
20
|
+
@logger = logger
|
21
|
+
@workers_pool = threadpool
|
22
|
+
end
|
23
|
+
|
24
|
+
def upload_async(file, options = {})
|
25
|
+
@workers_pool.post do
|
26
|
+
upload(file, options)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def upload(file, options = {})
|
31
|
+
upload_options = options.fetch(:upload_options, {})
|
32
|
+
|
33
|
+
file_md5 = Digest::MD5.file(file.path).to_s
|
34
|
+
|
35
|
+
upload_headers = {
|
36
|
+
"content_md5" => file_md5,
|
37
|
+
"body" => ::File.open(file.path)
|
38
|
+
}
|
39
|
+
|
40
|
+
if !upload_options[:server_side_encryption_algorithm].nil?
|
41
|
+
base64_key = Base64.strict_encode64 upload_options[:customer_key]
|
42
|
+
key_md5 = Digest::MD5.hexdigest upload_options[:customer_key]
|
43
|
+
base64_key_md5 = Base64.strict_encode64 key_md5
|
44
|
+
upload_headers.merge!({
|
45
|
+
"x_qs_encryption_customer_algorithm" => upload_options[:server_side_encryption_algorithm],
|
46
|
+
"x_qs_encryption_customer_key" => base64_key,
|
47
|
+
"x_qs_encryption_customer_key_md5" => base64_key_md5,
|
48
|
+
})
|
49
|
+
end
|
50
|
+
@logger.debug("uploading file", :file => file.key)
|
51
|
+
bucket.put_object(file.key, upload_headers)
|
52
|
+
|
53
|
+
options[:on_complete].call(file) unless options[:on_complete].nil?
|
54
|
+
end
|
55
|
+
|
56
|
+
def stop
|
57
|
+
@workers_pool.shutdown
|
58
|
+
@workers_pool.wait_for_termination(nil)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
@@ -0,0 +1,253 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash-core"
|
3
|
+
require "logstash/outputs/base"
|
4
|
+
require "logstash/namespace"
|
5
|
+
require "tmpdir"
|
6
|
+
require "qingstor/sdk"
|
7
|
+
require "concurrent"
|
8
|
+
|
9
|
+
class LogStash::Outputs::Qingstor < LogStash::Outputs::Base
|
10
|
+
require "logstash/outputs/qingstor/temporary_file"
|
11
|
+
require "logstash/outputs/qingstor/temporary_file_factory"
|
12
|
+
require "logstash/outputs/qingstor/file_repository"
|
13
|
+
require "logstash/outputs/qingstor/size_rotation_policy"
|
14
|
+
require "logstash/outputs/qingstor/time_rotation_policy"
|
15
|
+
require "logstash/outputs/qingstor/size_and_time_rotation_policy"
|
16
|
+
require "logstash/outputs/qingstor/uploader"
|
17
|
+
require "logstash/outputs/qingstor/qingstor_validator"
|
18
|
+
|
19
|
+
PERIODIC_CHECK_INTERVAL_IN_SECONDS = 15
|
20
|
+
CRASH_RECOVERY_THREADPOOL = Concurrent::ThreadPoolExecutor.new({
|
21
|
+
:min_threads => 1,
|
22
|
+
:max_threads => 2,
|
23
|
+
:fallback_policy => :caller_runs
|
24
|
+
})
|
25
|
+
config_name "qingstor"
|
26
|
+
|
27
|
+
# When configured as :single a single instance of the Output will be shared among the
|
28
|
+
# pipeline worker threads. Access to the `#multi_receive/#multi_receive_encoded/#receive` method will be synchronized
|
29
|
+
# i.e. only one thread will be active at a time making threadsafety much simpler.
|
30
|
+
#
|
31
|
+
# You can set this to :shared if your output is threadsafe. This will maximize
|
32
|
+
# concurrency but you will need to make appropriate uses of mutexes in `#multi_receive/#receive`.
|
33
|
+
#
|
34
|
+
# Only the `#multi_receive/#multi_receive_encoded` methods need to actually be threadsafe, the other methods
|
35
|
+
# will only be executed in a single thread
|
36
|
+
concurrency :shared
|
37
|
+
|
38
|
+
# The key id to access your QingStor
|
39
|
+
config :access_key_id, :validate => :string, :required => true
|
40
|
+
|
41
|
+
# The key to access your QingStor
|
42
|
+
config :secret_access_key, :validate => :string, :required => true
|
43
|
+
|
44
|
+
# The name of the qingstor bucket
|
45
|
+
config :bucket, :validate => :string, :required => true
|
46
|
+
|
47
|
+
# The region of the QingStor bucket
|
48
|
+
config :region, :validate => ["pek3a", "sh1a"], :default => "pek3a"
|
49
|
+
|
50
|
+
# The prefix of filenames which will work as directory in qingstor
|
51
|
+
config :prefix, :validate => :string, :default => ''
|
52
|
+
|
53
|
+
# Set the directory where logstash store the tmp files before
|
54
|
+
# sending it to qingstor, default directory in linux /tmp/logstash2qingstor
|
55
|
+
config :tmpdir, :validate => :string, :default => File.join(Dir.tmpdir, "logstash2qingstor")
|
56
|
+
|
57
|
+
# Define tags to append to the file on the qingstor bucket
|
58
|
+
config :tags, :validate => :array, :default => []
|
59
|
+
|
60
|
+
# Specify the content encoding. Supports ("gzip"), defaults to "none"
|
61
|
+
config :encoding, :validate => ["gzip", "none"], :default => "none"
|
62
|
+
|
63
|
+
# Define the strategy to use to decide when we need to rotate the file and push it to S3,
|
64
|
+
# The default strategy is to check for both size and time, the first one to match will rotate the file.
|
65
|
+
config :rotation_strategy, :validate => ["size_and_time", "size", "time"], :default => "size_and_time"
|
66
|
+
|
67
|
+
# Define the size requirement for each file to upload to qingstor. In byte.
|
68
|
+
config :size_file, :validate => :number, :default => 1024 * 1024 * 5
|
69
|
+
|
70
|
+
# Define the time interval for each file to upload to qingstor. In minutes.
|
71
|
+
config :time_file, :validate => :number, :default => 15
|
72
|
+
|
73
|
+
# Specify maximum number of workers to use to upload the files to Qingstor
|
74
|
+
config :upload_workers_count, :validate => :number, :default => (Concurrent.processor_count * 0.5).ceil
|
75
|
+
|
76
|
+
# Number of items we can keep in the local queue before uploading them
|
77
|
+
config :upload_queue_size, :validate => :number, :default => 2 * (Concurrent.processor_count * 0.25).ceil
|
78
|
+
|
79
|
+
# Specifies what type of encryption to use when SSE is enabled.
|
80
|
+
config :server_side_encryption_algorithm, :validate => ["AES256", "none"], :default => "none"
|
81
|
+
|
82
|
+
# Specifies the encryption customer key that would be used in server side
|
83
|
+
config :customer_key, :validate => :string
|
84
|
+
|
85
|
+
# Specifies if set to true, it would upload existing file in targeting folder at the beginning.
|
86
|
+
config :restore, :validate => :boolean, :default => false
|
87
|
+
|
88
|
+
public
|
89
|
+
def register
|
90
|
+
QingstorValidator.prefix_valid?(@prefix) unless @prefix.nil?
|
91
|
+
|
92
|
+
if !directory_valid?(@tmpdir)
|
93
|
+
raise LogStash::ConfigurationError, "Logstash must have the permissions to write to the temporary directory: #{@tmpdir}"
|
94
|
+
end
|
95
|
+
|
96
|
+
@file_repository = FileRepository.new(@tags, @encoding, @tmpdir)
|
97
|
+
|
98
|
+
@rotation = rotation_strategy
|
99
|
+
|
100
|
+
executor = Concurrent::ThreadPoolExecutor.new({
|
101
|
+
:min_threads => 1,
|
102
|
+
:max_threads => @upload_workers_count,
|
103
|
+
:max_queue => @upload_queue_size,
|
104
|
+
:fallback_policy => :caller_runs
|
105
|
+
})
|
106
|
+
|
107
|
+
@qs_bucket = get_bucket
|
108
|
+
QingstorValidator.bucket_valid?(@qs_bucket)
|
109
|
+
|
110
|
+
@uploader = Uploader.new(@qs_bucket, @logger, executor)
|
111
|
+
|
112
|
+
start_periodic_check if @rotation.needs_periodic?
|
113
|
+
|
114
|
+
restore_from_crash if @restore
|
115
|
+
end # def register
|
116
|
+
|
117
|
+
public
|
118
|
+
def multi_receive_encoded(events_and_encoded)
|
119
|
+
prefix_written_to = Set.new
|
120
|
+
|
121
|
+
events_and_encoded.each do |event, encoded|
|
122
|
+
prefix_key = event.sprintf(@prefix)
|
123
|
+
prefix_written_to << prefix_key
|
124
|
+
|
125
|
+
begin
|
126
|
+
@file_repository.get_file(prefix_key) { |file| file.write(encoded) }
|
127
|
+
rescue Errno::ENOSPC => e
|
128
|
+
@logger.error("QingStor: Nospace left in temporary directory", :tmpdir => @tmpdir)
|
129
|
+
raise e
|
130
|
+
end
|
131
|
+
end # end of each method
|
132
|
+
|
133
|
+
# check the file after file writing
|
134
|
+
# Groups IO calls to optimize fstat checks
|
135
|
+
rotate_if_needed(prefix_written_to)
|
136
|
+
end # def multi_receive_encoded
|
137
|
+
|
138
|
+
def rotation_strategy
|
139
|
+
case @rotation_strategy
|
140
|
+
when "size"
|
141
|
+
SizeRotationPolicy.new(@size_file)
|
142
|
+
when "time"
|
143
|
+
TimeRotationPolicy.new(@time_file)
|
144
|
+
when "size_and_time"
|
145
|
+
SizeAndTimeRotationPolicy.new(@size_file, @time_file)
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
149
|
+
def rotate_if_needed(prefixs)
|
150
|
+
prefixs.each do |prefix|
|
151
|
+
@file_repository.get_factory(prefix) do |factory|
|
152
|
+
tmp_file = factory.current
|
153
|
+
|
154
|
+
if @rotation.rotate?(tmp_file)
|
155
|
+
@logger.debug("Rotate file",
|
156
|
+
:strategy => tmp_file.key,
|
157
|
+
:path => tmp_file.path)
|
158
|
+
upload_file(tmp_file)
|
159
|
+
factory.rotate!
|
160
|
+
end
|
161
|
+
end
|
162
|
+
end
|
163
|
+
end # def rotate_if_needed
|
164
|
+
|
165
|
+
def upload_file(file)
|
166
|
+
@logger.debug("Add file to uploading queue", :key => file.key)
|
167
|
+
file.close
|
168
|
+
@logger.debug("upload options", :upload_options => upload_options)
|
169
|
+
@uploader.upload_async(file,
|
170
|
+
:on_complete => method(:clean_temporary_file),
|
171
|
+
:upload_options => upload_options)
|
172
|
+
end
|
173
|
+
|
174
|
+
def get_bucket
|
175
|
+
@qs_config = QingStor::SDK::Config.init @access_key_id, @secret_access_key
|
176
|
+
@qs_service = QingStor::SDK::Service.new @qs_config
|
177
|
+
@qs_service.bucket @bucket, @region
|
178
|
+
end
|
179
|
+
|
180
|
+
def close
|
181
|
+
stop_periodic_check if @rotation.needs_periodic?
|
182
|
+
|
183
|
+
@logger.debug("uploading current workspace")
|
184
|
+
@file_repository.each_files do |file|
|
185
|
+
upload_file(file)
|
186
|
+
end
|
187
|
+
|
188
|
+
@file_repository.shutdown
|
189
|
+
|
190
|
+
@uploader.stop
|
191
|
+
|
192
|
+
@crash_uploader.stop if @restore
|
193
|
+
end
|
194
|
+
|
195
|
+
def upload_options
|
196
|
+
options = {
|
197
|
+
:content_encoding => @encoding == "gzip" ? "gzip" : nil
|
198
|
+
}
|
199
|
+
|
200
|
+
if @server_side_encryption_algorithm == "AES256" && !@customer_key.nil?
|
201
|
+
options.merge!({
|
202
|
+
:server_side_encryption_algorithm => @server_side_encryption_algorithm,
|
203
|
+
:customer_key => @customer_key
|
204
|
+
})
|
205
|
+
end
|
206
|
+
|
207
|
+
options
|
208
|
+
end
|
209
|
+
|
210
|
+
def clean_temporary_file(file)
|
211
|
+
@logger.debug("Removing temporary file", :file => file.path)
|
212
|
+
file.delete!
|
213
|
+
end
|
214
|
+
|
215
|
+
def start_periodic_check
|
216
|
+
@logger.debug("Start periodic rotation check")
|
217
|
+
|
218
|
+
@periodic_check = Concurrent::TimerTask.new(:execution_interval => PERIODIC_CHECK_INTERVAL_IN_SECONDS) do
|
219
|
+
@logger.debug("Periodic check for stale files")
|
220
|
+
|
221
|
+
rotate_if_needed(@file_repository.keys)
|
222
|
+
end
|
223
|
+
|
224
|
+
@periodic_check.execute
|
225
|
+
end
|
226
|
+
|
227
|
+
def stop_periodic_check
|
228
|
+
@periodic_check.shutdown
|
229
|
+
end
|
230
|
+
|
231
|
+
def directory_valid?(path)
|
232
|
+
begin
|
233
|
+
FileUtils.mkdir_p(path) unless Dir.exist?(path)
|
234
|
+
::File.writable?(path)
|
235
|
+
rescue
|
236
|
+
false
|
237
|
+
end
|
238
|
+
end
|
239
|
+
|
240
|
+
def restore_from_crash
|
241
|
+
@crash_uploader = Uploader.new(@qs_bucket, @logger, CRASH_RECOVERY_THREADPOOL)
|
242
|
+
|
243
|
+
temp_folder_path = Pathname.new(@tmpdir)
|
244
|
+
Dir.glob(::File.join(@tmpdir, "**/*"))
|
245
|
+
.select { |file| ::File.file?(file) }
|
246
|
+
.each do |file|
|
247
|
+
temp_file = TemporaryFile.create_from_existing_file(file, temp_folder_path)
|
248
|
+
@logger.debug("Recoving from crash and uploading", :file => temp_file.path)
|
249
|
+
@crash_uploader.upload_async(temp_file, :on_complete => method(:clean_temporary_file),
|
250
|
+
:upload_options => upload_options)
|
251
|
+
end
|
252
|
+
end
|
253
|
+
end # class LogStash::Outputs::Qingstor
|
@@ -0,0 +1,28 @@
|
|
1
|
+
Gem::Specification.new do |s|
|
2
|
+
s.name = 'logstash-output-qingstor'
|
3
|
+
s.version = '0.1.0'
|
4
|
+
s.licenses = ['Apache License (2.0)']
|
5
|
+
s.summary = 'logstash output plugin for qingstor'
|
6
|
+
s.description = 'Put the outcomes of logstash into qingstor'
|
7
|
+
s.homepage = 'https://github.com/tacinight/logstash-output-qingstor'
|
8
|
+
s.authors = ['Evan Zhao']
|
9
|
+
s.email = 'tacingiht@gmail.com'
|
10
|
+
s.require_paths = ['lib']
|
11
|
+
|
12
|
+
# Files
|
13
|
+
s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
|
14
|
+
# Tests
|
15
|
+
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
16
|
+
|
17
|
+
# Special flag to let us know this is actually a logstash plugin
|
18
|
+
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
|
19
|
+
|
20
|
+
# Gem dependencies
|
21
|
+
s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
|
22
|
+
s.add_runtime_dependency "logstash-codec-plain"
|
23
|
+
s.add_runtime_dependency "qingstor-sdk", "=1.9.2"
|
24
|
+
s.add_runtime_dependency "concurrent-ruby"
|
25
|
+
|
26
|
+
s.add_development_dependency "stud", "~> 0.0.22"
|
27
|
+
s.add_development_dependency "logstash-devutils"
|
28
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require "logstash/devutils/rspec/spec_helper"
|
2
|
+
require "logstash/outputs/qingstor/temporary_file"
|
3
|
+
require "logstash/outputs/qingstor/temporary_file_factory"
|
4
|
+
require "logstash/outputs/qingstor/file_repository"
|
5
|
+
require "tmpdir"
|
6
|
+
|
7
|
+
describe LogStash::Outputs::Qingstor::FileRepository do
|
8
|
+
let(:tags) { ["tag1", "tag2", "tag3"]}
|
9
|
+
let(:encoding) { "none" }
|
10
|
+
let(:tmpdir) { File.join(Dir.tmpdir, "lg2qs") }
|
11
|
+
let(:prefix) { "aprefix" }
|
12
|
+
|
13
|
+
subject { described_class.new(tags, encoding, tmpdir) }
|
14
|
+
|
15
|
+
it "can get current file io" do
|
16
|
+
subject.get_file(prefix) do |file|
|
17
|
+
expect(file).to be_kind_of(LogStash::Outputs::Qingstor::TemporaryFile)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
it "can get current file factory" do
|
22
|
+
subject.get_factory(prefix) do |factory|
|
23
|
+
expect(factory).to be_kind_of(LogStash::Outputs::Qingstor::TemporaryFileFactory)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/devutils/rspec/spec_helper"
|
3
|
+
require "logstash/outputs/qingstor/qingstor_validator"
|
4
|
+
require_relative "../qs_access_helper"
|
5
|
+
|
6
|
+
describe LogStash::Outputs::Qingstor::QingstorValidator do
|
7
|
+
let(:normal_prefix) { "super/bucket" }
|
8
|
+
let(:wrong_prefix1) { "/wrong/prefix" }
|
9
|
+
let(:wrong_prefix2) { normal_prefix * 100 }
|
10
|
+
let(:bucket) { qs_bucket_init }
|
11
|
+
|
12
|
+
it "raise error if the prefix is not valid" do
|
13
|
+
expect{ described_class.prefix_valid?(wrong_prefix1) }.to raise_error(LogStash::ConfigurationError)
|
14
|
+
expect{ described_class.prefix_valid?(wrong_prefix2) }.to raise_error(LogStash::ConfigurationError)
|
15
|
+
end
|
16
|
+
|
17
|
+
it "return true if the prefix is valid" do
|
18
|
+
expect(described_class.prefix_valid?(normal_prefix)).to be_truthy
|
19
|
+
end
|
20
|
+
|
21
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/devutils/rspec/spec_helper"
|
3
|
+
require "logstash/outputs/qingstor/temporary_file"
|
4
|
+
require "logstash/outputs/qingstor/size_and_time_rotation_policy"
|
5
|
+
|
6
|
+
describe LogStash::Outputs::Qingstor::SizeAndTimeRotationPolicy do
|
7
|
+
let(:size_file) { 1024 * 2 }
|
8
|
+
let(:time_file) { 2 }
|
9
|
+
let(:name) { "foobar" }
|
10
|
+
let(:tmp_file) { Stud::Temporary.file }
|
11
|
+
let(:tmp_dir) { tmp_file.path }
|
12
|
+
let(:file) { LogStash::Outputs::Qingstor::TemporaryFile.new(name, tmp_file, tmp_dir) }
|
13
|
+
let(:content) { "May the code be with you" * 100 }
|
14
|
+
subject { described_class.new(size_file, time_file) }
|
15
|
+
|
16
|
+
it "raise error if time_file is no grater then 0" do
|
17
|
+
expect{ described_class.new(0, 0) }.to raise_error(LogStash::ConfigurationError)
|
18
|
+
expect{ described_class.new(-1, 0) }.to raise_error(LogStash::ConfigurationError)
|
19
|
+
expect{ described_class.new(0, -1) }.to raise_error(LogStash::ConfigurationError)
|
20
|
+
expect{ described_class.new(-1, -1) }.to raise_error(LogStash::ConfigurationError)
|
21
|
+
|
22
|
+
end
|
23
|
+
|
24
|
+
it "return false if the file is not old enough" do
|
25
|
+
expect(subject.rotate?(file)).to be_falsey
|
26
|
+
end
|
27
|
+
|
28
|
+
it "return false if the file is old enough with file size 0" do
|
29
|
+
allow(file).to receive(:ctime).and_return(Time.now - (time_file * 2 * 60))
|
30
|
+
expect(subject.rotate?(file)).to be_falsey
|
31
|
+
end
|
32
|
+
|
33
|
+
it "return truth if the file is old enough and non-empty" do
|
34
|
+
file.write(content)
|
35
|
+
file.fsync
|
36
|
+
allow(file).to receive(:ctime).and_return(Time.now - (time_file * 2 * 60))
|
37
|
+
expect(subject.rotate?(file)).to be_truthy
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/devutils/rspec/spec_helper"
|
3
|
+
require "logstash/outputs/qingstor/temporary_file"
|
4
|
+
require "logstash/outputs/qingstor/size_rotation_policy"
|
5
|
+
|
6
|
+
describe LogStash::Outputs::Qingstor::SizeRotationPolicy do
|
7
|
+
let(:size_file) { 1024 * 2 }
|
8
|
+
let(:name) { "foobar" }
|
9
|
+
let(:tmp_file) { Stud::Temporary.file }
|
10
|
+
let(:tmp_dir) { tmp_file.path }
|
11
|
+
let(:file) { LogStash::Outputs::Qingstor::TemporaryFile.new(name, tmp_file, tmp_dir) }
|
12
|
+
let(:content) { "May the code be with you" * 100 }
|
13
|
+
subject { described_class.new(size_file) }
|
14
|
+
|
15
|
+
it "raise error if size_file is no grater then 0" do
|
16
|
+
expect{described_class.new(0)}.to raise_error(LogStash::ConfigurationError)
|
17
|
+
expect{described_class.new(-1)}.to raise_error(LogStash::ConfigurationError)
|
18
|
+
end
|
19
|
+
|
20
|
+
it "return true if the file has a bigger size value then 'size_file'" do
|
21
|
+
file.write(content)
|
22
|
+
file.fsync
|
23
|
+
expect(subject.rotate?(file)).to be_truthy
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,82 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/devutils/rspec/spec_helper"
|
3
|
+
require "logstash/outputs/qingstor/temporary_file"
|
4
|
+
require "logstash/outputs/qingstor/temporary_file_factory"
|
5
|
+
require "fileutils"
|
6
|
+
require "tmpdir"
|
7
|
+
|
8
|
+
describe LogStash::Outputs::Qingstor::TemporaryFileFactory do
|
9
|
+
let(:prefix) { "lg2qs" }
|
10
|
+
let(:tags) { [ ] }
|
11
|
+
let(:tmpdir) { File.join(Dir.tmpdir, "logstash-qs") }
|
12
|
+
|
13
|
+
subject { described_class.new(prefix, tags, encoding, tmpdir) }
|
14
|
+
|
15
|
+
shared_examples "file factory" do
|
16
|
+
it "creates the file on disk" do
|
17
|
+
expect(File.exist?(subject.current.path)).to be_truthy
|
18
|
+
end
|
19
|
+
|
20
|
+
it "create a temporary file when initialized" do
|
21
|
+
expect(subject.current).to be_kind_of(LogStash::Outputs::Qingstor::TemporaryFile)
|
22
|
+
end
|
23
|
+
|
24
|
+
it "create a file in the right format" do
|
25
|
+
expect(subject.current.path).to match(extension)
|
26
|
+
end
|
27
|
+
|
28
|
+
it "allow to rotate the file" do
|
29
|
+
file_path = subject.current.path
|
30
|
+
expect(subject.rotate!.path).not_to eq(file_path)
|
31
|
+
end
|
32
|
+
|
33
|
+
it "increments the part name on rotation" do
|
34
|
+
expect(subject.current.path).to match(/part0/)
|
35
|
+
expect(subject.rotate!.path).to match(/part1/)
|
36
|
+
end
|
37
|
+
|
38
|
+
it "includes the date" do
|
39
|
+
n = Time.now
|
40
|
+
expect(subject.current.path).to include(n.strftime("%Y-%m-%dT"))
|
41
|
+
end
|
42
|
+
|
43
|
+
it "include the file key in the path" do
|
44
|
+
file = subject.current
|
45
|
+
expect(file.path).to match(/#{file.key}/)
|
46
|
+
end
|
47
|
+
|
48
|
+
it "create a unique directory in the temporary directory for each file" do
|
49
|
+
uuid = "hola"
|
50
|
+
expect(SecureRandom).to receive(:uuid).and_return(uuid).twice
|
51
|
+
expect(subject.current.path).to include(uuid)
|
52
|
+
end
|
53
|
+
|
54
|
+
context "with tags supplied" do
|
55
|
+
let(:tags) { ["secret", "service"] }
|
56
|
+
|
57
|
+
it "adds tags to the filename" do
|
58
|
+
expect(subject.current.path).to match(/tag_#{tags.join('.')}.part/)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
context "without tags" do
|
63
|
+
it "doesn't add tags to the filename" do
|
64
|
+
expect(subject.current.path).not_to match(/tag_/)
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
context "when gzip" do
|
70
|
+
let(:encoding) { "gzip" }
|
71
|
+
let(:extension) { /\.log.gz$/ }
|
72
|
+
|
73
|
+
include_examples "file factory"
|
74
|
+
end
|
75
|
+
|
76
|
+
context "when encoding set to `none`" do
|
77
|
+
let(:encoding) { "none" }
|
78
|
+
let(:extension) { /\.log$/ }
|
79
|
+
|
80
|
+
include_examples "file factory"
|
81
|
+
end
|
82
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/devutils/rspec/spec_helper"
|
3
|
+
require "logstash/outputs/qingstor/temporary_file"
|
4
|
+
require "fileutils"
|
5
|
+
require "tmpdir"
|
6
|
+
|
7
|
+
describe LogStash::Outputs::Qingstor::TemporaryFile do
|
8
|
+
let(:key) { "foo.log" }
|
9
|
+
let(:content) { "May the code be with you!" }
|
10
|
+
let(:tmp_path) { File.join(Dir.tmpdir, key) }
|
11
|
+
let(:file_mode) { "w+" }
|
12
|
+
let(:io){ ::File.open(tmp_path, file_mode)}
|
13
|
+
|
14
|
+
after(:all) do
|
15
|
+
FileUtils.rm_r("/tmp/foo.log") if File.exist?("/tmp/foo.log")
|
16
|
+
end
|
17
|
+
|
18
|
+
subject { described_class.new(key, io, tmp_path) }
|
19
|
+
|
20
|
+
it "return the key of the file" do
|
21
|
+
expect(subject.key).to eq(key)
|
22
|
+
end
|
23
|
+
|
24
|
+
it "is writable and readable" do
|
25
|
+
subject.write(content)
|
26
|
+
subject.close
|
27
|
+
expect(File.read(subject.path).strip).to eq(content)
|
28
|
+
end
|
29
|
+
|
30
|
+
it "can return the correct file size" do
|
31
|
+
subject.write(content)
|
32
|
+
subject.close
|
33
|
+
expect(subject.size).to eq(File.size(tmp_path))
|
34
|
+
end
|
35
|
+
|
36
|
+
it "return the tmp_path of the file" do
|
37
|
+
expect(subject.tmp_path).to eq(tmp_path)
|
38
|
+
end
|
39
|
+
|
40
|
+
it "return the creation time" do
|
41
|
+
expect(subject.ctime).to be < Time.now + 1.0
|
42
|
+
end
|
43
|
+
|
44
|
+
it "can delete file correctly" do
|
45
|
+
expect(File.exist?(subject.tmp_path)).to be_truthy
|
46
|
+
subject.delete!
|
47
|
+
expect(File.exist?(subject.tmp_path)).to be_falsey
|
48
|
+
end
|
49
|
+
|
50
|
+
it "return if the file is empty" do
|
51
|
+
expect(subject.empty?).to be_truthy
|
52
|
+
subject.write(content)
|
53
|
+
expect(subject.empty?).to be_falsey
|
54
|
+
end
|
55
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/devutils/rspec/spec_helper"
|
3
|
+
require "logstash/outputs/qingstor/temporary_file"
|
4
|
+
require "logstash/outputs/qingstor/time_rotation_policy"
|
5
|
+
|
6
|
+
describe LogStash::Outputs::Qingstor::TimeRotationPolicy do
|
7
|
+
let(:time_file) { 2 }
|
8
|
+
let(:name) { "foobar" }
|
9
|
+
let(:tmp_file) { Stud::Temporary.file }
|
10
|
+
let(:tmp_dir) { tmp_file.path }
|
11
|
+
let(:file) { LogStash::Outputs::Qingstor::TemporaryFile.new(name, tmp_file, tmp_dir) }
|
12
|
+
let(:content) { "May the code be with you" * 100 }
|
13
|
+
subject { described_class.new(time_file) }
|
14
|
+
|
15
|
+
it "raise error if time_file is no grater then 0" do
|
16
|
+
expect{described_class.new(0)}.to raise_error(LogStash::ConfigurationError)
|
17
|
+
expect{described_class.new(-1)}.to raise_error(LogStash::ConfigurationError)
|
18
|
+
end
|
19
|
+
|
20
|
+
it "return false if the file is not old enough" do
|
21
|
+
expect(subject.rotate?(file)).to be_falsey
|
22
|
+
end
|
23
|
+
|
24
|
+
it "return false if the file is old enough with file size 0" do
|
25
|
+
allow(file).to receive(:ctime).and_return(Time.now - (time_file * 2 * 60))
|
26
|
+
expect(subject.rotate?(file)).to be_falsey
|
27
|
+
end
|
28
|
+
|
29
|
+
it "return truth if the file is old enough and non-empty" do
|
30
|
+
file.write(content)
|
31
|
+
file.fsync
|
32
|
+
allow(file).to receive(:ctime).and_return(Time.now - (time_file * 2 * 60))
|
33
|
+
expect(subject.rotate?(file)).to be_truthy
|
34
|
+
end
|
35
|
+
end
|