logstash-output-qingstor 0.1.3 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +0 -1
- data/README.md +1 -1
- data/lib/logstash/outputs/qingstor/file_repository.rb +64 -49
- data/lib/logstash/outputs/qingstor/qingstor_validator.rb +19 -18
- data/lib/logstash/outputs/qingstor/rotation_policy.rb +35 -28
- data/lib/logstash/outputs/qingstor/temporary_file.rb +38 -36
- data/lib/logstash/outputs/qingstor/temporary_file_factory.rb +61 -57
- data/lib/logstash/outputs/qingstor/uploader.rb +41 -36
- data/lib/logstash/outputs/qingstor.rb +144 -124
- data/logstash-output-qingstor.gemspec +12 -12
- data/spec/outputs/qingstor/file_repository_spec.rb +24 -22
- data/spec/outputs/qingstor/qingstor_validator_spec.rb +14 -12
- data/spec/outputs/qingstor/rotation_policy_spec.rb +58 -43
- data/spec/outputs/qingstor/temporary_file_factory_spec.rb +34 -32
- data/spec/outputs/qingstor/temporary_file_spec.rb +31 -30
- data/spec/outputs/qingstor/uploader_spec.rb +29 -30
- data/spec/outputs/qingstor_spec.rb +53 -44
- data/spec/outputs/qs_access_helper.rb +8 -8
- data/spec/outputs/spec_helper.rb +3 -3
- metadata +3 -5
- data/lib/logstash/outputs/qingstor/size_rotation_policy.rb +0 -26
- data/lib/logstash/outputs/qingstor/time_rotation_policy.rb +0 -26
@@ -1,81 +1,96 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
-
|
3
|
-
require
|
4
|
-
require
|
2
|
+
|
3
|
+
require 'logstash/devutils/rspec/spec_helper'
|
4
|
+
require 'logstash/outputs/qingstor/temporary_file'
|
5
|
+
require 'logstash/outputs/qingstor/rotation_policy'
|
5
6
|
|
6
7
|
describe LogStash::Outputs::Qingstor::RotationPolicy do
|
7
8
|
let(:size_file) { 1024 * 2 }
|
8
9
|
let(:time_file) { 2 }
|
9
|
-
let(:name) {
|
10
|
+
let(:name) { 'foobar' }
|
10
11
|
let(:tmp_file) { Stud::Temporary.file }
|
11
12
|
let(:tmp_dir) { tmp_file.path }
|
12
|
-
let(:file)
|
13
|
-
|
14
|
-
|
13
|
+
let(:file) do
|
14
|
+
LogStash::Outputs::Qingstor::TemporaryFile.new(name, tmp_file, tmp_dir)
|
15
|
+
end
|
16
|
+
let(:content) { 'May the code be with you' * 100 }
|
15
17
|
|
16
|
-
context
|
17
|
-
subject { described_class.new(
|
18
|
+
context 'when size_and_time policy' do
|
19
|
+
subject { described_class.new('size_and_time', size_file, time_file) }
|
18
20
|
|
19
|
-
it
|
20
|
-
expect{ described_class.new(
|
21
|
-
|
22
|
-
expect{ described_class.new(
|
23
|
-
|
24
|
-
|
25
|
-
|
21
|
+
it 'raise error if time_file is no grater then 0' do
|
22
|
+
expect { described_class.new('size_and_time', 0, 0) }
|
23
|
+
.to raise_error(LogStash::ConfigurationError)
|
24
|
+
expect { described_class.new('size_and_time', -1, 0) }
|
25
|
+
.to raise_error(LogStash::ConfigurationError)
|
26
|
+
expect { described_class.new('size_and_time', 0, -1) }
|
27
|
+
.to raise_error(LogStash::ConfigurationError)
|
28
|
+
expect { described_class.new('size_and_time', -1, -1) }
|
29
|
+
.to raise_error(LogStash::ConfigurationError)
|
30
|
+
end
|
26
31
|
|
27
|
-
it
|
32
|
+
it 'return false if the file is not old enough' do
|
28
33
|
expect(subject.rotate?(file)).to be_falsey
|
29
34
|
end
|
30
35
|
|
31
|
-
it
|
32
|
-
allow(file)
|
36
|
+
it 'return false if the file is old enough with file size 0' do
|
37
|
+
allow(file)
|
38
|
+
.to receive(:ctime).and_return(Time.now - (time_file * 2 * 60))
|
33
39
|
expect(subject.rotate?(file)).to be_falsey
|
34
40
|
end
|
35
41
|
|
36
|
-
it
|
42
|
+
it 'return truth if the file is old enough and non-empty' do
|
37
43
|
file.write(content)
|
38
44
|
file.fsync
|
39
|
-
allow(file)
|
45
|
+
allow(file)
|
46
|
+
.to receive(:ctime).and_return(Time.now - (time_file * 2 * 60))
|
40
47
|
expect(subject.rotate?(file)).to be_truthy
|
41
48
|
end
|
42
|
-
end
|
49
|
+
end
|
50
|
+
|
51
|
+
context 'when size policy' do
|
52
|
+
subject { described_class.new('size', size_file, time_file) }
|
43
53
|
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
expect{described_class.new(
|
48
|
-
|
49
|
-
end
|
54
|
+
it 'raise error if size_file is no grater then 0' do
|
55
|
+
expect { described_class.new('size', 0, 0) }
|
56
|
+
.to raise_error(LogStash::ConfigurationError)
|
57
|
+
expect { described_class.new('size', -1, 0) }
|
58
|
+
.to raise_error(LogStash::ConfigurationError)
|
59
|
+
end
|
50
60
|
|
51
|
-
it "return true if the file has a bigger size value then 'size_file'" do
|
61
|
+
it "return true if the file has a bigger size value then 'size_file'" do
|
52
62
|
file.write(content)
|
53
63
|
file.fsync
|
54
64
|
expect(subject.rotate?(file)).to be_truthy
|
55
65
|
end
|
56
|
-
end
|
66
|
+
end
|
67
|
+
|
68
|
+
context 'when time policy' do
|
69
|
+
subject { described_class.new('time', size_file, time_file) }
|
57
70
|
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
expect{described_class.new(
|
62
|
-
|
63
|
-
end
|
71
|
+
it 'raise error if time_file is no grater then 0' do
|
72
|
+
expect { described_class.new('time', 0, 0) }
|
73
|
+
.to raise_error(LogStash::ConfigurationError)
|
74
|
+
expect { described_class.new('time', 0, -1) }
|
75
|
+
.to raise_error(LogStash::ConfigurationError)
|
76
|
+
end
|
64
77
|
|
65
|
-
it
|
78
|
+
it 'return false if the file is not old enough' do
|
66
79
|
expect(subject.rotate?(file)).to be_falsey
|
67
80
|
end
|
68
81
|
|
69
|
-
it
|
70
|
-
allow(file)
|
82
|
+
it 'return false if the file is old enough with file size 0' do
|
83
|
+
allow(file)
|
84
|
+
.to receive(:ctime).and_return(Time.now - (time_file * 2 * 60))
|
71
85
|
expect(subject.rotate?(file)).to be_falsey
|
72
86
|
end
|
73
87
|
|
74
|
-
it
|
88
|
+
it 'return truth if the file is old enough and non-empty' do
|
75
89
|
file.write(content)
|
76
90
|
file.fsync
|
77
|
-
allow(file)
|
91
|
+
allow(file)
|
92
|
+
.to receive(:ctime).and_return(Time.now - (time_file * 2 * 60))
|
78
93
|
expect(subject.rotate?(file)).to be_truthy
|
79
94
|
end
|
80
|
-
end
|
81
|
-
end
|
95
|
+
end
|
96
|
+
end
|
@@ -1,82 +1,84 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
-
require "logstash/devutils/rspec/spec_helper"
|
3
|
-
require "logstash/outputs/qingstor/temporary_file"
|
4
|
-
require "logstash/outputs/qingstor/temporary_file_factory"
|
5
|
-
require "fileutils"
|
6
|
-
require "tmpdir"
|
7
2
|
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
3
|
+
require 'logstash/devutils/rspec/spec_helper'
|
4
|
+
require 'logstash/outputs/qingstor/temporary_file'
|
5
|
+
require 'logstash/outputs/qingstor/temporary_file_factory'
|
6
|
+
require 'fileutils'
|
7
|
+
require 'tmpdir'
|
12
8
|
|
9
|
+
describe LogStash::Outputs::Qingstor::TemporaryFileFactory do
|
13
10
|
subject { described_class.new(prefix, tags, encoding, tmpdir) }
|
14
11
|
|
15
|
-
|
16
|
-
|
12
|
+
let(:prefix) { 'lg2qs' }
|
13
|
+
let(:tags) { [] }
|
14
|
+
let(:tmpdir) { File.join(Dir.tmpdir, 'logstash-qs') }
|
15
|
+
|
16
|
+
shared_examples 'file factory' do
|
17
|
+
it 'creates the file on disk' do
|
17
18
|
expect(File.exist?(subject.current.path)).to be_truthy
|
18
19
|
end
|
19
20
|
|
20
|
-
it
|
21
|
-
expect(subject.current)
|
21
|
+
it 'create a temporary file when initialized' do
|
22
|
+
expect(subject.current)
|
23
|
+
.to be_kind_of(LogStash::Outputs::Qingstor::TemporaryFile)
|
22
24
|
end
|
23
25
|
|
24
|
-
it
|
26
|
+
it 'create a file in the right format' do
|
25
27
|
expect(subject.current.path).to match(extension)
|
26
28
|
end
|
27
29
|
|
28
|
-
it
|
30
|
+
it 'allow to rotate the file' do
|
29
31
|
file_path = subject.current.path
|
30
32
|
expect(subject.rotate!.path).not_to eq(file_path)
|
31
33
|
end
|
32
34
|
|
33
|
-
it
|
35
|
+
it 'increments the part name on rotation' do
|
34
36
|
expect(subject.current.path).to match(/part0/)
|
35
37
|
expect(subject.rotate!.path).to match(/part1/)
|
36
38
|
end
|
37
39
|
|
38
|
-
it
|
40
|
+
it 'includes the date' do
|
39
41
|
n = Time.now
|
40
|
-
expect(subject.current.path).to include(n.strftime(
|
42
|
+
expect(subject.current.path).to include(n.strftime('%Y-%m-%dT'))
|
41
43
|
end
|
42
44
|
|
43
|
-
it
|
45
|
+
it 'include the file key in the path' do
|
44
46
|
file = subject.current
|
45
47
|
expect(file.path).to match(/#{file.key}/)
|
46
48
|
end
|
47
49
|
|
48
|
-
it
|
49
|
-
uuid =
|
50
|
+
it 'create a unique directory in the temporary directory for each file' do
|
51
|
+
uuid = 'hola'
|
50
52
|
expect(SecureRandom).to receive(:uuid).and_return(uuid).twice
|
51
53
|
expect(subject.current.path).to include(uuid)
|
52
54
|
end
|
53
55
|
|
54
|
-
context
|
55
|
-
let(:tags) { [
|
56
|
+
context 'with tags supplied' do
|
57
|
+
let(:tags) { %w[secret service] }
|
56
58
|
|
57
|
-
it
|
59
|
+
it 'adds tags to the filename' do
|
58
60
|
expect(subject.current.path).to match(/tag_#{tags.join('.')}.part/)
|
59
61
|
end
|
60
62
|
end
|
61
63
|
|
62
|
-
context
|
64
|
+
context 'without tags' do
|
63
65
|
it "doesn't add tags to the filename" do
|
64
66
|
expect(subject.current.path).not_to match(/tag_/)
|
65
67
|
end
|
66
68
|
end
|
67
69
|
end
|
68
70
|
|
69
|
-
context
|
70
|
-
let(:encoding) {
|
71
|
+
context 'when gzip' do
|
72
|
+
let(:encoding) { 'gzip' }
|
71
73
|
let(:extension) { /\.log.gz$/ }
|
72
74
|
|
73
|
-
include_examples
|
75
|
+
include_examples 'file factory'
|
74
76
|
end
|
75
77
|
|
76
|
-
context
|
77
|
-
let(:encoding) {
|
78
|
+
context 'when encoding set to `none`' do
|
79
|
+
let(:encoding) { 'none' }
|
78
80
|
let(:extension) { /\.log$/ }
|
79
81
|
|
80
|
-
include_examples
|
82
|
+
include_examples 'file factory'
|
81
83
|
end
|
82
|
-
end
|
84
|
+
end
|
@@ -1,55 +1,56 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
-
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
2
|
+
|
3
|
+
require 'logstash/devutils/rspec/spec_helper'
|
4
|
+
require 'logstash/outputs/qingstor/temporary_file'
|
5
|
+
require 'fileutils'
|
6
|
+
require 'tmpdir'
|
6
7
|
|
7
8
|
describe LogStash::Outputs::Qingstor::TemporaryFile do
|
8
|
-
|
9
|
-
let(:content) { "May the code be with you!" }
|
10
|
-
let(:tmp_path) { File.join(Dir.tmpdir, key) }
|
11
|
-
let(:file_mode) { "w+" }
|
12
|
-
let(:io){ ::File.open(tmp_path, file_mode)}
|
9
|
+
subject { described_class.new(key, io, tmp_path) }
|
13
10
|
|
14
|
-
|
15
|
-
|
16
|
-
|
11
|
+
let(:key) { 'foo.log' }
|
12
|
+
let(:content) { 'May the code be with you!' }
|
13
|
+
let(:tmp_path) { File.join(Dir.tmpdir, key) }
|
14
|
+
let(:file_mode) { 'w+' }
|
15
|
+
let(:io) { ::File.open(tmp_path, file_mode) }
|
17
16
|
|
18
|
-
|
17
|
+
after(:all) do
|
18
|
+
FileUtils.rm_r('/tmp/foo.log') if File.exist?('/tmp/foo.log')
|
19
|
+
end
|
19
20
|
|
20
|
-
it
|
21
|
-
expect(subject.key).to eq(key)
|
22
|
-
end
|
21
|
+
it 'return the key of the file' do
|
22
|
+
expect(subject.key).to eq(key)
|
23
|
+
end
|
23
24
|
|
24
|
-
it
|
25
|
+
it 'is writable and readable' do
|
25
26
|
subject.write(content)
|
26
|
-
subject.close
|
27
|
+
subject.close
|
27
28
|
expect(File.read(subject.path).strip).to eq(content)
|
28
|
-
end
|
29
|
+
end
|
29
30
|
|
30
|
-
it
|
31
|
+
it 'can return the correct file size' do
|
31
32
|
subject.write(content)
|
32
|
-
subject.close
|
33
|
+
subject.close
|
33
34
|
expect(subject.size).to eq(File.size(tmp_path))
|
34
|
-
end
|
35
|
+
end
|
35
36
|
|
36
|
-
it
|
37
|
+
it 'return the tmp_path of the file' do
|
37
38
|
expect(subject.tmp_path).to eq(tmp_path)
|
38
|
-
end
|
39
|
+
end
|
39
40
|
|
40
|
-
it
|
41
|
+
it 'return the creation time' do
|
41
42
|
expect(subject.ctime).to be < Time.now + 1.0
|
42
|
-
end
|
43
|
+
end
|
43
44
|
|
44
|
-
it
|
45
|
+
it 'can delete file correctly' do
|
45
46
|
expect(File.exist?(subject.tmp_path)).to be_truthy
|
46
47
|
subject.delete!
|
47
48
|
expect(File.exist?(subject.tmp_path)).to be_falsey
|
48
49
|
end
|
49
50
|
|
50
|
-
it
|
51
|
+
it 'return if the file is empty' do
|
51
52
|
expect(subject.empty?).to be_truthy
|
52
53
|
subject.write(content)
|
53
54
|
expect(subject.empty?).to be_falsey
|
54
|
-
end
|
55
|
-
end
|
55
|
+
end
|
56
|
+
end
|
@@ -1,49 +1,48 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
-
require "logstash/devutils/rspec/spec_helper"
|
3
|
-
require "logstash/outputs/qingstor/uploader"
|
4
|
-
require "logstash/outputs/qingstor/temporary_file"
|
5
|
-
require "qingstor/sdk"
|
6
|
-
require "stud/temporary"
|
7
2
|
|
8
|
-
|
3
|
+
require 'logstash/devutils/rspec/spec_helper'
|
4
|
+
require 'logstash/outputs/qingstor/uploader'
|
5
|
+
require 'logstash/outputs/qingstor/temporary_file'
|
6
|
+
require 'qingstor/sdk'
|
7
|
+
require 'stud/temporary'
|
8
|
+
|
9
|
+
require_relative '../qs_access_helper'
|
10
|
+
|
11
|
+
describe LogStash::Outputs::Qingstor::Uploader do
|
12
|
+
subject { described_class.new(bucket, logger, threadpool) }
|
9
13
|
|
10
|
-
describe LogStash::Outputs::Qingstor::Uploader do
|
11
14
|
let(:bucket) { qs_init_bucket }
|
12
|
-
let(:key) {
|
15
|
+
let(:key) { 'foobar' }
|
13
16
|
let(:tmp_file) { Stud::Temporary.file }
|
14
17
|
let(:tmp_path) { tmp_file.path }
|
15
18
|
let(:logger) { spy(:logger) }
|
16
19
|
|
17
|
-
let(:threadpool) do
|
18
|
-
Concurrent::ThreadPoolExecutor.new(
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
let(:file) do
|
20
|
+
let(:threadpool) do
|
21
|
+
Concurrent::ThreadPoolExecutor.new(:min_threads => 1,
|
22
|
+
:max_threads => 4,
|
23
|
+
:max_queue => 1,
|
24
|
+
:fallback_policy => :caller_runs)
|
25
|
+
end
|
26
|
+
|
27
|
+
let(:file) do
|
27
28
|
f = LogStash::Outputs::Qingstor::TemporaryFile.new(key, tmp_file, tmp_path)
|
28
|
-
f.write(
|
29
|
+
f.write('May the code be with you!')
|
29
30
|
f.fsync
|
30
31
|
f
|
31
32
|
end
|
32
33
|
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
delete_remote_file key
|
37
|
-
end
|
34
|
+
after do
|
35
|
+
delete_remote_file key
|
36
|
+
end
|
38
37
|
|
39
|
-
it
|
38
|
+
it 'upload file to the qingstor bucket' do
|
40
39
|
subject.upload(file)
|
41
40
|
expect(list_remote_file.size).to eq(1)
|
42
|
-
end
|
41
|
+
end
|
43
42
|
|
44
|
-
it
|
43
|
+
it 'execute a callback when the upload is complete' do
|
45
44
|
callback = proc { |f| }
|
46
45
|
expect(callback).to receive(:call).with(file)
|
47
|
-
subject.upload(file,
|
48
|
-
end
|
49
|
-
end
|
46
|
+
subject.upload(file, :on_complete => callback)
|
47
|
+
end
|
48
|
+
end
|
@@ -1,60 +1,69 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
-
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
6
|
-
require
|
7
|
-
|
8
|
-
require_relative
|
2
|
+
|
3
|
+
require 'logstash/devutils/rspec/spec_helper'
|
4
|
+
require 'logstash/outputs/qingstor'
|
5
|
+
require 'logstash/event'
|
6
|
+
require 'openssl'
|
7
|
+
require 'fileutils'
|
8
|
+
require_relative './qs_access_helper'
|
9
|
+
require_relative './spec_helper'
|
9
10
|
|
10
11
|
describe LogStash::Outputs::Qingstor do
|
11
|
-
let(:prefix) {
|
12
|
-
let(:event) { LogStash::Event.new(
|
13
|
-
let(:event_encoded) {
|
14
|
-
let(:events_and_encoded) {{ event => event_encoded}}
|
15
|
-
let(:options)
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
12
|
+
let(:prefix) { 'ss/%{server}' }
|
13
|
+
let(:event) { LogStash::Event.new('server' => 'overwatch') }
|
14
|
+
let(:event_encoded) { 'May the code be with you!' }
|
15
|
+
let(:events_and_encoded) { { event => event_encoded } }
|
16
|
+
let(:options) do
|
17
|
+
{
|
18
|
+
'access_key_id' => ENV['access_key_id'],
|
19
|
+
'secret_access_key' => ENV['secret_access_key'],
|
20
|
+
'bucket' => ENV['bucket'],
|
21
|
+
'region' => ENV['region'],
|
22
|
+
'prefix' => prefix
|
23
|
+
}
|
24
|
+
end
|
25
|
+
let(:tmpdir) { File.join(Dir.tmpdir, 'logstash_restore_dir') }
|
26
|
+
|
27
|
+
after do
|
25
28
|
clean_remote_files
|
26
|
-
end
|
29
|
+
end
|
27
30
|
|
28
|
-
it
|
31
|
+
it 'done work with minimal options' do
|
29
32
|
fetch_event(options, events_and_encoded)
|
30
33
|
expect(list_remote_file.size).to eq(1)
|
31
|
-
end
|
34
|
+
end
|
32
35
|
|
33
|
-
it
|
36
|
+
it 'use aes256 to encrpytion in the server side' do
|
34
37
|
cipher = OpenSSL::Cipher::AES256.new(:CBC)
|
35
38
|
cipher.encrypt
|
36
39
|
key = cipher.random_key
|
37
|
-
fetch_event(options.merge(
|
40
|
+
fetch_event(options.merge('server_side_encryption_algorithm' => 'AES256',
|
41
|
+
'customer_key' => key),
|
42
|
+
events_and_encoded)
|
38
43
|
expect(list_remote_file.size).to eq(1)
|
39
|
-
end
|
44
|
+
end
|
40
45
|
|
41
|
-
it
|
42
|
-
|
43
|
-
|
46
|
+
it 'upload existing file if turn on restore function' do
|
47
|
+
unless File.exist?(File.join(tmpdir, 'uuid_dir'))
|
48
|
+
FileUtils.mkdir_p(File.join(tmpdir, 'uuid_dir'))
|
49
|
+
end
|
50
|
+
f = File.open(File.join(tmpdir, 'uuid_dir/temp'), 'w')
|
44
51
|
f.write(event_encoded * 10)
|
45
52
|
f.close
|
46
|
-
fetch_event(options.merge(
|
53
|
+
fetch_event(options.merge('restore' => true, 'tmpdir' => tmpdir),
|
54
|
+
events_and_encoded)
|
47
55
|
expect(list_remote_file.size).to eq(2)
|
48
|
-
end
|
49
|
-
|
50
|
-
it
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
end
|
56
|
+
end
|
57
|
+
|
58
|
+
it 'redirect to the specified host without specified port' do
|
59
|
+
new_options = options.merge('host' => 'qingstor.dev', 'port' => 444)
|
60
|
+
expect { fetch_event(new_options, events_and_encoded) }
|
61
|
+
.to raise_error(Net::HTTP::Persistent::Error)
|
62
|
+
end
|
63
|
+
|
64
|
+
it 'redirect to the specified host without specified port' do
|
65
|
+
new_options = options.merge('host' => 'qingstor.dev', 'port' => 444)
|
66
|
+
expect { fetch_event(new_options, events_and_encoded) }
|
67
|
+
.to raise_error(Net::HTTP::Persistent::Error)
|
68
|
+
end
|
69
|
+
end
|
@@ -1,20 +1,20 @@
|
|
1
|
-
def fetch_event(settings, events_and_encoded)
|
1
|
+
def fetch_event(settings, events_and_encoded)
|
2
2
|
qs = LogStash::Outputs::Qingstor.new(settings)
|
3
3
|
qs.register
|
4
4
|
qs.multi_receive_encoded(events_and_encoded)
|
5
5
|
qs.close
|
6
|
-
end
|
6
|
+
end
|
7
7
|
|
8
8
|
def qs_init_config(access_key_id = ENV['access_key_id'],
|
9
9
|
secret_access_key = ENV['secret_access_key'])
|
10
10
|
QingStor::SDK::Config.init access_key_id, secret_access_key
|
11
|
-
end
|
11
|
+
end
|
12
12
|
|
13
13
|
def qs_init_bucket(bucket = ENV['bucket'], region = ENV['region'])
|
14
14
|
config = qs_init_config
|
15
|
-
properties = {'bucket-name' => bucket, 'zone' => region }
|
15
|
+
properties = { 'bucket-name' => bucket, 'zone' => region }
|
16
16
|
QingStor::SDK::Bucket.new(config, properties)
|
17
|
-
end
|
17
|
+
end
|
18
18
|
|
19
19
|
def delete_remote_file(key)
|
20
20
|
bucket = qs_init_bucket
|
@@ -29,10 +29,10 @@ end
|
|
29
29
|
def delete_bucket(bucket)
|
30
30
|
bucket = qs_init_bucket(bucket)
|
31
31
|
bucket.delete
|
32
|
-
end
|
32
|
+
end
|
33
33
|
|
34
34
|
def clean_remote_files
|
35
35
|
list_remote_file.each do |record|
|
36
36
|
qs_init_bucket.delete_object record[:key]
|
37
|
-
end
|
38
|
-
end
|
37
|
+
end
|
38
|
+
end
|
data/spec/outputs/spec_helper.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
require
|
2
|
-
require
|
1
|
+
require 'logstash/devutils/rspec/spec_helper'
|
2
|
+
require 'logstash/logging/logger'
|
3
3
|
|
4
|
-
LogStash::Logging::Logger
|
4
|
+
LogStash::Logging::Logger.configure_logging('warn')
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-qingstor
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Evan Zhao
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-08-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -113,10 +113,8 @@ files:
|
|
113
113
|
- lib/logstash/outputs/qingstor/file_repository.rb
|
114
114
|
- lib/logstash/outputs/qingstor/qingstor_validator.rb
|
115
115
|
- lib/logstash/outputs/qingstor/rotation_policy.rb
|
116
|
-
- lib/logstash/outputs/qingstor/size_rotation_policy.rb
|
117
116
|
- lib/logstash/outputs/qingstor/temporary_file.rb
|
118
117
|
- lib/logstash/outputs/qingstor/temporary_file_factory.rb
|
119
|
-
- lib/logstash/outputs/qingstor/time_rotation_policy.rb
|
120
118
|
- lib/logstash/outputs/qingstor/uploader.rb
|
121
119
|
- logstash-output-qingstor.gemspec
|
122
120
|
- spec/outputs/qingstor/file_repository_spec.rb
|
@@ -128,7 +126,7 @@ files:
|
|
128
126
|
- spec/outputs/qingstor_spec.rb
|
129
127
|
- spec/outputs/qs_access_helper.rb
|
130
128
|
- spec/outputs/spec_helper.rb
|
131
|
-
homepage: https://github.com/
|
129
|
+
homepage: https://github.com/yunify/logstash-output-qingstor
|
132
130
|
licenses:
|
133
131
|
- Apache License (2.0)
|
134
132
|
metadata:
|
@@ -1,26 +0,0 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
module LogStash
|
3
|
-
module Outputs
|
4
|
-
class Qingstor
|
5
|
-
class SizeRotationPolicy
|
6
|
-
attr_reader :size_file
|
7
|
-
|
8
|
-
def initialize(size_file)
|
9
|
-
if size_file <= 0
|
10
|
-
raise LogStash::ConfigurationError, "'size_file' need to be greater than 0"
|
11
|
-
end
|
12
|
-
|
13
|
-
@size_file = size_file
|
14
|
-
end
|
15
|
-
|
16
|
-
def rotate?(file)
|
17
|
-
file.size >= size_file
|
18
|
-
end
|
19
|
-
|
20
|
-
def needs_periodic?
|
21
|
-
false
|
22
|
-
end
|
23
|
-
end
|
24
|
-
end
|
25
|
-
end
|
26
|
-
end
|