logstash-input-qingstor 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +0 -1
- data/README.md +5 -121
- data/README_zh_CN.md +116 -0
- data/lib/logstash/inputs/qingstor.rb +103 -177
- data/lib/logstash/inputs/qingstor/log_reader.rb +53 -0
- data/lib/logstash/inputs/qingstor/qingstor_validator.rb +32 -27
- data/lib/logstash/inputs/qingstor/sincedb.rb +36 -0
- data/lib/logstash/inputs/qingstor/uploader.rb +56 -0
- data/logstash-input-qingstor.gemspec +11 -9
- data/spec/logstash/inputs/qingstor/log_reader_spec.rb +48 -0
- data/spec/logstash/inputs/qingstor/sincedb_spec.rb +27 -0
- data/spec/logstash/inputs/qingstor/uploader_spec.rb +42 -0
- data/spec/logstash/inputs/qingstor_spec.rb +98 -0
- data/spec/{inputs → logstash/inputs}/qs_access_helper.rb +12 -14
- metadata +18 -10
- data/spec/inputs/qingstor_spec.rb +0 -66
- data/spec/inputs/qingstor_spec_validator_spec.rb +0 -37
@@ -4,28 +4,26 @@ def fetch_events(settings)
|
|
4
4
|
qs.register
|
5
5
|
qs.process_files(queue)
|
6
6
|
queue
|
7
|
-
end
|
7
|
+
end
|
8
8
|
|
9
9
|
def qs_init_config(access_key_id = ENV['access_key_id'],
|
10
10
|
secret_access_key = ENV['secret_access_key'])
|
11
|
-
|
12
|
-
end
|
11
|
+
QingStor::SDK::Config.init access_key_id, secret_access_key
|
12
|
+
end
|
13
13
|
|
14
14
|
def qs_init_bucket(bucket = ENV['bucket'], region = ENV['region'])
|
15
15
|
config = qs_init_config
|
16
|
-
properties = {'bucket-name' => bucket, 'zone' => region }
|
17
|
-
|
18
|
-
end
|
16
|
+
properties = { 'bucket-name' => bucket, 'zone' => region }
|
17
|
+
QingStor::SDK::Bucket.new(config, properties)
|
18
|
+
end
|
19
19
|
|
20
20
|
def upload_file(local_file, remote_file)
|
21
21
|
bucket = qs_init_bucket
|
22
22
|
file = File.expand_path(File.join(File.dirname(__FILE__), local_file))
|
23
23
|
md5_string = Digest::MD5.file(file).to_s
|
24
|
-
bucket.put_object
|
25
|
-
|
26
|
-
|
27
|
-
}
|
28
|
-
end
|
24
|
+
bucket.put_object(remote_file, 'content_md5' => md5_string,
|
25
|
+
'body' => File.open(file))
|
26
|
+
end
|
29
27
|
|
30
28
|
def delete_remote_file(key)
|
31
29
|
bucket = qs_init_bucket
|
@@ -34,13 +32,13 @@ end
|
|
34
32
|
|
35
33
|
def list_remote_file(bucket = ENV['bucket'], region = ENV['region'])
|
36
34
|
bucket = qs_init_bucket(bucket, region)
|
37
|
-
|
35
|
+
bucket.list_objects[:keys]
|
38
36
|
end
|
39
37
|
|
40
38
|
def clean_and_delete_bucket(bucket)
|
41
39
|
bucket = qs_init_bucket(bucket)
|
42
40
|
bucket.list_objects[:keys].each do |file|
|
43
41
|
bucket.delete_object file[:key]
|
44
|
-
end
|
42
|
+
end
|
45
43
|
bucket.delete
|
46
|
-
end
|
44
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-qingstor
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Evan Zhao
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-06-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -86,7 +86,7 @@ dependencies:
|
|
86
86
|
- - ">="
|
87
87
|
- !ruby/object:Gem::Version
|
88
88
|
version: '0'
|
89
|
-
description:
|
89
|
+
description: Fetch file from Qingstor as the input of logstash
|
90
90
|
email: tacingiht@gmail.com
|
91
91
|
executables: []
|
92
92
|
extensions: []
|
@@ -100,15 +100,21 @@ files:
|
|
100
100
|
- Gemfile
|
101
101
|
- LICENSE
|
102
102
|
- README.md
|
103
|
+
- README_zh_CN.md
|
103
104
|
- lib/logstash/inputs/qingstor.rb
|
105
|
+
- lib/logstash/inputs/qingstor/log_reader.rb
|
104
106
|
- lib/logstash/inputs/qingstor/qingstor_validator.rb
|
107
|
+
- lib/logstash/inputs/qingstor/sincedb.rb
|
108
|
+
- lib/logstash/inputs/qingstor/uploader.rb
|
105
109
|
- logstash-input-qingstor.gemspec
|
106
110
|
- spec/fixtures/logstash.log
|
107
111
|
- spec/fixtures/logstash.log.gz
|
108
|
-
- spec/inputs/
|
109
|
-
- spec/inputs/
|
110
|
-
- spec/inputs/
|
111
|
-
|
112
|
+
- spec/logstash/inputs/qingstor/log_reader_spec.rb
|
113
|
+
- spec/logstash/inputs/qingstor/sincedb_spec.rb
|
114
|
+
- spec/logstash/inputs/qingstor/uploader_spec.rb
|
115
|
+
- spec/logstash/inputs/qingstor_spec.rb
|
116
|
+
- spec/logstash/inputs/qs_access_helper.rb
|
117
|
+
homepage: https://github.com/yunify/logstash-input-qingstor
|
112
118
|
licenses:
|
113
119
|
- Apache License (2.0)
|
114
120
|
metadata:
|
@@ -137,6 +143,8 @@ summary: logstash input plugin for QingStor
|
|
137
143
|
test_files:
|
138
144
|
- spec/fixtures/logstash.log
|
139
145
|
- spec/fixtures/logstash.log.gz
|
140
|
-
- spec/inputs/
|
141
|
-
- spec/inputs/
|
142
|
-
- spec/inputs/
|
146
|
+
- spec/logstash/inputs/qingstor/log_reader_spec.rb
|
147
|
+
- spec/logstash/inputs/qingstor/sincedb_spec.rb
|
148
|
+
- spec/logstash/inputs/qingstor/uploader_spec.rb
|
149
|
+
- spec/logstash/inputs/qingstor_spec.rb
|
150
|
+
- spec/logstash/inputs/qs_access_helper.rb
|
@@ -1,66 +0,0 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
require "logstash/devutils/rspec/spec_helper"
|
3
|
-
require "logstash/inputs/qingstor"
|
4
|
-
require_relative "./qs_access_helper"
|
5
|
-
require "tmpdir"
|
6
|
-
|
7
|
-
describe LogStash::Inputs::Qingstor do
|
8
|
-
|
9
|
-
before do
|
10
|
-
Thread.abort_on_exception = true
|
11
|
-
|
12
|
-
upload_file("../fixtures/logstash.log", "log3.log")
|
13
|
-
upload_file("../fixtures/logstash.log.gz", "log3.log.gz")
|
14
|
-
end
|
15
|
-
|
16
|
-
after do
|
17
|
-
delete_remote_file "log3.log"
|
18
|
-
delete_remote_file "log3.log.gz"
|
19
|
-
end
|
20
|
-
|
21
|
-
let(:config) { {
|
22
|
-
"access_key_id" => ENV['access_key_id'],
|
23
|
-
"secret_access_key" => ENV['secret_access_key'],
|
24
|
-
"bucket" => ENV['bucket'],
|
25
|
-
"region" => ENV['region']
|
26
|
-
} }
|
27
|
-
let(:key1) { "log3.log" }
|
28
|
-
let(:key2) { "log3.log.gz" }
|
29
|
-
let(:backup) { "logstash-backup" }
|
30
|
-
let(:local_backup_dir) { File.join(Dir.tmpdir, backup) }
|
31
|
-
|
32
|
-
context "at the local end" do
|
33
|
-
it "backup to local dir" do
|
34
|
-
fetch_events(config.merge({"backup_local_dir" => local_backup_dir }))
|
35
|
-
expect(File.exists?(File.join(local_backup_dir, key1))).to be_truthy
|
36
|
-
expect(File.exists?(File.join(local_backup_dir, key2))).to be_truthy
|
37
|
-
end
|
38
|
-
|
39
|
-
after do
|
40
|
-
FileUtils.rm_r(File.join(local_backup_dir, key1))
|
41
|
-
FileUtils.rm_r(File.join(local_backup_dir, key2))
|
42
|
-
end
|
43
|
-
end
|
44
|
-
|
45
|
-
context "at the remote end " do
|
46
|
-
it "backup to another bucket" do
|
47
|
-
fetch_events(config.merge({"backup_bucket" => backup}))
|
48
|
-
expect(list_remote_file(backup).size).to eq(2)
|
49
|
-
end
|
50
|
-
|
51
|
-
after do
|
52
|
-
clean_and_delete_bucket(backup)
|
53
|
-
end
|
54
|
-
end
|
55
|
-
|
56
|
-
context "advance config" do
|
57
|
-
it "redirect to the specified host without specified port" do
|
58
|
-
expect{fetch_events(config.merge({"host" => "qingstor.dev"}))}.to raise_error(Net::HTTP::Persistent::Error)
|
59
|
-
end
|
60
|
-
|
61
|
-
it "redirect to the specified host without specified port" do
|
62
|
-
expect{fetch_events(config.merge({"host" => "qingstor.dev", "port" => 444}))}.to raise_error(Net::HTTP::Persistent::Error)
|
63
|
-
end
|
64
|
-
end
|
65
|
-
|
66
|
-
end
|
@@ -1,37 +0,0 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
require "logstash/devutils/rspec/spec_helper"
|
3
|
-
require "logstash/inputs/qingstor"
|
4
|
-
|
5
|
-
describe LogStash::Inputs::Qingstor do
|
6
|
-
let(:config) { {
|
7
|
-
"access_key_id" => ENV['access_key_id'],
|
8
|
-
"secret_access_key" => ENV['secret_access_key'],
|
9
|
-
"bucket" => ENV['bucket'],
|
10
|
-
"region" => ENV['region']
|
11
|
-
} }
|
12
|
-
|
13
|
-
it "raise error if it has incorrect key id or access key" do
|
14
|
-
config["access_key_id"] = "wrongid"
|
15
|
-
expect{ described_class.new(config).register }.to raise_error(LogStash::ConfigurationError)
|
16
|
-
end
|
17
|
-
|
18
|
-
it "raise error if it has incorrect key id or access key" do
|
19
|
-
config["secret_access_key"] = "wrongaccesskey"
|
20
|
-
expect{ described_class.new(config).register }.to raise_error(LogStash::ConfigurationError)
|
21
|
-
end
|
22
|
-
|
23
|
-
it "raise error if it has incorrect bucket/region name" do
|
24
|
-
config["bucket"] = "wrongbucket"
|
25
|
-
expect{ described_class.new(config).register }.to raise_error(LogStash::ConfigurationError)
|
26
|
-
end
|
27
|
-
|
28
|
-
it "raise error if it has incorrect bucket/region name" do
|
29
|
-
config["region"] = "wrongregion"
|
30
|
-
expect{ described_class.new(config).register }.to raise_error(LogStash::ConfigurationError)
|
31
|
-
end
|
32
|
-
|
33
|
-
it "use default region if it is not set" do
|
34
|
-
config.delete("region")
|
35
|
-
expect(described_class.new(config).register ).to be_truthy
|
36
|
-
end
|
37
|
-
end
|