logstash-input-qingstor 0.1.3 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,28 +4,26 @@ def fetch_events(settings)
4
4
  qs.register
5
5
  qs.process_files(queue)
6
6
  queue
7
- end
7
+ end
8
8
 
9
9
  def qs_init_config(access_key_id = ENV['access_key_id'],
10
10
  secret_access_key = ENV['secret_access_key'])
11
- return QingStor::SDK::Config.init access_key_id, secret_access_key
12
- end
11
+ QingStor::SDK::Config.init access_key_id, secret_access_key
12
+ end
13
13
 
14
14
  def qs_init_bucket(bucket = ENV['bucket'], region = ENV['region'])
15
15
  config = qs_init_config
16
- properties = {'bucket-name' => bucket, 'zone' => region }
17
- return QingStor::SDK::Bucket.new(config, properties)
18
- end
16
+ properties = { 'bucket-name' => bucket, 'zone' => region }
17
+ QingStor::SDK::Bucket.new(config, properties)
18
+ end
19
19
 
20
20
  def upload_file(local_file, remote_file)
21
21
  bucket = qs_init_bucket
22
22
  file = File.expand_path(File.join(File.dirname(__FILE__), local_file))
23
23
  md5_string = Digest::MD5.file(file).to_s
24
- bucket.put_object remote_file, {
25
- 'content_md5' => md5_string,
26
- 'body' => File.open(file)
27
- }
28
- end
24
+ bucket.put_object(remote_file, 'content_md5' => md5_string,
25
+ 'body' => File.open(file))
26
+ end
29
27
 
30
28
  def delete_remote_file(key)
31
29
  bucket = qs_init_bucket
@@ -34,13 +32,13 @@ end
34
32
 
35
33
  def list_remote_file(bucket = ENV['bucket'], region = ENV['region'])
36
34
  bucket = qs_init_bucket(bucket, region)
37
- return bucket.list_objects[:keys]
35
+ bucket.list_objects[:keys]
38
36
  end
39
37
 
40
38
  def clean_and_delete_bucket(bucket)
41
39
  bucket = qs_init_bucket(bucket)
42
40
  bucket.list_objects[:keys].each do |file|
43
41
  bucket.delete_object file[:key]
44
- end
42
+ end
45
43
  bucket.delete
46
- end
44
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-input-qingstor
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.3
4
+ version: 0.1.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Evan Zhao
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-04-05 00:00:00.000000000 Z
11
+ date: 2017-06-06 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -86,7 +86,7 @@ dependencies:
86
86
  - - ">="
87
87
  - !ruby/object:Gem::Version
88
88
  version: '0'
89
- description: Use this plugin to fetch file from Qingstor as the input of logstash
89
+ description: Fetch file from Qingstor as the input of logstash
90
90
  email: tacingiht@gmail.com
91
91
  executables: []
92
92
  extensions: []
@@ -100,15 +100,21 @@ files:
100
100
  - Gemfile
101
101
  - LICENSE
102
102
  - README.md
103
+ - README_zh_CN.md
103
104
  - lib/logstash/inputs/qingstor.rb
105
+ - lib/logstash/inputs/qingstor/log_reader.rb
104
106
  - lib/logstash/inputs/qingstor/qingstor_validator.rb
107
+ - lib/logstash/inputs/qingstor/sincedb.rb
108
+ - lib/logstash/inputs/qingstor/uploader.rb
105
109
  - logstash-input-qingstor.gemspec
106
110
  - spec/fixtures/logstash.log
107
111
  - spec/fixtures/logstash.log.gz
108
- - spec/inputs/qingstor_spec.rb
109
- - spec/inputs/qingstor_spec_validator_spec.rb
110
- - spec/inputs/qs_access_helper.rb
111
- homepage: https://github.com/Tacinight/logstash-input-qingstor
112
+ - spec/logstash/inputs/qingstor/log_reader_spec.rb
113
+ - spec/logstash/inputs/qingstor/sincedb_spec.rb
114
+ - spec/logstash/inputs/qingstor/uploader_spec.rb
115
+ - spec/logstash/inputs/qingstor_spec.rb
116
+ - spec/logstash/inputs/qs_access_helper.rb
117
+ homepage: https://github.com/yunify/logstash-input-qingstor
112
118
  licenses:
113
119
  - Apache License (2.0)
114
120
  metadata:
@@ -137,6 +143,8 @@ summary: logstash input plugin for QingStor
137
143
  test_files:
138
144
  - spec/fixtures/logstash.log
139
145
  - spec/fixtures/logstash.log.gz
140
- - spec/inputs/qingstor_spec.rb
141
- - spec/inputs/qingstor_spec_validator_spec.rb
142
- - spec/inputs/qs_access_helper.rb
146
+ - spec/logstash/inputs/qingstor/log_reader_spec.rb
147
+ - spec/logstash/inputs/qingstor/sincedb_spec.rb
148
+ - spec/logstash/inputs/qingstor/uploader_spec.rb
149
+ - spec/logstash/inputs/qingstor_spec.rb
150
+ - spec/logstash/inputs/qs_access_helper.rb
@@ -1,66 +0,0 @@
1
- # encoding: utf-8
2
- require "logstash/devutils/rspec/spec_helper"
3
- require "logstash/inputs/qingstor"
4
- require_relative "./qs_access_helper"
5
- require "tmpdir"
6
-
7
- describe LogStash::Inputs::Qingstor do
8
-
9
- before do
10
- Thread.abort_on_exception = true
11
-
12
- upload_file("../fixtures/logstash.log", "log3.log")
13
- upload_file("../fixtures/logstash.log.gz", "log3.log.gz")
14
- end
15
-
16
- after do
17
- delete_remote_file "log3.log"
18
- delete_remote_file "log3.log.gz"
19
- end
20
-
21
- let(:config) { {
22
- "access_key_id" => ENV['access_key_id'],
23
- "secret_access_key" => ENV['secret_access_key'],
24
- "bucket" => ENV['bucket'],
25
- "region" => ENV['region']
26
- } }
27
- let(:key1) { "log3.log" }
28
- let(:key2) { "log3.log.gz" }
29
- let(:backup) { "logstash-backup" }
30
- let(:local_backup_dir) { File.join(Dir.tmpdir, backup) }
31
-
32
- context "at the local end" do
33
- it "backup to local dir" do
34
- fetch_events(config.merge({"backup_local_dir" => local_backup_dir }))
35
- expect(File.exists?(File.join(local_backup_dir, key1))).to be_truthy
36
- expect(File.exists?(File.join(local_backup_dir, key2))).to be_truthy
37
- end
38
-
39
- after do
40
- FileUtils.rm_r(File.join(local_backup_dir, key1))
41
- FileUtils.rm_r(File.join(local_backup_dir, key2))
42
- end
43
- end
44
-
45
- context "at the remote end " do
46
- it "backup to another bucket" do
47
- fetch_events(config.merge({"backup_bucket" => backup}))
48
- expect(list_remote_file(backup).size).to eq(2)
49
- end
50
-
51
- after do
52
- clean_and_delete_bucket(backup)
53
- end
54
- end
55
-
56
- context "advance config" do
57
- it "redirect to the specified host without specified port" do
58
- expect{fetch_events(config.merge({"host" => "qingstor.dev"}))}.to raise_error(Net::HTTP::Persistent::Error)
59
- end
60
-
61
- it "redirect to the specified host without specified port" do
62
- expect{fetch_events(config.merge({"host" => "qingstor.dev", "port" => 444}))}.to raise_error(Net::HTTP::Persistent::Error)
63
- end
64
- end
65
-
66
- end
@@ -1,37 +0,0 @@
1
- # encoding: utf-8
2
- require "logstash/devutils/rspec/spec_helper"
3
- require "logstash/inputs/qingstor"
4
-
5
- describe LogStash::Inputs::Qingstor do
6
- let(:config) { {
7
- "access_key_id" => ENV['access_key_id'],
8
- "secret_access_key" => ENV['secret_access_key'],
9
- "bucket" => ENV['bucket'],
10
- "region" => ENV['region']
11
- } }
12
-
13
- it "raise error if it has incorrect key id or access key" do
14
- config["access_key_id"] = "wrongid"
15
- expect{ described_class.new(config).register }.to raise_error(LogStash::ConfigurationError)
16
- end
17
-
18
- it "raise error if it has incorrect key id or access key" do
19
- config["secret_access_key"] = "wrongaccesskey"
20
- expect{ described_class.new(config).register }.to raise_error(LogStash::ConfigurationError)
21
- end
22
-
23
- it "raise error if it has incorrect bucket/region name" do
24
- config["bucket"] = "wrongbucket"
25
- expect{ described_class.new(config).register }.to raise_error(LogStash::ConfigurationError)
26
- end
27
-
28
- it "raise error if it has incorrect bucket/region name" do
29
- config["region"] = "wrongregion"
30
- expect{ described_class.new(config).register }.to raise_error(LogStash::ConfigurationError)
31
- end
32
-
33
- it "use default region if it is not set" do
34
- config.delete("region")
35
- expect(described_class.new(config).register ).to be_truthy
36
- end
37
- end