logstash-input-qingstor 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
 - data/lib/logstash/inputs/qingstor.rb +11 -11
 - data/logstash-input-qingstor.gemspec +2 -2
 - data/spec/inputs/qingstor_spec.rb +1 -3
 - metadata +4 -4
 
    
        checksums.yaml
    CHANGED
    
    | 
         @@ -1,7 +1,7 @@ 
     | 
|
| 
       1 
1 
     | 
    
         
             
            ---
         
     | 
| 
       2 
2 
     | 
    
         
             
            SHA1:
         
     | 
| 
       3 
     | 
    
         
            -
              metadata.gz:  
     | 
| 
       4 
     | 
    
         
            -
              data.tar.gz:  
     | 
| 
      
 3 
     | 
    
         
            +
              metadata.gz: c156065e6f20cd2f8e096288335596f2c6ab31cf
         
     | 
| 
      
 4 
     | 
    
         
            +
              data.tar.gz: 843ab1800defe7b983f19aec17991e5d1aa90494
         
     | 
| 
       5 
5 
     | 
    
         
             
            SHA512:
         
     | 
| 
       6 
     | 
    
         
            -
              metadata.gz:  
     | 
| 
       7 
     | 
    
         
            -
              data.tar.gz:  
     | 
| 
      
 6 
     | 
    
         
            +
              metadata.gz: 3470074fc6bd016a3ee83eb1de998823f4c3912305c18a9065793f03e4b5134e9580a2ce95f05d39fe8c316078463a5bb1bcceabcad868478c8aecd5a59b97a0
         
     | 
| 
      
 7 
     | 
    
         
            +
              data.tar.gz: 74b562849a1f53e8322360ef22f74147226dd0da64512c37f049cf84c04f859d5640c43a2d270b2cf486760ac4ed44921b569cf16d5ddecc7a56eac5101a049f
         
     | 
| 
         @@ -36,7 +36,7 @@ class LogStash::Inputs::Qingstor < LogStash::Inputs::Base 
     | 
|
| 
       36 
36 
     | 
    
         | 
| 
       37 
37 
     | 
    
         
             
              # If this set to true, the file will backup to a local dir,
         
     | 
| 
       38 
38 
     | 
    
         
             
              # please make sure you can access to this dir.
         
     | 
| 
       39 
     | 
    
         
            -
              config : 
     | 
| 
      
 39 
     | 
    
         
            +
              config :backup_local_dir, :validate => :string, :default => File.expand_path("~/")
         
     | 
| 
       40 
40 
     | 
    
         | 
| 
       41 
41 
     | 
    
         
             
              # If specified, the file will be upload to this bucket of the given region
         
     | 
| 
       42 
42 
     | 
    
         
             
              config :backup_bucket, :validate => :string, :default => nil
         
     | 
| 
         @@ -44,8 +44,8 @@ class LogStash::Inputs::Qingstor < LogStash::Inputs::Base 
     | 
|
| 
       44 
44 
     | 
    
         
             
              # Specified the backup region in Qingstor.
         
     | 
| 
       45 
45 
     | 
    
         
             
              config :backup_region, :validate => ["pek3a", "sh1a"], :default => "pek3a"
         
     | 
| 
       46 
46 
     | 
    
         | 
| 
       47 
     | 
    
         
            -
              # This prefix will add before backup filename 
     | 
| 
       48 
     | 
    
         
            -
              config :backup_prefix, :validate => :string, :default =>  
     | 
| 
      
 47 
     | 
    
         
            +
              # This prefix, specially in qingstor, will add before the backup filename
         
     | 
| 
      
 48 
     | 
    
         
            +
              config :backup_prefix, :validate => :string, :default => ""
         
     | 
| 
       49 
49 
     | 
    
         | 
| 
       50 
50 
     | 
    
         
             
              # Use sincedb to record the last download time 
         
     | 
| 
       51 
51 
     | 
    
         
             
              config :sincedb_path, :validate => :string, :default => nil 
         
     | 
| 
         @@ -61,8 +61,8 @@ class LogStash::Inputs::Qingstor < LogStash::Inputs::Base 
     | 
|
| 
       61 
61 
     | 
    
         
             
                  raise LogStash::ConfigurationError, "Logstash must have the permissions to write to the temporary directory: #{@tmpdir}"
         
     | 
| 
       62 
62 
     | 
    
         
             
                end
         
     | 
| 
       63 
63 
     | 
    
         | 
| 
       64 
     | 
    
         
            -
                if !@ 
     | 
| 
       65 
     | 
    
         
            -
                  raise LogStash::ConfigurationError, "Logstash must have the permissions to write to the temporary directory: #{@ 
     | 
| 
      
 64 
     | 
    
         
            +
                if !@backup_local_dir.nil? && !directory_valid?(@backup_local_dir)
         
     | 
| 
      
 65 
     | 
    
         
            +
                  raise LogStash::ConfigurationError, "Logstash must have the permissions to write to the temporary directory: #{@backup_local_dir}"
         
     | 
| 
       66 
66 
     | 
    
         
             
                end
         
     | 
| 
       67 
67 
     | 
    
         | 
| 
       68 
68 
     | 
    
         | 
| 
         @@ -95,7 +95,7 @@ class LogStash::Inputs::Qingstor < LogStash::Inputs::Base 
     | 
|
| 
       95 
95 
     | 
    
         
             
                objects.each do |key, time|
         
     | 
| 
       96 
96 
     | 
    
         
             
                  process_log(queue, key)
         
     | 
| 
       97 
97 
     | 
    
         
             
                  backup_to_bucket key unless @backup_bucket.nil?
         
     | 
| 
       98 
     | 
    
         
            -
                  backup_to_local_dir unless @ 
     | 
| 
      
 98 
     | 
    
         
            +
                  backup_to_local_dir unless @backup_local_dir.nil?
         
     | 
| 
       99 
99 
     | 
    
         
             
                  @qs_bucket.delete_object key if @delete_remote_files
         
     | 
| 
       100 
100 
     | 
    
         
             
                end
         
     | 
| 
       101 
101 
     | 
    
         | 
| 
         @@ -143,10 +143,10 @@ class LogStash::Inputs::Qingstor < LogStash::Inputs::Base 
     | 
|
| 
       143 
143 
     | 
    
         | 
| 
       144 
144 
     | 
    
         
             
                md5_string = Digest::MD5.file(@tmp_file_path).to_s
         
     | 
| 
       145 
145 
     | 
    
         | 
| 
       146 
     | 
    
         
            -
                new_key = if backup_prefix.end_with?('/')
         
     | 
| 
       147 
     | 
    
         
            -
                            backup_prefix + key 
         
     | 
| 
      
 146 
     | 
    
         
            +
                new_key = if @backup_prefix.end_with?('/')
         
     | 
| 
      
 147 
     | 
    
         
            +
                            @backup_prefix + key 
         
     | 
| 
       148 
148 
     | 
    
         
             
                          else 
         
     | 
| 
       149 
     | 
    
         
            -
                            backup_prefix + '/' + key 
         
     | 
| 
      
 149 
     | 
    
         
            +
                            @backup_prefix + '/' + key 
         
     | 
| 
       150 
150 
     | 
    
         
             
                          end 
         
     | 
| 
       151 
151 
     | 
    
         | 
| 
       152 
152 
     | 
    
         
             
                bucket.put_object new_key, { 
         
     | 
| 
         @@ -156,8 +156,8 @@ class LogStash::Inputs::Qingstor < LogStash::Inputs::Base 
     | 
|
| 
       156 
156 
     | 
    
         
             
              end 
         
     | 
| 
       157 
157 
     | 
    
         | 
| 
       158 
158 
     | 
    
         
             
              def backup_to_local_dir
         
     | 
| 
       159 
     | 
    
         
            -
                FileUtils.mkdir_p @ 
     | 
| 
       160 
     | 
    
         
            -
                FileUtils.cp @tmp_file_path, @ 
     | 
| 
      
 159 
     | 
    
         
            +
                FileUtils.mkdir_p @backup_local_dir unless File.exist? @backup_local_dir
         
     | 
| 
      
 160 
     | 
    
         
            +
                FileUtils.cp @tmp_file_path, @backup_local_dir
         
     | 
| 
       161 
161 
     | 
    
         
             
              end
         
     | 
| 
       162 
162 
     | 
    
         | 
| 
       163 
163 
     | 
    
         
             
              def process_local_log(queue, filename)
         
     | 
| 
         @@ -1,6 +1,6 @@ 
     | 
|
| 
       1 
1 
     | 
    
         
             
            Gem::Specification.new do |s|
         
     | 
| 
       2 
2 
     | 
    
         
             
              s.name          = 'logstash-input-qingstor'
         
     | 
| 
       3 
     | 
    
         
            -
              s.version       = '0.1. 
     | 
| 
      
 3 
     | 
    
         
            +
              s.version       = '0.1.2'
         
     | 
| 
       4 
4 
     | 
    
         
             
              s.licenses      = ['Apache License (2.0)']
         
     | 
| 
       5 
5 
     | 
    
         
             
              s.summary       = 'logstash input plugin for QingStor'
         
     | 
| 
       6 
6 
     | 
    
         
             
              s.description   = 'Use this plugin to fetch file as a input of logstash from Qingstor'
         
     | 
| 
         @@ -21,7 +21,7 @@ Gem::Specification.new do |s| 
     | 
|
| 
       21 
21 
     | 
    
         
             
              s.add_runtime_dependency "logstash-core-plugin-api", ">=1.6", "<=2.99"
         
     | 
| 
       22 
22 
     | 
    
         
             
              s.add_runtime_dependency 'logstash-codec-plain'
         
     | 
| 
       23 
23 
     | 
    
         
             
              s.add_runtime_dependency 'stud', '>= 0.0.22'
         
     | 
| 
       24 
     | 
    
         
            -
              s.add_runtime_dependency "qingstor-sdk", " 
     | 
| 
      
 24 
     | 
    
         
            +
              s.add_runtime_dependency "qingstor-sdk", ">=1.9.2"
         
     | 
| 
       25 
25 
     | 
    
         | 
| 
       26 
26 
     | 
    
         
             
              s.add_development_dependency 'logstash-devutils'
         
     | 
| 
       27 
27 
     | 
    
         
             
            end
         
     | 
| 
         @@ -31,7 +31,7 @@ describe LogStash::Inputs::Qingstor do 
     | 
|
| 
       31 
31 
     | 
    
         | 
| 
       32 
32 
     | 
    
         
             
              context "local backup" do 
         
     | 
| 
       33 
33 
     | 
    
         
             
                it "backup to local dir" do 
         
     | 
| 
       34 
     | 
    
         
            -
                  fetch_events(config.merge({" 
     | 
| 
      
 34 
     | 
    
         
            +
                  fetch_events(config.merge({"backup_local_dir" => local_backup_dir }))
         
     | 
| 
       35 
35 
     | 
    
         
             
                  expect(File.exists?(File.join(local_backup_dir, key1))).to be_truthy
         
     | 
| 
       36 
36 
     | 
    
         
             
                  expect(File.exists?(File.join(local_backup_dir, key2))).to be_truthy
         
     | 
| 
       37 
37 
     | 
    
         
             
                end 
         
     | 
| 
         @@ -53,6 +53,4 @@ describe LogStash::Inputs::Qingstor do 
     | 
|
| 
       53 
53 
     | 
    
         
             
                end 
         
     | 
| 
       54 
54 
     | 
    
         
             
              end 
         
     | 
| 
       55 
55 
     | 
    
         | 
| 
       56 
     | 
    
         
            -
              context ""
         
     | 
| 
       57 
     | 
    
         
            -
              
         
     | 
| 
       58 
56 
     | 
    
         
             
            end
         
     | 
    
        metadata
    CHANGED
    
    | 
         @@ -1,14 +1,14 @@ 
     | 
|
| 
       1 
1 
     | 
    
         
             
            --- !ruby/object:Gem::Specification
         
     | 
| 
       2 
2 
     | 
    
         
             
            name: logstash-input-qingstor
         
     | 
| 
       3 
3 
     | 
    
         
             
            version: !ruby/object:Gem::Version
         
     | 
| 
       4 
     | 
    
         
            -
              version: 0.1. 
     | 
| 
      
 4 
     | 
    
         
            +
              version: 0.1.2
         
     | 
| 
       5 
5 
     | 
    
         
             
            platform: ruby
         
     | 
| 
       6 
6 
     | 
    
         
             
            authors:
         
     | 
| 
       7 
7 
     | 
    
         
             
            - Evan Zhao
         
     | 
| 
       8 
8 
     | 
    
         
             
            autorequire:
         
     | 
| 
       9 
9 
     | 
    
         
             
            bindir: bin
         
     | 
| 
       10 
10 
     | 
    
         
             
            cert_chain: []
         
     | 
| 
       11 
     | 
    
         
            -
            date: 2017-03- 
     | 
| 
      
 11 
     | 
    
         
            +
            date: 2017-03-16 00:00:00.000000000 Z
         
     | 
| 
       12 
12 
     | 
    
         
             
            dependencies:
         
     | 
| 
       13 
13 
     | 
    
         
             
            - !ruby/object:Gem::Dependency
         
     | 
| 
       14 
14 
     | 
    
         
             
              requirement: !ruby/object:Gem::Requirement
         
     | 
| 
         @@ -61,7 +61,7 @@ dependencies: 
     | 
|
| 
       61 
61 
     | 
    
         
             
            - !ruby/object:Gem::Dependency
         
     | 
| 
       62 
62 
     | 
    
         
             
              requirement: !ruby/object:Gem::Requirement
         
     | 
| 
       63 
63 
     | 
    
         
             
                requirements:
         
     | 
| 
       64 
     | 
    
         
            -
                - -  
     | 
| 
      
 64 
     | 
    
         
            +
                - - ">="
         
     | 
| 
       65 
65 
     | 
    
         
             
                  - !ruby/object:Gem::Version
         
     | 
| 
       66 
66 
     | 
    
         
             
                    version: 1.9.2
         
     | 
| 
       67 
67 
     | 
    
         
             
              name: qingstor-sdk
         
     | 
| 
         @@ -69,7 +69,7 @@ dependencies: 
     | 
|
| 
       69 
69 
     | 
    
         
             
              type: :runtime
         
     | 
| 
       70 
70 
     | 
    
         
             
              version_requirements: !ruby/object:Gem::Requirement
         
     | 
| 
       71 
71 
     | 
    
         
             
                requirements:
         
     | 
| 
       72 
     | 
    
         
            -
                - -  
     | 
| 
      
 72 
     | 
    
         
            +
                - - ">="
         
     | 
| 
       73 
73 
     | 
    
         
             
                  - !ruby/object:Gem::Version
         
     | 
| 
       74 
74 
     | 
    
         
             
                    version: 1.9.2
         
     | 
| 
       75 
75 
     | 
    
         
             
            - !ruby/object:Gem::Dependency
         
     |