logstash-input-qingstor 0.1.3 → 0.1.5

data/spec/{inputs → logstash/inputs}/qs_access_helper.rb

@@ -4,28 +4,26 @@ def fetch_events(settings)
   qs.register
   qs.process_files(queue)
   queue
-end  
+end
 
 def qs_init_config(access_key_id = ENV['access_key_id'],
                    secret_access_key = ENV['secret_access_key'])
-  return QingStor::SDK::Config.init access_key_id, secret_access_key
-end 
+  QingStor::SDK::Config.init access_key_id, secret_access_key
+end
 
 def qs_init_bucket(bucket = ENV['bucket'], region = ENV['region'])
   config = qs_init_config
-  properties = {'bucket-name' => bucket, 'zone' => region }
-  return QingStor::SDK::Bucket.new(config, properties)
-end 
+  properties = { 'bucket-name' => bucket, 'zone' => region }
+  QingStor::SDK::Bucket.new(config, properties)
+end
 
 def upload_file(local_file, remote_file)
   bucket = qs_init_bucket
   file = File.expand_path(File.join(File.dirname(__FILE__), local_file))
   md5_string = Digest::MD5.file(file).to_s
-  bucket.put_object remote_file, { 
-      'content_md5' => md5_string,
-      'body' => File.open(file)
-  }
-end 
+  bucket.put_object(remote_file, 'content_md5' => md5_string,
+                                 'body' => File.open(file))
+end
 
 def delete_remote_file(key)
   bucket = qs_init_bucket
@@ -34,13 +32,13 @@ end
 
 def list_remote_file(bucket = ENV['bucket'], region = ENV['region'])
   bucket = qs_init_bucket(bucket, region)
-  return bucket.list_objects[:keys]
+  bucket.list_objects[:keys]
 end
 
 def clean_and_delete_bucket(bucket)
   bucket = qs_init_bucket(bucket)
   bucket.list_objects[:keys].each do |file|
     bucket.delete_object file[:key]
-  end 
+  end
   bucket.delete
-end 
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-qingstor
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.5
 platform: ruby
 authors:
 - Evan Zhao
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-04-05 00:00:00.000000000 Z
+date: 2017-06-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -86,7 +86,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: Use this plugin to fetch file from Qingstor as the input of logstash
+description: Fetch file from Qingstor as the input of logstash
 email: tacingiht@gmail.com
 executables: []
 extensions: []
@@ -100,15 +100,21 @@ files:
 - Gemfile
 - LICENSE
 - README.md
+- README_zh_CN.md
 - lib/logstash/inputs/qingstor.rb
+- lib/logstash/inputs/qingstor/log_reader.rb
 - lib/logstash/inputs/qingstor/qingstor_validator.rb
+- lib/logstash/inputs/qingstor/sincedb.rb
+- lib/logstash/inputs/qingstor/uploader.rb
 - logstash-input-qingstor.gemspec
 - spec/fixtures/logstash.log
 - spec/fixtures/logstash.log.gz
-- spec/inputs/qingstor_spec.rb
-- spec/inputs/qingstor_spec_validator_spec.rb
-- spec/inputs/qs_access_helper.rb
-homepage: https://github.com/Tacinight/logstash-input-qingstor
+- spec/logstash/inputs/qingstor/log_reader_spec.rb
+- spec/logstash/inputs/qingstor/sincedb_spec.rb
+- spec/logstash/inputs/qingstor/uploader_spec.rb
+- spec/logstash/inputs/qingstor_spec.rb
+- spec/logstash/inputs/qs_access_helper.rb
+homepage: https://github.com/yunify/logstash-input-qingstor
 licenses:
 - Apache License (2.0)
 metadata:
@@ -137,6 +143,8 @@ summary: logstash input plugin for QingStor
 test_files:
 - spec/fixtures/logstash.log
 - spec/fixtures/logstash.log.gz
-- spec/inputs/qingstor_spec.rb
-- spec/inputs/qingstor_spec_validator_spec.rb
-- spec/inputs/qs_access_helper.rb
+- spec/logstash/inputs/qingstor/log_reader_spec.rb
+- spec/logstash/inputs/qingstor/sincedb_spec.rb
+- spec/logstash/inputs/qingstor/uploader_spec.rb
+- spec/logstash/inputs/qingstor_spec.rb
+- spec/logstash/inputs/qs_access_helper.rb

data/spec/inputs/qingstor_spec.rb

@@ -1,66 +0,0 @@
-# encoding: utf-8
-require "logstash/devutils/rspec/spec_helper"
-require "logstash/inputs/qingstor"
-require_relative "./qs_access_helper"
-require "tmpdir"
-
-describe LogStash::Inputs::Qingstor do
-
-  before do 
-    Thread.abort_on_exception = true
-
-    upload_file("../fixtures/logstash.log", "log3.log")
-    upload_file("../fixtures/logstash.log.gz", "log3.log.gz")
-  end
-
-  after do 
-    delete_remote_file "log3.log"    
-    delete_remote_file "log3.log.gz"
-  end
-
-  let(:config) { { 
-      "access_key_id" => ENV['access_key_id'],
-      "secret_access_key" => ENV['secret_access_key'],
-      "bucket" => ENV['bucket'],
-      "region" => ENV['region']
-  } }
-  let(:key1) { "log3.log" }
-  let(:key2) { "log3.log.gz" }
-  let(:backup) { "logstash-backup" }
-  let(:local_backup_dir) { File.join(Dir.tmpdir, backup) }
-
-  context "at the local end" do 
-    it "backup to local dir" do 
-      fetch_events(config.merge({"backup_local_dir" => local_backup_dir }))
-      expect(File.exists?(File.join(local_backup_dir, key1))).to be_truthy
-      expect(File.exists?(File.join(local_backup_dir, key2))).to be_truthy
-    end 
-    
-    after do 
-      FileUtils.rm_r(File.join(local_backup_dir, key1))
-      FileUtils.rm_r(File.join(local_backup_dir, key2))
-    end 
-  end 
-
-  context "at the remote end " do 
-    it "backup to another bucket" do 
-      fetch_events(config.merge({"backup_bucket" => backup}))
-      expect(list_remote_file(backup).size).to eq(2)
-    end 
-
-    after do 
-      clean_and_delete_bucket(backup)
-    end 
-  end 
-
-  context "advance config" do
-    it "redirect to the specified host without specified port" do 
-      expect{fetch_events(config.merge({"host" => "qingstor.dev"}))}.to raise_error(Net::HTTP::Persistent::Error)
-    end 
-
-    it "redirect to the specified host without specified port" do 
-      expect{fetch_events(config.merge({"host" => "qingstor.dev", "port" => 444}))}.to raise_error(Net::HTTP::Persistent::Error)
-    end 
-  end
-
-end

data/spec/inputs/qingstor_spec_validator_spec.rb

@@ -1,37 +0,0 @@
-# encoding: utf-8
-require "logstash/devutils/rspec/spec_helper"
-require "logstash/inputs/qingstor"
-
-describe LogStash::Inputs::Qingstor do
-  let(:config) { { 
-      "access_key_id" => ENV['access_key_id'],
-      "secret_access_key" => ENV['secret_access_key'],
-      "bucket" => ENV['bucket'],
-      "region" => ENV['region']
-  } }
-  
-  it "raise error if it has incorrect key id or access key" do
-    config["access_key_id"] = "wrongid"
-    expect{ described_class.new(config).register }.to raise_error(LogStash::ConfigurationError)
-  end
-
-  it "raise error if it has incorrect key id or access key" do
-    config["secret_access_key"] = "wrongaccesskey"
-    expect{ described_class.new(config).register }.to raise_error(LogStash::ConfigurationError)
-  end
-
-  it "raise error if it has incorrect bucket/region name" do
-    config["bucket"] = "wrongbucket"
-    expect{ described_class.new(config).register }.to raise_error(LogStash::ConfigurationError)
-  end
-
-  it "raise error if it has incorrect bucket/region name" do
-    config["region"] = "wrongregion"
-    expect{ described_class.new(config).register }.to raise_error(LogStash::ConfigurationError)
-  end
-
-  it "use default region if it is not set" do 
-    config.delete("region")
-    expect(described_class.new(config).register ).to be_truthy
-  end
-end