logstash-output-qingstor 0.3.1 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b634b7aee3754a9bfc4f88269a0ef6fc4135249d1143440ccae417c67852b47f
-  data.tar.gz: 05c469d151a6647bb985b15a368dc3c607d69bf6eee496e6ba1d5841185b2b75
+  metadata.gz: 7cd73fbefc5156daf65ed1b5cbf65294847cafb9a36543ff2a33d2250ae8ee6c
+  data.tar.gz: 760d9f12c2facce1b5cf73eb309ff55a24c0857d110549d3166920477eef53a8
 SHA512:
-  metadata.gz: 2579b496ce9f0da3c9831d4de53f70eb23c7f51febe23b7c82052df1cb2d99337c23724b34a3f8d66ddd385efe468e17f158bb8d786b4ceb5a6ee309ace88a2c
-  data.tar.gz: 545228916a49aa445d47b8242ef062c1d78834590352c7d32f81fa6cb4f93acc7208c78d3bbed7b133f5a3fbee2abd46aa6ab2dc5e7912cacb2db303dc25005e
+  metadata.gz: '08b89f4023ca52c4db4a2ab24e973b139b4ca8dd23b63b8cf0b7e68f406ecb504c871176015c3da23fc8d5879552c49620908603ae8b6315d617fd5296c0956e'
+  data.tar.gz: 7140d939f04a284fd1c7e65d310a4e6e0ed25780e15c3843e74c02f7d857b6021f47e453b8532eeca5a131906c00c297f46e94bc38aa0440f1b53524b1182bb0

data/CHANGELOG.md

@@ -1,3 +1,5 @@
+
+
 ## 0.3.1
   - Refactor module: rotation policy
   

data/README_zh_CN.md

@@ -58,11 +58,11 @@ output {
         # 默认: "size_and_time". 可选枚举值["size_and_time", "size", "time"].
         rotation_strategy => "size_and_time"
 
-        # 配合"size_and_time", "size"的可选配置型, 单位byte
-        # 默认: 1024*1024*5 (byte) = 5 (MB)
-        size_file => 1024*1024*5
+        # 配合"size_and_time", "size"的可选配置型, 单位 MB(megabyte)
+        # 默认: 5 (MB)
+        size_file => 5
 
-        # 配合"size_and_time", "time"的可选配置型, 单位minute
+        # 配合"size_and_time", "time"的可选配置型, 单位 minute
         # 默认: 15 (minutes)
         time_file => 15 
 

data/lib/logstash/outputs/qingstor/rotation_policy.rb

@@ -73,7 +73,15 @@ module LogStash
         end
 
         def initialize(policy, file_size, file_time)
-          @policy = Policy(policy, file_size, file_time)
+          @policy = Policy(policy, to_bytes(file_size), to_seconds(file_time))
+        end
+
+        def to_seconds(file_time)
+          file_time * 60
+        end
+
+        def to_bytes(file_size)
+          file_size * 1024 * 1024
         end
 
         def rotate?(file)

data/lib/logstash/outputs/qingstor/temporary_file.rb

@@ -14,12 +14,12 @@ module LogStash
 
         def_delegators :@fd, :path, :write, :close, :fsync
 
-        attr_reader :fd
+        attr_reader :fd, :dir_path
 
-        def initialize(key, fd, tmp_path)
+        def initialize(key, fd, dir_path)
           @key = key
           @fd = fd
-          @tmp_path = tmp_path
+          @dir_path = dir_path
           @created_at = Time.now
         end
 
@@ -27,8 +27,6 @@ module LogStash
           @created_at
         end
 
-        attr_reader :tmp_path
-
         def size
           @fd.size
         rescue IOError
@@ -49,7 +47,7 @@ module LogStash
           rescue
             IOError
           end
-          FileUtils.rm(path, :force => true)
+          FileUtils.rm_f(path)
         end
 
         def empty?

data/lib/logstash/outputs/qingstor.rb

@@ -76,8 +76,8 @@ class LogStash::Outputs::Qingstor < LogStash::Outputs::Base
   config :rotation_strategy, :validate => %w[size_and_time size time],
                              :default => 'size_and_time'
 
-  # Define the size requirement for each file to upload to QingStor. In byte.
-  config :file_size, :validate => :number, :default => 1024 * 1024 * 5
+  # Define the size requirement for each file to upload to QingStor. In megabyte(MB).
+  config :file_size, :validate => :number, :default => 5
 
   # Define the time interval for each file to upload to QingStor. In minutes.
   config :file_time, :validate => :number, :default => 15

data/logstash-output-qingstor.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-output-qingstor'
-  s.version       = '0.3.1'
+  s.version       = '0.3.2'
   s.licenses      = ['Apache License (2.0)']
   s.summary       = 'logstash output plugin for qingstor'
   s.description   = 'Collect the outputs of logstash and store into QingStor'

data/spec/outputs/qingstor/temporary_file_spec.rb

@@ -6,16 +6,19 @@ require 'fileutils'
 require 'tmpdir'
 
 describe LogStash::Outputs::Qingstor::TemporaryFile do
+  def file_path
+    File.join(Dir.tmpdir, 'foo.log')
+  end
+
   subject { described_class.new(key, io, Dir.tmpdir) }
 
   let(:key) { 'foo.log' }
   let(:content) { 'May the code be with you!' }
-  let(:tmp_path) { File.join(Dir.tmpdir, key) }
   let(:file_mode) { 'w+' }
-  let(:io) { ::File.open(tmp_path, file_mode) }
+  let(:io) { ::File.open(file_path, file_mode) }
 
   after(:all) do
-    FileUtils.rm('/tmp/foo.log') if File.exist?('/tmp/foo.log')
+    FileUtils.rm(file_path) if File.exist?(file_path)
   end
 
   it 'return the key of the file' do
@@ -31,11 +34,11 @@ describe LogStash::Outputs::Qingstor::TemporaryFile do
   it 'can return the correct file size' do
     subject.write(content)
     subject.close
-    expect(subject.size).to eq(File.size(tmp_path))
+    expect(subject.size).to eq(File.size(file_path))
   end
 
-  it 'return the tmp_path of the file' do
-    expect(File.join(subject.tmp_path, subject.key)).to eq(tmp_path)
+  it 'return the file_path of the file' do
+    expect(File.join(subject.dir_path, subject.key)).to eq(file_path)
   end
 
   it 'return the creation time' do
@@ -43,9 +46,9 @@ describe LogStash::Outputs::Qingstor::TemporaryFile do
   end
 
   it 'can delete file correctly' do
-    expect(File.exist?(tmp_path)).to be_truthy
+    expect(File.exist?(file_path)).to be_truthy
     subject.delete!
-    expect(File.exist?(tmp_path)).to be_falsey
+    expect(File.exist?(file_path)).to be_falsey
   end
 
   it 'return if the file is empty' do

data/spec/outputs/qingstor/uploader_spec.rb

@@ -5,7 +5,7 @@ require 'logstash/outputs/qingstor/uploader'
 require 'logstash/outputs/qingstor/temporary_file'
 require 'qingstor/sdk'
 require 'stud/temporary'
-
+require 'fileutils'
 require_relative '../qs_access_helper'
 
 describe LogStash::Outputs::Qingstor::Uploader do
@@ -31,8 +31,13 @@ describe LogStash::Outputs::Qingstor::Uploader do
     f
   end
 
-  after do
+  before(:all) do
+    clean_remote_files
+  end
+
+  after(:each) do
     delete_remote_file key
+    FileUtils.rm_r(tmp_file)
   end
 
   it 'upload file to the qingstor bucket' do
@@ -40,6 +45,12 @@ describe LogStash::Outputs::Qingstor::Uploader do
     expect(list_remote_file.size).to eq(1)
   end
 
+  it 'async upload file to qingstor' do
+    subject.upload_async(file)
+    sleep 2
+    expect(list_remote_file.size).to eq(1)
+  end
+
   it 'execute a callback when the upload is complete' do
     callback = proc { |f| }
     expect(callback).to receive(:call).with(file)

data/spec/outputs/qingstor_spec.rb

@@ -9,10 +9,15 @@ require_relative './qs_access_helper'
 require_relative './spec_helper'
 
 describe LogStash::Outputs::Qingstor do
+  def tmp_dir_path
+    File.join(Dir.tmpdir, 'lg_qs_spec')
+  end
+
   let(:prefix) { 'ss/%{server}' }
   let(:event) { LogStash::Event.new('server' => 'overwatch') }
   let(:event_encoded) { 'May the code be with you!' }
   let(:events_and_encoded) { { event => event_encoded } }
+  let(:tmpdir) { tmp_dir_path }
   let(:options) do
     {
       'access_key_id' => ENV['access_key_id'],
@@ -22,14 +27,17 @@ describe LogStash::Outputs::Qingstor do
       'prefix' => prefix
     }
   end
-  let(:tmpdir) { File.join(Dir.tmpdir, 'logstash_restore_dir') }
+  
+  before(:all) do
+    FileUtils.mkdir_p(tmp_dir_path) unless File.exist?(tmp_dir_path)
+  end
 
-  after do
+  after(:each) do
     clean_remote_files
   end
 
-  before do
-    FileUtils.mkdir_p(tmpdir) unless File.exist?(tmpdir)
+  after(:all) do
+    FileUtils.rm_rf(tmp_dir_path)
   end
 
   it 'done work with minimal options' do
@@ -37,7 +45,7 @@ describe LogStash::Outputs::Qingstor do
     expect(list_remote_file.size).to eq(1)
   end
 
-  it 'use aes256 to encrpytion in the server side' do
+  it 'use aes256 to encrpytion in the server side', :target => true do
     cipher = OpenSSL::Cipher::AES256.new(:CBC)
     cipher.encrypt
     key = cipher.random_key
@@ -48,6 +56,7 @@ describe LogStash::Outputs::Qingstor do
   end
 
   it 'upload existing file if turn on restore function' do
+    binding.pry
     non_empty_file = File.open(File.join(tmpdir, 'non-empty-file'), 'w')
     non_empty_file.write(event_encoded * 10)
     non_empty_file.close

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-qingstor
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - Evan Zhao
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-01-04 00:00:00.000000000 Z
+date: 2018-01-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement