logstash-output-qingstor 0.1.3 → 0.2.0

data/spec/outputs/qingstor/rotation_policy_spec.rb

@@ -1,81 +1,96 @@
 # encoding: utf-8
-require "logstash/devutils/rspec/spec_helper"
-require "logstash/outputs/qingstor/temporary_file"
-require "logstash/outputs/qingstor/rotation_policy"
+
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/outputs/qingstor/temporary_file'
+require 'logstash/outputs/qingstor/rotation_policy'
 
 describe LogStash::Outputs::Qingstor::RotationPolicy do
   let(:size_file) { 1024 * 2 }
   let(:time_file) { 2 }
-  let(:name) { "foobar" }
+  let(:name) { 'foobar' }
   let(:tmp_file) { Stud::Temporary.file }
   let(:tmp_dir) { tmp_file.path }
-  let(:file) { LogStash::Outputs::Qingstor::TemporaryFile.new(name, tmp_file, tmp_dir) }
-  let(:content) { "May the code be with you" * 100 }
-  
+  let(:file) do
+    LogStash::Outputs::Qingstor::TemporaryFile.new(name, tmp_file, tmp_dir)
+  end
+  let(:content) { 'May the code be with you' * 100 }
 
-  context "when size_and_time policy" do 
-    subject { described_class.new("size_and_time", size_file, time_file) }
+  context 'when size_and_time policy' do
+    subject { described_class.new('size_and_time', size_file, time_file) }
 
-    it "raise error if time_file is no grater then 0" do 
-      expect{ described_class.new("size_and_time", 0, 0) }.to raise_error(LogStash::ConfigurationError)
-      expect{ described_class.new("size_and_time", -1, 0) }.to raise_error(LogStash::ConfigurationError)
-      expect{ described_class.new("size_and_time", 0, -1) }.to raise_error(LogStash::ConfigurationError)
-      expect{ described_class.new("size_and_time", -1, -1) }.to raise_error(LogStash::ConfigurationError)
-    
-    end 
+    it 'raise error if time_file is no grater then 0' do
+      expect { described_class.new('size_and_time', 0, 0) }
+        .to raise_error(LogStash::ConfigurationError)
+      expect { described_class.new('size_and_time', -1, 0) }
+        .to raise_error(LogStash::ConfigurationError)
+      expect { described_class.new('size_and_time', 0, -1) }
+        .to raise_error(LogStash::ConfigurationError)
+      expect { described_class.new('size_and_time', -1, -1) }
+        .to raise_error(LogStash::ConfigurationError)
+    end
 
-    it "return false if the file is not old enough" do 
+    it 'return false if the file is not old enough' do
       expect(subject.rotate?(file)).to be_falsey
     end
 
-    it "return false if the file is old enough with file size 0" do
-      allow(file).to receive(:ctime).and_return(Time.now - (time_file * 2 * 60))
+    it 'return false if the file is old enough with file size 0' do
+      allow(file)
+        .to receive(:ctime).and_return(Time.now - (time_file * 2 * 60))
       expect(subject.rotate?(file)).to be_falsey
     end
 
-    it "return truth if the file is old enough and non-empty" do 
+    it 'return truth if the file is old enough and non-empty' do
       file.write(content)
       file.fsync
-      allow(file).to receive(:ctime).and_return(Time.now - (time_file * 2 * 60))
+      allow(file)
+        .to receive(:ctime).and_return(Time.now - (time_file * 2 * 60))
       expect(subject.rotate?(file)).to be_truthy
     end
-  end 
+  end
+
+  context 'when size policy' do
+    subject { described_class.new('size', size_file, time_file) }
 
-  context "when size policy" do 
-    subject { described_class.new("size", size_file, time_file) }
-    it "raise error if size_file is no grater then 0" do 
-      expect{described_class.new("size", 0, 0)}.to raise_error(LogStash::ConfigurationError)
-      expect{described_class.new("size", -1, 0)}.to raise_error(LogStash::ConfigurationError)
-    end 
+    it 'raise error if size_file is no grater then 0' do
+      expect { described_class.new('size', 0, 0) }
+        .to raise_error(LogStash::ConfigurationError)
+      expect { described_class.new('size', -1, 0) }
+        .to raise_error(LogStash::ConfigurationError)
+    end
 
-    it "return true if the file has a bigger size value then 'size_file'" do 
+    it "return true if the file has a bigger size value then 'size_file'" do
       file.write(content)
       file.fsync
       expect(subject.rotate?(file)).to be_truthy
     end
-  end 
+  end
+
+  context 'when time policy' do
+    subject { described_class.new('time', size_file, time_file) }
 
-  context "when time policy" do 
-    subject { described_class.new("time", size_file, time_file) }
-    it "raise error if time_file is no grater then 0" do 
-      expect{described_class.new("time", 0, 0)}.to raise_error(LogStash::ConfigurationError)
-      expect{described_class.new("time", 0, -1)}.to raise_error(LogStash::ConfigurationError)
-    end 
+    it 'raise error if time_file is no grater then 0' do
+      expect { described_class.new('time', 0, 0) }
+        .to raise_error(LogStash::ConfigurationError)
+      expect { described_class.new('time', 0, -1) }
+        .to raise_error(LogStash::ConfigurationError)
+    end
 
-    it "return false if the file is not old enough" do 
+    it 'return false if the file is not old enough' do
       expect(subject.rotate?(file)).to be_falsey
     end
 
-    it "return false if the file is old enough with file size 0" do
-      allow(file).to receive(:ctime).and_return(Time.now - (time_file * 2 * 60))
+    it 'return false if the file is old enough with file size 0' do
+      allow(file)
+        .to receive(:ctime).and_return(Time.now - (time_file * 2 * 60))
       expect(subject.rotate?(file)).to be_falsey
     end
 
-    it "return truth if the file is old enough and non-empty" do 
+    it 'return truth if the file is old enough and non-empty' do
       file.write(content)
       file.fsync
-      allow(file).to receive(:ctime).and_return(Time.now - (time_file * 2 * 60))
+      allow(file)
+       .to receive(:ctime).and_return(Time.now - (time_file * 2 * 60))
       expect(subject.rotate?(file)).to be_truthy
     end
-  end 
-end 
+  end
+end

data/spec/outputs/qingstor/temporary_file_factory_spec.rb

@@ -1,82 +1,84 @@
 # encoding: utf-8
-require "logstash/devutils/rspec/spec_helper"
-require "logstash/outputs/qingstor/temporary_file"
-require "logstash/outputs/qingstor/temporary_file_factory"
-require "fileutils"
-require "tmpdir"
 
-describe LogStash::Outputs::Qingstor::TemporaryFileFactory do
-  let(:prefix) { "lg2qs" }
-  let(:tags) { [ ] }
-  let(:tmpdir) { File.join(Dir.tmpdir, "logstash-qs") }
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/outputs/qingstor/temporary_file'
+require 'logstash/outputs/qingstor/temporary_file_factory'
+require 'fileutils'
+require 'tmpdir'
 
+describe LogStash::Outputs::Qingstor::TemporaryFileFactory do
   subject { described_class.new(prefix, tags, encoding, tmpdir) }
 
-  shared_examples "file factory" do
-    it "creates the file on disk" do
+  let(:prefix) { 'lg2qs' }
+  let(:tags) { [] }
+  let(:tmpdir) { File.join(Dir.tmpdir, 'logstash-qs') }
+
+  shared_examples 'file factory' do
+    it 'creates the file on disk' do
       expect(File.exist?(subject.current.path)).to be_truthy
     end
 
-    it "create a temporary file when initialized" do
-      expect(subject.current).to be_kind_of(LogStash::Outputs::Qingstor::TemporaryFile)
+    it 'create a temporary file when initialized' do
+      expect(subject.current)
+        .to be_kind_of(LogStash::Outputs::Qingstor::TemporaryFile)
     end
 
-    it "create a file in the right format" do
+    it 'create a file in the right format' do
       expect(subject.current.path).to match(extension)
     end
 
-    it "allow to rotate the file" do
+    it 'allow to rotate the file' do
       file_path = subject.current.path
       expect(subject.rotate!.path).not_to eq(file_path)
     end
 
-    it "increments the part name on rotation" do
+    it 'increments the part name on rotation' do
       expect(subject.current.path).to match(/part0/)
       expect(subject.rotate!.path).to match(/part1/)
     end
 
-    it "includes the date" do
+    it 'includes the date' do
       n = Time.now
-      expect(subject.current.path).to include(n.strftime("%Y-%m-%dT"))
+      expect(subject.current.path).to include(n.strftime('%Y-%m-%dT'))
     end
 
-    it "include the file key in the path"  do
+    it 'include the file key in the path' do
       file = subject.current
       expect(file.path).to match(/#{file.key}/)
     end
 
-    it "create a unique directory in the temporary directory for each file" do
-      uuid = "hola"
+    it 'create a unique directory in the temporary directory for each file' do
+      uuid = 'hola'
       expect(SecureRandom).to receive(:uuid).and_return(uuid).twice
       expect(subject.current.path).to include(uuid)
     end
 
-    context "with tags supplied" do
-      let(:tags) { ["secret", "service"] }
+    context 'with tags supplied' do
+      let(:tags) { %w[secret service] }
 
-      it "adds tags to the filename" do
+      it 'adds tags to the filename' do
         expect(subject.current.path).to match(/tag_#{tags.join('.')}.part/)
       end
     end
 
-    context "without tags" do
+    context 'without tags' do
       it "doesn't add tags to the filename" do
         expect(subject.current.path).not_to match(/tag_/)
       end
     end
   end
 
-  context "when gzip" do
-    let(:encoding) { "gzip" }
+  context 'when gzip' do
+    let(:encoding) { 'gzip' }
     let(:extension) { /\.log.gz$/ }
 
-    include_examples "file factory"
+    include_examples 'file factory'
   end
 
-  context "when encoding set to `none`" do
-    let(:encoding) { "none" }
+  context 'when encoding set to `none`' do
+    let(:encoding) { 'none' }
     let(:extension) { /\.log$/ }
 
-    include_examples "file factory"
+    include_examples 'file factory'
   end
-end 
+end

data/spec/outputs/qingstor/temporary_file_spec.rb

@@ -1,55 +1,56 @@
 # encoding: utf-8
-require "logstash/devutils/rspec/spec_helper"
-require "logstash/outputs/qingstor/temporary_file"
-require "fileutils"
-require "tmpdir"
+
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/outputs/qingstor/temporary_file'
+require 'fileutils'
+require 'tmpdir'
 
 describe LogStash::Outputs::Qingstor::TemporaryFile do
-  let(:key) { "foo.log" }
-  let(:content) { "May the code be with you!" }
-  let(:tmp_path) { File.join(Dir.tmpdir, key) }
-  let(:file_mode) { "w+" }
-  let(:io){ ::File.open(tmp_path, file_mode)}
+  subject { described_class.new(key, io, tmp_path) }
 
-  after(:all) do 
-    FileUtils.rm_r("/tmp/foo.log") if File.exist?("/tmp/foo.log")
-  end 
+  let(:key) { 'foo.log' }
+  let(:content) { 'May the code be with you!' }
+  let(:tmp_path) { File.join(Dir.tmpdir, key) }
+  let(:file_mode) { 'w+' }
+  let(:io) { ::File.open(tmp_path, file_mode) }
 
-  subject { described_class.new(key, io, tmp_path) }
+  after(:all) do
+    FileUtils.rm_r('/tmp/foo.log') if File.exist?('/tmp/foo.log')
+  end
 
-  it "return the key of the file" do 
-    expect(subject.key).to eq(key) 
-  end 
+  it 'return the key of the file' do
+    expect(subject.key).to eq(key)
+  end
 
-  it "is writable and readable" do 
+  it 'is writable and readable' do
     subject.write(content)
-    subject.close 
+    subject.close
     expect(File.read(subject.path).strip).to eq(content)
-  end 
+  end
 
-  it "can return the correct file size" do 
+  it 'can return the correct file size' do
     subject.write(content)
-    subject.close 
+    subject.close
     expect(subject.size).to eq(File.size(tmp_path))
-  end 
+  end
 
-  it "return the tmp_path of the file" do 
+  it 'return the tmp_path of the file' do
     expect(subject.tmp_path).to eq(tmp_path)
-  end 
+  end
 
-  it "return the creation time" do 
+  it 'return the creation time' do
     expect(subject.ctime).to be < Time.now + 1.0
-  end 
+  end
 
-  it "can delete file correctly" do
+  it 'can delete file correctly' do
     expect(File.exist?(subject.tmp_path)).to be_truthy
     subject.delete!
     expect(File.exist?(subject.tmp_path)).to be_falsey
   end
 
-  it "return if the file is empty" do 
+  it 'return if the file is empty' do
     expect(subject.empty?).to be_truthy
     subject.write(content)
     expect(subject.empty?).to be_falsey
-  end 
-end 
+  end
+end

data/spec/outputs/qingstor/uploader_spec.rb

@@ -1,49 +1,48 @@
 # encoding: utf-8
-require "logstash/devutils/rspec/spec_helper"
-require "logstash/outputs/qingstor/uploader"
-require "logstash/outputs/qingstor/temporary_file"
-require "qingstor/sdk"
-require "stud/temporary"
 
-require_relative "../qs_access_helper"
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/outputs/qingstor/uploader'
+require 'logstash/outputs/qingstor/temporary_file'
+require 'qingstor/sdk'
+require 'stud/temporary'
+
+require_relative '../qs_access_helper'
+
+describe LogStash::Outputs::Qingstor::Uploader do
+  subject { described_class.new(bucket, logger, threadpool) }
 
-describe LogStash::Outputs::Qingstor::Uploader do 
   let(:bucket) { qs_init_bucket }
-  let(:key) { "foobar" }
+  let(:key) { 'foobar' }
   let(:tmp_file) { Stud::Temporary.file }
   let(:tmp_path) { tmp_file.path }
   let(:logger) { spy(:logger) }
 
-  let(:threadpool) do 
-    Concurrent::ThreadPoolExecutor.new({
-                                         :min_threads => 1,
-                                         :max_threads => 4,
-                                         :max_queue => 1,
-                                         :fallback_policy => :caller_runs
-                                       })
-  end 
-  
-  let(:file) do 
+  let(:threadpool) do
+    Concurrent::ThreadPoolExecutor.new(:min_threads => 1,
+                                       :max_threads => 4,
+                                       :max_queue => 1,
+                                       :fallback_policy => :caller_runs)
+  end
+
+  let(:file) do
     f = LogStash::Outputs::Qingstor::TemporaryFile.new(key, tmp_file, tmp_path)
-    f.write("May the code be with you!")
+    f.write('May the code be with you!')
     f.fsync
     f
   end
 
-  subject { described_class.new(bucket, logger, threadpool) }
-
-  after do 
-    delete_remote_file key 
-  end 
+  after do
+    delete_remote_file key
+  end
 
-  it "upload file to the qingstor bucket" do 
+  it 'upload file to the qingstor bucket' do
     subject.upload(file)
     expect(list_remote_file.size).to eq(1)
-  end 
+  end
 
-  it "execute a callback when the upload is complete" do 
+  it 'execute a callback when the upload is complete' do
     callback = proc { |f| }
     expect(callback).to receive(:call).with(file)
-    subject.upload(file, { :on_complete => callback })
-  end 
-end 
+    subject.upload(file, :on_complete => callback)
+  end
+end

data/spec/outputs/qingstor_spec.rb

@@ -1,60 +1,69 @@
 # encoding: utf-8
-require "logstash/devutils/rspec/spec_helper"
-require "logstash/outputs/qingstor"
-require "logstash/event"
-require "openssl"
-require "fileutils"
-require_relative "./qs_access_helper"
-require_relative "./spec_helper"
+
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/outputs/qingstor'
+require 'logstash/event'
+require 'openssl'
+require 'fileutils'
+require_relative './qs_access_helper'
+require_relative './spec_helper'
 
 describe LogStash::Outputs::Qingstor do
-  let(:prefix) { "ss/%{server}"}
-  let(:event) { LogStash::Event.new({ "server" => "overwatch" }) }
-  let(:event_encoded) { "May the code be with you!" }
-  let(:events_and_encoded) {{ event => event_encoded}}
-  let(:options) {{
-      "access_key_id" => ENV['access_key_id'],
-      "secret_access_key" => ENV['secret_access_key'],
-      "bucket" => ENV['bucket'],
-      "region" => ENV['region'],
-      "prefix" => prefix
-  }}
-  let(:tmpdir){File.join(Dir.tmpdir, "logstash_restore_dir")}
-
-  after do 
+  let(:prefix) { 'ss/%{server}' }
+  let(:event) { LogStash::Event.new('server' => 'overwatch') }
+  let(:event_encoded) { 'May the code be with you!' }
+  let(:events_and_encoded) { { event => event_encoded } }
+  let(:options) do
+    {
+      'access_key_id' => ENV['access_key_id'],
+      'secret_access_key' => ENV['secret_access_key'],
+      'bucket' => ENV['bucket'],
+      'region' => ENV['region'],
+      'prefix' => prefix
+    }
+  end
+  let(:tmpdir) { File.join(Dir.tmpdir, 'logstash_restore_dir') }
+
+  after do
     clean_remote_files
-  end 
+  end
 
-  it "done work with minimal options" do 
+  it 'done work with minimal options' do
     fetch_event(options, events_and_encoded)
     expect(list_remote_file.size).to eq(1)
-  end 
+  end
 
-  it "use aes256 to encrpytion in the server side" do 
+  it 'use aes256 to encrpytion in the server side' do
     cipher = OpenSSL::Cipher::AES256.new(:CBC)
     cipher.encrypt
     key = cipher.random_key
-    fetch_event(options.merge({"server_side_encryption_algorithm" => "AES256","customer_key" => key}), events_and_encoded)
+    fetch_event(options.merge('server_side_encryption_algorithm' => 'AES256',
+                              'customer_key' => key),
+                events_and_encoded)
     expect(list_remote_file.size).to eq(1)
-  end 
+  end
 
-  it "upload existing file if turn on restore function" do 
-    FileUtils.mkdir_p(File.join(tmpdir, "uuid_dir")) unless File.exists?(File.join(tmpdir, "uuid_dir"))
-    f = File.open(File.join(tmpdir, "uuid_dir/temp"), 'w')
+  it 'upload existing file if turn on restore function' do
+    unless File.exist?(File.join(tmpdir, 'uuid_dir'))
+      FileUtils.mkdir_p(File.join(tmpdir, 'uuid_dir'))
+    end
+    f = File.open(File.join(tmpdir, 'uuid_dir/temp'), 'w')
     f.write(event_encoded * 10)
     f.close
-    fetch_event(options.merge({"restore" => true, "tmpdir" => tmpdir}), events_and_encoded)
+    fetch_event(options.merge('restore' => true, 'tmpdir' => tmpdir),
+                events_and_encoded)
     expect(list_remote_file.size).to eq(2)
-  end 
-
-  it "redirect to the specified host without specified port" do 
-      new_options = options.merge({"host" => "qingstor.dev", "port" => 444})
-      expect{fetch_event(new_options, events_and_encoded)}.to raise_error(Net::HTTP::Persistent::Error)
-    end 
-
-  it "redirect to the specified host without specified port" do 
-    new_options = options.merge({"host" => "qingstor.dev", "port" => 444})
-    expect{fetch_event(new_options, events_and_encoded)}.to raise_error(Net::HTTP::Persistent::Error)
-  end 
-  
-end 
+  end
+
+  it 'redirect to the specified host without specified port' do
+    new_options = options.merge('host' => 'qingstor.dev', 'port' => 444)
+    expect { fetch_event(new_options, events_and_encoded) }
+      .to raise_error(Net::HTTP::Persistent::Error)
+  end
+
+  it 'redirect to the specified host without specified port' do
+    new_options = options.merge('host' => 'qingstor.dev', 'port' => 444)
+    expect { fetch_event(new_options, events_and_encoded) }
+      .to raise_error(Net::HTTP::Persistent::Error)
+  end
+end

data/spec/outputs/qs_access_helper.rb

@@ -1,20 +1,20 @@
-def fetch_event(settings, events_and_encoded) 
+def fetch_event(settings, events_and_encoded)
   qs = LogStash::Outputs::Qingstor.new(settings)
   qs.register
   qs.multi_receive_encoded(events_and_encoded)
   qs.close
-end 
+end
 
 def qs_init_config(access_key_id = ENV['access_key_id'],
                    secret_access_key = ENV['secret_access_key'])
   QingStor::SDK::Config.init access_key_id, secret_access_key
-end 
+end
 
 def qs_init_bucket(bucket = ENV['bucket'], region = ENV['region'])
   config = qs_init_config
-  properties = {'bucket-name' => bucket, 'zone' => region }
+  properties = { 'bucket-name' => bucket, 'zone' => region }
   QingStor::SDK::Bucket.new(config, properties)
-end 
+end
 
 def delete_remote_file(key)
   bucket = qs_init_bucket
@@ -29,10 +29,10 @@ end
 def delete_bucket(bucket)
   bucket = qs_init_bucket(bucket)
   bucket.delete
-end 
+end
 
 def clean_remote_files
   list_remote_file.each do |record|
     qs_init_bucket.delete_object record[:key]
-  end 
-end 
+  end
+end

data/spec/outputs/spec_helper.rb

@@ -1,4 +1,4 @@
-require "logstash/devutils/rspec/spec_helper"
-require "logstash/logging/logger"
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/logging/logger'
 
-LogStash::Logging::Logger::configure_logging("warn")
+LogStash::Logging::Logger.configure_logging('warn')

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-qingstor
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.0
 platform: ruby
 authors:
 - Evan Zhao
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-05-22 00:00:00.000000000 Z
+date: 2017-08-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -113,10 +113,8 @@ files:
 - lib/logstash/outputs/qingstor/file_repository.rb
 - lib/logstash/outputs/qingstor/qingstor_validator.rb
 - lib/logstash/outputs/qingstor/rotation_policy.rb
-- lib/logstash/outputs/qingstor/size_rotation_policy.rb
 - lib/logstash/outputs/qingstor/temporary_file.rb
 - lib/logstash/outputs/qingstor/temporary_file_factory.rb
-- lib/logstash/outputs/qingstor/time_rotation_policy.rb
 - lib/logstash/outputs/qingstor/uploader.rb
 - logstash-output-qingstor.gemspec
 - spec/outputs/qingstor/file_repository_spec.rb
@@ -128,7 +126,7 @@ files:
 - spec/outputs/qingstor_spec.rb
 - spec/outputs/qs_access_helper.rb
 - spec/outputs/spec_helper.rb
-homepage: https://github.com/tacinight/logstash-output-qingstor
+homepage: https://github.com/yunify/logstash-output-qingstor
 licenses:
 - Apache License (2.0)
 metadata:

data/lib/logstash/outputs/qingstor/size_rotation_policy.rb

@@ -1,26 +0,0 @@
-# encoding: utf-8
-module LogStash
-  module Outputs
-    class Qingstor
-      class SizeRotationPolicy
-        attr_reader :size_file
-
-        def initialize(size_file)
-          if size_file <= 0
-            raise LogStash::ConfigurationError, "'size_file' need to be greater than 0"
-          end 
-
-          @size_file = size_file
-        end 
-
-        def rotate?(file)
-          file.size >= size_file
-        end 
-
-        def needs_periodic?
-          false 
-        end 
-      end 
-    end 
-  end 
-end