logstash-input-qingstor 0.1.3 → 0.1.5

data/lib/logstash/inputs/qingstor/log_reader.rb

@@ -0,0 +1,53 @@
+require 'logstash/inputs/qingstor'
+require 'zlib'
+
+module LogStash
+  module Inputs
+    class Qingstor
+      # define class LogReader to read log files
+      class LogReader
+        attr_accessor :filepath
+
+        def initialize(filepath)
+          @filepath = filepath
+        end
+
+        def read_file(&block)
+          if gzip?(@filepath)
+            read_gzip_file(block)
+          else
+            read_plain_file(block)
+          end
+        end
+
+        def read_gzip_file(block)
+          Zlib::GzipReader.open(@filepath) do |decoder|
+            decoder.each_line { |line| block.call(line) }
+          end
+        rescue Zlib::Error, Zlib::GzipFile::Error => e
+          @logger.error('Gzip codec: Cannot uncompress the file',
+                        :filepath => @filepath)
+          raise e
+        end
+
+        def read_plain_file(block)
+          ::File.open(@filepath, 'rb') do |file|
+            file.each(&block)
+          end
+        end
+
+        def valid_format?(filepath)
+          logger?(filepath) || gzip?(filepath)
+        end
+
+        def logger?(filepath)
+          filepath.end_with?('.log', '.txt')
+        end
+
+        def gzip?(filepath)
+          filepath.end_with?('.gz')
+        end
+      end # class LogReader
+    end # class QingStor
+  end #  module Inputs
+end # module LogStash

data/lib/logstash/inputs/qingstor/qingstor_validator.rb

@@ -1,30 +1,35 @@
-# encoding: utf-8
-require "qingstor/sdk"
-require "fileutils"
+require 'logstash/inputs/qingstor'
+require 'qingstor/sdk'
 
-module LogStash
-  module Inputs
-    class Qingstor
-      class QingstorValidator
+# Validator for check the avaliablity of setting in QingStor
+module QingstorValidator
+  def bucket_valid?(bucket)
+    res = bucket.head
+    case res[:status_code]
+    when 401
+      raise LogStash::ConfigurationError,
+            'Incorrect key id or access key.'
+    when 404
+      raise LogStash::ConfigurationError,
+            'Incorrect bucket/region name.'
+    end
+    true
+  end
 
-        def self.bucket_valid?(bucket)
-            res = bucket.head
-            case res[:status_code] 
-            when 401
-              raise LogStash::ConfigurationError, "Incorrect key id or access key."
-            when 404
-              raise LogStash::ConfigurationError, "Incorrect bucket/region name."
-            end 
-            true
-        end 
+  def prefix_valid?(prefix)
+    if prefix.start_with?('/') || prefix.length >= 1024
+      raise LogStash::ConfigurationError, 'Prefix must not start with '\
+          + "'/' with length less than 1024"
+    end
+    true
+  end
 
-        def self.prefix_valid?(prefix)
-          if prefix.start_with?("/") || prefix.length >= 1024
-            raise LogStash::ConfigurationError, "Prefix must not start with '/' with length less than 1024 "
-          end 
-          true
-        end 
-      end 
-    end 
-  end 
-end 
+  def create_if_not_exist(bucket)
+    return if bucket.head[:status_code] == 200
+    res = bucket.put
+    if res[:status_code] != 201
+      @logger.error('ERROR : cannot create the bucket ', res[:message])
+      raise LogStash::ConfigurationError, 'cannot create the bucket'
+    end
+  end # def create_if_not_exist
+end # module QingstorValidator

data/lib/logstash/inputs/qingstor/sincedb.rb

@@ -0,0 +1,36 @@
+require 'logstash/inputs/qingstor'
+require 'fileutils'
+
+# module used for record the download history
+module LogStash
+  module Inputs
+    class Qingstor
+      # define the class SinceDB::File
+      class SinceDB
+        def initialize(file)
+          @sincedb_path = file
+        end
+
+        def newer?(date)
+          Time.at(date) > read
+        end
+
+        def read
+          if ::File.exist?(@sincedb_path)
+            content = ::File.read(@sincedb_path).chomp.strip
+            content.empty? ? Time.new(0) : Time.parse(content)
+          else
+            Time.new(0)
+          end
+        end
+
+        def write(since = nil)
+          since = Time.now if since.nil?
+          dir = ::File.dirname(@sincedb_path)
+          FileUtils.mkdir_p(dir) unless ::File.directory?(dir)
+          ::File.open(@sincedb_path, 'w') { |file| file.write(since.to_s) }
+        end
+      end # class FILE
+    end # class QingStor
+  end # module Inputs
+end # module LogStash

data/lib/logstash/inputs/qingstor/uploader.rb

@@ -0,0 +1,56 @@
+require 'logstash/inputs/qingstor'
+require 'qingstor/sdk'
+require 'concurrent'
+
+module LogStash
+  module Inputs
+    class Qingstor
+      # define class Uploader to process upload jobs
+      class Uploader
+        require 'logstash/inputs/qingstor/qingstor_validator'
+        include QingstorValidator
+
+        TIME_BEFORE_RETRYING_SECONDS = 1
+        DEFAULT_THREADPOOL = Concurrent::ThreadPoolExecutor.new(
+          :min_thread => 1,
+          :max_thread => 8,
+          :max_queue => 2,
+          :fallback_policy => :caller_runs
+        )
+
+        attr_reader :bucket, :prefix, :logger
+
+        def initialize(bucket, prefix, logger)
+          @bucket = bucket
+          @prefix = prefix
+          @logger = logger
+          @workers_pool = DEFAULT_THREADPOOL
+        end
+
+        def upload_async(filename, filepath)
+          @workers_pool.post do
+            upload(filename, filepath)
+          end
+        end
+
+        def upload(filename, filepath)
+          create_if_not_exist(@bucket)
+          file_md5 = Digest::MD5.file(filepath).to_s
+          key = if @prefix.end_with?('/') || @prefix.empty?
+                  @prefix + filename
+                else
+                  @prefix + '/' + filename
+                end
+          @logger.debug('uploading backup file', :file => filename)
+          @bucket.put_object(key, 'content_md5' => file_md5,
+                                  'body' => ::File.open(filepath))
+        end
+
+        def stop
+          @workers_pool.shutdown
+          @workers_pool.wait_for_termination(nil)
+        end
+      end # class Uploader
+    end # class QingStor
+  end # module Inputs
+end # module LogStash

data/logstash-input-qingstor.gemspec

@@ -1,27 +1,29 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-input-qingstor'
-  s.version       = '0.1.3'
+  s.version       = '0.1.5'
   s.licenses      = ['Apache License (2.0)']
   s.summary       = 'logstash input plugin for QingStor'
-  s.description   = 'Use this plugin to fetch file from Qingstor as the input of logstash'
-  s.homepage      = 'https://github.com/Tacinight/logstash-input-qingstor'
+  s.description   = 'Fetch file from Qingstor as the input of logstash'
+  s.homepage      = 'https://github.com/yunify/logstash-input-qingstor'
   s.authors       = ['Evan Zhao']
   s.email         = 'tacingiht@gmail.com'
   s.require_paths = ['lib']
 
   # Files
-  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
-   # Tests
+  s.files = Dir['lib/**/*', 'spec/**/*', 'vendor/**/*', '*.gemspec', '*.md',
+                'CONTRIBUTORS', 'Gemfile', 'LICENSE', 'NOTICE.TXT']
+
+  # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
 
   # Special flag to let us know this is actually a logstash plugin
-  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
+  s.metadata = { 'logstash_plugin' => 'true', 'logstash_group' => 'input' }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core-plugin-api", ">=1.6", "<=2.99"
+  s.add_runtime_dependency 'logstash-core-plugin-api', '>=1.6', '<=2.99'
   s.add_runtime_dependency 'logstash-codec-plain'
   s.add_runtime_dependency 'stud', '>= 0.0.22'
-  s.add_runtime_dependency "qingstor-sdk", ">=1.9.2"
+  s.add_runtime_dependency 'qingstor-sdk', '>=1.9.2'
 
   s.add_development_dependency 'logstash-devutils'
-end
+end

data/spec/logstash/inputs/qingstor/log_reader_spec.rb

@@ -0,0 +1,48 @@
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/inputs/qingstor/log_reader'
+require 'tmpdir'
+
+describe LogStash::Inputs::Qingstor::LogReader do
+  subject(:log_reader) { described_class.new('/a/example/path') }
+
+  let(:content) { 'may the code be with you!' }
+  let(:plain_file_path) { File.join(Dir.tmpdir, 'plain.log') }
+  let(:gzip_file_path) { File.join(Dir.tmpdir, 'gzip.gz') }
+  let(:invalid_file_path) { File.join(Dir.tmpdir, 'invalid.ivd') }
+
+  context 'when read plain file' do
+    before do
+      File.open(plain_file_path, 'w') do |f|
+        f.write(content)
+      end
+    end
+
+    it do
+      log_reader.filepath = plain_file_path
+      log_reader.read_file do |f|
+        expect(f).to eq(content)
+      end
+    end
+  end
+
+  context 'when read gzip file' do
+    before do
+      Zlib::GzipWriter.open(gzip_file_path) do |gz|
+        gz.write(content)
+      end
+    end
+
+    it do
+      log_reader.filepath = gzip_file_path
+      log_reader.read_file do |f|
+        expect(f).to eq(content)
+      end
+    end
+  end
+
+  context 'when valid format' do
+    it { expect(log_reader.valid_format?(plain_file_path)).to be_truthy }
+    it { expect(log_reader.valid_format?(gzip_file_path)).to be_truthy }
+    it { expect(log_reader.valid_format?(invalid_file_path)).to be_falsey }
+  end
+end

data/spec/logstash/inputs/qingstor/sincedb_spec.rb

@@ -0,0 +1,27 @@
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/inputs/qingstor/sincedb'
+require 'tmpdir'
+
+describe LogStash::Inputs::Qingstor::SinceDB do
+  subject(:sincedb) { described_class.new(sincedb_path) }
+
+  let(:sincedb_path) { File.join(Dir.tmpdir, 'log_tmp_dir/log_tmp.log') }
+
+  context 'when run at first time' do
+    before do
+      File.delete(sincedb_path) if File.exist?(sincedb_path)
+    end
+
+    it { expect(sincedb.read).to eq(Time.new(0)) }
+    it { expect(sincedb.newer?(Time.now)).to be_truthy }
+  end
+
+  context 'when write the record' do
+    it do
+      time = Time.now
+      sincedb.write(time)
+      content = File.read(sincedb_path).chomp.strip
+      expect(content).to eq(time.to_s)
+    end
+  end
+end

data/spec/logstash/inputs/qingstor/uploader_spec.rb

@@ -0,0 +1,42 @@
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/inputs/qingstor/uploader'
+require 'qingstor/sdk'
+require 'stud/temporary'
+require_relative '../qs_access_helper'
+
+describe LogStash::Inputs::Qingstor::Uploader do
+  let(:bucket) { qs_init_bucket }
+  let(:new_bucket) { qs_init_bucket }
+  let(:key) { 'foobar' }
+  let(:file) { Stud::Temporary.file }
+  let(:filepath) { file.path }
+  let(:logger) { spy(:logger) }
+
+  context 'when upload file' do
+    let(:prefix) { '' }
+
+    after do
+      delete_remote_file(prefix + key)
+    end
+
+    it do
+      uploader = described_class.new(bucket, prefix, logger)
+      uploader.upload(key, filepath)
+      expect(list_remote_file.size).to eq(1)
+    end
+  end
+
+  context 'when upload file with a prefix' do
+    let(:prefix) { 'a/prefix/' }
+
+    after do
+      delete_remote_file(prefix + key)
+    end
+
+    it do
+      uploader = described_class.new(bucket, prefix, logger)
+      uploader.upload(key, filepath)
+      expect(list_remote_file.size).to eq(1)
+    end
+  end
+end

data/spec/logstash/inputs/qingstor_spec.rb

@@ -0,0 +1,98 @@
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/inputs/qingstor'
+require_relative './qs_access_helper'
+require 'tmpdir'
+
+describe LogStash::Inputs::Qingstor do
+  before do
+    Thread.abort_on_exception = true
+
+    upload_file('../../fixtures/logstash.log', 'log3.log')
+    upload_file('../../fixtures/logstash.log.gz', 'log3.log.gz')
+  end
+
+  after do
+    delete_remote_file 'log3.log'
+    delete_remote_file 'log3.log.gz'
+  end
+
+  let(:config) do
+    { 'access_key_id' => ENV['access_key_id'],
+      'secret_access_key' => ENV['secret_access_key'],
+      'bucket' => ENV['bucket'],
+      'region' => ENV['region'] }
+  end
+
+  let(:key1) { 'log3.log' }
+  let(:key2) { 'log3.log.gz' }
+  let(:backup) { 'evamax' }
+  let(:local_backup_dir) { File.join(Dir.tmpdir, backup) }
+
+  context 'when at the local' do
+    it 'backup to local dir' do
+      fetch_events(config.merge('backup_local_dir' => local_backup_dir))
+      expect(File.exist?(File.join(local_backup_dir, key1))).to be_truthy
+      expect(File.exist?(File.join(local_backup_dir, key2))).to be_truthy
+    end
+
+    after do
+      FileUtils.rm_r(File.join(local_backup_dir, key1))
+      FileUtils.rm_r(File.join(local_backup_dir, key2))
+    end
+  end
+
+  context 'when backup to the remote end' do
+    it do
+      fetch_events(config.merge('backup_bucket' => backup))
+      expect(list_remote_file(backup).size).to eq(2)
+    end
+
+    after do
+      clean_and_delete_bucket(backup)
+    end
+  end
+
+  context 'when test host redirection' do
+    it 'redirect without a port number' do
+      expect { fetch_events(config.merge('host' => 'qingstor.dev')) }
+        .to raise_error(Net::HTTP::Persistent::Error)
+    end
+
+    it 'redirect with a port number' do
+      new_config = config.merge('host' => 'qingstor.dev', 'port' => 444)
+      expect { fetch_events(new_config) }
+        .to raise_error(Net::HTTP::Persistent::Error)
+    end
+  end
+
+  context 'when test with various config values' do
+    it do
+      config['access_key_id'] = 'wrongid'
+      expect { described_class.new(config).register }
+        .to raise_error(LogStash::ConfigurationError)
+    end
+
+    it do
+      config['secret_access_key'] = 'wrongaccesskey'
+      expect { described_class.new(config).register }
+        .to raise_error(LogStash::ConfigurationError)
+    end
+
+    it do
+      config['bucket'] = 'wrongbucket'
+      expect { described_class.new(config).register }
+        .to raise_error(LogStash::ConfigurationError)
+    end
+
+    it do
+      config['region'] = 'wrongregion'
+      expect { described_class.new(config).register }
+        .to raise_error(LogStash::ConfigurationError)
+    end
+
+    it do
+      config.delete('region')
+      expect(described_class.new(config).register).to be_truthy
+    end
+  end
+end