logstash-output-qingstor 0.1.0

data/spec/outputs/qingstor/uploader_spec.rb

@@ -0,0 +1,49 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/qingstor/uploader"
+require "logstash/outputs/qingstor/temporary_file"
+require "qingstor/sdk"
+require "stud/temporary"
+
+require_relative "../qs_access_helper"
+
+describe LogStash::Outputs::Qingstor::Uploader do 
+  let(:bucket) { qs_init_bucket }
+  let(:key) { "foobar" }
+  let(:tmp_file) { Stud::Temporary.file }
+  let(:tmp_path) { tmp_file.path }
+  let(:logger) { spy(:logger) }
+
+  let(:threadpool) do 
+    Concurrent::ThreadPoolExecutor.new({
+                                         :min_threads => 1,
+                                         :max_threads => 4,
+                                         :max_queue => 1,
+                                         :fallback_policy => :caller_runs
+                                       })
+  end 
+  
+  let(:file) do 
+    f = LogStash::Outputs::Qingstor::TemporaryFile.new(key, tmp_file, tmp_path)
+    f.write("May the code be with you!")
+    f.fsync
+    f
+  end
+
+  subject { described_class.new(bucket, logger, threadpool) }
+
+  after do 
+    delete_remote_file key 
+  end 
+
+  it "upload file to the qingstor bucket" do 
+    subject.upload(file)
+    expect(list_remote_file.size).to eq(3)
+  end 
+
+  it "execute a callback when the upload is complete" do 
+    callback = proc { |f| }
+    expect(callback).to receive(:call).with(file)
+    subject.upload(file, { :on_complete => callback })
+  end 
+end 

data/spec/outputs/qingstor_spec.rb

@@ -0,0 +1,50 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/qingstor"
+require "logstash/event"
+require "openssl"
+require "fileutils"
+require_relative "./qs_access_helper"
+require_relative "./spec_helper"
+
+describe LogStash::Outputs::Qingstor do
+  let(:prefix) { "ss/%{server}"}
+  let(:event) { LogStash::Event.new({ "server" => "overwatch" }) }
+  let(:event_encoded) { "May the code be with you!" }
+  let(:events_and_encoded) {{ event => event_encoded}}
+  let(:options) {{
+      "access_key_id" => ENV['access_key_id'],
+      "secret_access_key" => ENV['secret_access_key'],
+      "bucket" => ENV['bucket'],
+      "region" => ENV['region'],
+      "prefix" => prefix
+  }}
+  let(:tmpdir){File.join(Dir.tmpdir, "logstash_restore_dir")}
+
+  after do 
+    clean_remote_files
+  end 
+
+  it "done work with minimal options" do 
+    fetch_event(options, events_and_encoded)
+    expect(list_remote_file.size).to eq(1)
+  end 
+
+  it "use aes256 to encrpytion in the server side" do 
+    cipher = OpenSSL::Cipher::AES256.new(:CBC)
+    cipher.encrypt
+    key = cipher.random_key
+    fetch_event(options.merge({"server_side_encryption_algorithm" => "AES256","customer_key" => key}), events_and_encoded)
+    expect(list_remote_file.size).to eq(1)
+  end 
+
+  it "upload existing file if turn on restore function" do 
+    FileUtils.mkdir_p(File.join(tmpdir, "uuid_dir")) unless File.exists?(File.join(tmpdir, "uuid_dir"))
+    f = File.open(File.join(tmpdir, "uuid_dir/temp"), 'w')
+    f.write(event_encoded * 10)
+    f.close
+    fetch_event(options.merge({"restore" => true, "tmpdir" => tmpdir}), events_and_encoded)
+    expect(list_remote_file.size).to eq(2)
+  end 
+
+end 

data/spec/outputs/qs_access_helper.rb

@@ -0,0 +1,38 @@
+def fetch_event(settings, events_and_encoded) 
+  qs = LogStash::Outputs::Qingstor.new(settings)
+  qs.register
+  qs.multi_receive_encoded(events_and_encoded)
+  qs.close
+end 
+
+def qs_init_config(access_key_id = ENV['access_key_id'],
+                   secret_access_key = ENV['secret_access_key'])
+  QingStor::SDK::Config.init access_key_id, secret_access_key
+end 
+
+def qs_init_bucket(bucket = ENV['bucket'], region = ENV['region'])
+  config = qs_init_config
+  properties = {'bucket-name' => bucket, 'zone' => region }
+  QingStor::SDK::Bucket.new(config, properties)
+end 
+
+def delete_remote_file(key)
+  bucket = qs_init_bucket
+  bucket.delete_object key
+end
+
+def list_remote_file(bucket = ENV['bucket'], region = ENV['region'])
+  bucket = qs_init_bucket(bucket, region)
+  bucket.list_objects[:keys]
+end
+
+def delete_bucket(bucket)
+  bucket = qs_init_bucket(bucket)
+  bucket.delete
+end 
+
+def clean_remote_files
+  list_remote_file.each do |record|
+    qs_init_bucket.delete_object record[:key]
+  end 
+end 

data/spec/outputs/spec_helper.rb

@@ -0,0 +1,4 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/logging/logger"
+
+LogStash::Logging::Logger::configure_logging("debug")

metadata

@@ -0,0 +1,169 @@
+--- !ruby/object:Gem::Specification
+name: logstash-output-qingstor
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Evan Zhao
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2017-03-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-plain
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.9.2
+  name: qingstor-sdk
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.9.2
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: concurrent-ruby
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.22
+  name: stud
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.22
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Put the outcomes of logstash into qingstor
+email: tacingiht@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CODE_OF_CONDUCT.md
+- CONTRIBUTING.md
+- CONTRIBUTORS
+- DEVELOPER.md
+- Gemfile
+- LICENSE
+- README.md
+- lib/logstash/outputs/qingstor.rb
+- lib/logstash/outputs/qingstor/file_repository.rb
+- lib/logstash/outputs/qingstor/qingstor_validator.rb
+- lib/logstash/outputs/qingstor/size_and_time_rotation_policy.rb
+- lib/logstash/outputs/qingstor/size_rotation_policy.rb
+- lib/logstash/outputs/qingstor/temporary_file.rb
+- lib/logstash/outputs/qingstor/temporary_file_factory.rb
+- lib/logstash/outputs/qingstor/time_rotation_policy.rb
+- lib/logstash/outputs/qingstor/uploader.rb
+- logstash-output-qingstor.gemspec
+- spec/outputs/qingstor/file_repository_spec.rb
+- spec/outputs/qingstor/qingstor_validator_spec.rb
+- spec/outputs/qingstor/size_and_time_rotation_policy_spec.rb
+- spec/outputs/qingstor/size_rotation_policy_spec.rb
+- spec/outputs/qingstor/temporary_file_factory_spec.rb
+- spec/outputs/qingstor/temporary_file_spec.rb
+- spec/outputs/qingstor/time_rotation_policy_spec.rb
+- spec/outputs/qingstor/uploader_spec.rb
+- spec/outputs/qingstor_spec.rb
+- spec/outputs/qs_access_helper.rb
+- spec/outputs/spec_helper.rb
+homepage: https://github.com/tacinight/logstash-output-qingstor
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: output
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.8
+signing_key:
+specification_version: 4
+summary: logstash output plugin for qingstor
+test_files:
+- spec/outputs/qingstor/file_repository_spec.rb
+- spec/outputs/qingstor/qingstor_validator_spec.rb
+- spec/outputs/qingstor/size_and_time_rotation_policy_spec.rb
+- spec/outputs/qingstor/size_rotation_policy_spec.rb
+- spec/outputs/qingstor/temporary_file_factory_spec.rb
+- spec/outputs/qingstor/temporary_file_spec.rb
+- spec/outputs/qingstor/time_rotation_policy_spec.rb
+- spec/outputs/qingstor/uploader_spec.rb
+- spec/outputs/qingstor_spec.rb
+- spec/outputs/qs_access_helper.rb
+- spec/outputs/spec_helper.rb