logstash-input-oss 0.0.1-java

data/lib/logstash/inputs/version.rb

@@ -0,0 +1,14 @@
+# encoding: utf-8
+
+module LogStash
+  module Inputs
+    class OSS
+      class Version
+        VERSION = "0.0.1"
+        def self.version
+          VERSION
+        end
+      end
+    end
+  end
+end

data/logstash-input-oss.gemspec

@@ -0,0 +1,33 @@
+Gem::Specification.new do |s|
+  s.name          = 'logstash-input-oss'
+  s.version       = '0.0.1'
+  s.licenses      = ['Apache-2.0']
+  s.summary       = 'Streams events from files in an OSS(Aliyun Object Storage Service) bucket'
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gem-name. This gem is not a stand-alone program"
+  s.authors       = ['Jinhu Wu']
+  s.email         = 'jinhu.wu.nju@gmail.com'
+  s.require_paths = ['lib']
+  s.homepage        = "http://www.elastic.co/guide/en/logstash/current/index.html"
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
+  s.add_runtime_dependency "logstash-codec-plain", "~> 3.0"
+  s.add_runtime_dependency "concurrent-ruby", "~> 1.0"
+  s.add_runtime_dependency "nokogiri", '~> 1.10', '>= 1.10.0'
+  s.add_runtime_dependency "rest-client", '~> 1.8', '>= 1.8.0'
+  s.add_runtime_dependency "uuid", '~> 2.3', '>= 2.3.9'
+  s.add_runtime_dependency 'stud', '~> 0.0.18'
+  s.add_development_dependency "logstash-devutils", "~> 1.3"
+  s.add_development_dependency "logstash-codec-line", "~> 3.0"
+  s.platform = 'java'
+  s.add_runtime_dependency 'jar-dependencies', '~> 0.3'
+  s.requirements << 'jar com.aliyun.oss:aliyun-sdk-oss, 3.4.0'
+end

data/spec/integration/common.rb

@@ -0,0 +1,79 @@
+# encoding: UTF-8
+
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/logging/logger'
+require 'logstash/inputs/oss'
+
+java_import 'com.aliyun.oss.model.GetObjectRequest'
+
+# This file contains the common logic used by integration tests
+shared_context "plugin initialize" do
+  let(:endpoint) { ENV['OSS_ENDPOINT'] }
+  let(:bucket) { ENV['OSS_BUCKET'] }
+  let(:access_key_id) { ENV['OSS_ACCESS_KEY'] }
+  let(:access_key_secret) { ENV['OSS_SECRET_KEY'] }
+  let(:backup_add_prefix) { 'input-oss/' }
+  let(:backup_to_bucket) { ENV['BACKUP_BUCKET'] }
+  let(:backup_to_dir) { ENV['BACKUP_DIR'] }
+  let(:common_configurations) do
+    {
+        "endpoint" => endpoint,
+        "bucket" => bucket,
+        "access_key_id" => access_key_id,
+        "access_key_secret" => access_key_secret,
+        "stop_for_test" => true,
+        "include_object_properties" => true,
+        "mns_settings" => {
+          "endpoint" => ENV['MNS_ENDPOINT'],
+          "queue" => ENV['MNS_QUEUE'],
+          "poll_interval_seconds" => 3,
+          "wait_seconds" => 3
+        }
+    }
+  end
+
+  LogStash::Logging::Logger::configure_logging("debug") if ENV["DEBUG"]
+
+  let(:oss) { OSSClientBuilder.new().build(endpoint, access_key_id, access_key_secret) }
+end
+
+def fetch_events(settings, size)
+  queue = []
+  input = LogStash::Inputs::OSS.new(settings)
+  input.register
+  thread = Thread.start do
+    input.run(queue)
+  end
+
+  thread.join
+
+  expect(queue.size).to eq(size)
+end
+
+# remove object with `prefix`
+def clean_bucket(bucket)
+  oss.listObjects(bucket, "").getObjectSummaries().each do |objectSummary|
+    oss.deleteObject(bucket, objectSummary.getKey())
+  end
+end
+
+def delete_bucket(bucket)
+  oss.deleteBucket(bucket)
+end
+
+def list_remote_files(bucket, prefix)
+  oss.listObjects(bucket, prefix).getObjectSummaries().collect(&:getKey)
+end
+
+def upload_local_file(local_file, remote_file_name)
+  file = File.join(File.dirname(__FILE__), local_file)
+  oss.putObject(bucket, remote_file_name, java.io.File.new(file))
+end
+
+def upload(prefix)
+  upload_local_file('../sample/uncompressed.log', "uncompressed.log/")
+  upload_local_file('../sample/uncompressed.log', "uncompressed.log")
+  upload_local_file('../sample/uncompressed.log', "uncompressed.log.1.gz")
+  upload_local_file('../sample/uncompressed.log', "#{prefix}uncompressed.log")
+  upload_local_file('../sample/uncompressed.log.1.gz', "#{prefix}exclude/uncompressed.log.1.gz")
+end

data/spec/integration/oss_spec.rb

@@ -0,0 +1,73 @@
+# encoding: utf-8
+require 'rspec'
+require_relative 'common'
+
+describe 'integration tests', :integration => true do
+  include_context 'plugin initialize'
+
+  before do
+    Thread.abort_on_exception = true
+  end
+
+  after do
+    clean_bucket(bucket)
+    FileUtils.rm_rf(backup_to_dir)
+  end
+
+  it "backup to another bucket" do
+    prefix = rand(999999).to_s + '/'
+    upload(prefix)
+    fetch_events(common_configurations.merge(
+        { "backup_to_bucket" => backup_to_bucket,
+          "backup_add_prefix" => backup_add_prefix,
+          "prefix" => prefix,
+        }), 74)
+    expect(list_remote_files(backup_to_bucket, backup_add_prefix).size).to eq(2)
+    expect(list_remote_files(bucket, prefix).size).to eq(2)
+    clean_bucket(backup_to_bucket)
+    delete_bucket(backup_to_bucket)
+  end
+
+  it "delete after backup" do
+    prefix = rand(999999).to_s + '/'
+    upload(prefix)
+    fetch_events(common_configurations.merge(
+        { "backup_to_bucket" => backup_to_bucket,
+          "backup_add_prefix" => backup_add_prefix,
+          "prefix" => prefix,
+          "delete" => true
+        }), 74)
+    expect(list_remote_files(backup_to_bucket, backup_add_prefix).size).to eq(2)
+    expect(list_remote_files(bucket, prefix).size).to eq(0)
+    clean_bucket(backup_to_bucket)
+    delete_bucket(backup_to_bucket)
+  end
+
+  it "backup to local dir" do
+    prefix = rand(999999).to_s + '/'
+    upload(prefix)
+    fetch_events(common_configurations.merge(
+        {
+            "backup_to_dir" => backup_to_dir,
+            "prefix" => prefix,
+            "delete" => true
+        }), 74)
+    expect(list_remote_files(bucket, prefix).size).to eq(0)
+    expect(Dir.glob("#{backup_to_dir}/**/*").size).to eq(4)
+  end
+
+  it "exclude pattern" do
+    prefix = rand(999999).to_s + '/'
+    upload(prefix)
+    fetch_events(common_configurations.merge(
+        { "backup_to_bucket" => backup_to_bucket,
+          "backup_add_prefix" => backup_add_prefix,
+          "prefix" => prefix,
+          "exclude_pattern" => "^" + prefix + "exclude"
+        }), 37)
+    expect(list_remote_files(backup_to_bucket, backup_add_prefix).size).to eq(1)
+    expect(list_remote_files(bucket, prefix).size).to eq(2)
+    clean_bucket(backup_to_bucket)
+    delete_bucket(backup_to_bucket)
+  end
+end

data/spec/sample/uncompressed.log

@@ -0,0 +1,37 @@
+2019-06-11 20:45:35,798 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.172:50010 to delete [blk_1073891466_150647]
+2019-06-11 20:45:35,798 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.174:50010 to delete [blk_1073891466_150647]
+2019-06-11 20:45:38,799 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.175:50010 to delete [blk_1073891466_150647]
+2019-06-11 20:46:35,800 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.172:50010 to delete [blk_1073891467_150648]
+2019-06-11 20:46:35,800 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.175:50010 to delete [blk_1073891467_150648]
+2019-06-11 20:46:38,801 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.173:50010 to delete [blk_1073891467_150648]
+2019-06-11 20:47:35,802 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.174:50010 to delete [blk_1073891468_150649]
+2019-06-11 20:47:35,802 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.175:50010 to delete [blk_1073891468_150649]
+2019-06-11 20:47:38,803 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.172:50010 to delete [blk_1073891468_150649]
+2019-06-11 20:48:41,804 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.175:50010 to delete [blk_1073891469_150650]
+2019-06-11 20:48:41,805 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.174:50010 to delete [blk_1073891469_150650]
+2019-06-11 20:48:44,805 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.173:50010 to delete [blk_1073891469_150650]
+2019-06-11 20:49:44,807 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.174:50010 to delete [blk_1073891470_150651]
+2019-06-11 20:49:44,807 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.172:50010 to delete [blk_1073891470_150651]
+2019-06-11 20:49:47,807 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.173:50010 to delete [blk_1073891470_150651]
+2019-06-11 20:50:44,808 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.174:50010 to delete [blk_1073891471_150652]
+2019-06-11 20:50:44,809 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.172:50010 to delete [blk_1073891471_150652]
+2019-06-11 20:50:47,809 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.175:50010 to delete [blk_1073891471_150652]
+2019-06-11 20:51:44,810 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.174:50010 to delete [blk_1073891472_150653]
+2019-06-11 20:51:44,810 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.173:50010 to delete [blk_1073891472_150653]
+2019-06-11 20:51:47,811 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.172:50010 to delete [blk_1073891472_150653]
+2019-06-11 20:52:44,812 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.173:50010 to delete [blk_1073891473_150654]
+2019-06-11 20:52:44,812 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.174:50010 to delete [blk_1073891473_150654]
+2019-06-11 20:52:47,813 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.172:50010 to delete [blk_1073891473_150654]
+2019-06-11 20:53:50,815 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.175:50010 to delete [blk_1073891474_150655]
+2019-06-11 20:53:50,815 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.174:50010 to delete [blk_1073891474_150655]
+2019-06-11 20:53:53,815 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.173:50010 to delete [blk_1073891474_150655]
+2019-06-11 20:54:56,817 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.173:50010 to delete [blk_1073891475_150656]
+2019-06-11 20:54:56,817 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.172:50010 to delete [blk_1073891475_150656]
+2019-06-11 20:54:59,817 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.175:50010 to delete [blk_1073891475_150656]
+2019-06-11 20:55:56,819 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.173:50010 to delete [blk_1073891476_150657]
+2019-06-11 20:55:56,819 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.174:50010 to delete [blk_1073891476_150657]
+2019-06-11 20:55:59,819 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.175:50010 to delete [blk_1073891476_150657]
+2019-06-11 20:56:56,821 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.173:50010 to delete [blk_1073891477_150658]
+2019-06-11 20:56:56,821 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.174:50010 to delete [blk_1073891477_150658]
+2019-06-11 20:56:59,821 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.172:50010 to delete [blk_1073891477_150658]
+2019-06-11 20:57:38,822 INFO BlockStateChange: BLOCK* BlockManager: ask 192.168.0.175:50010 to delete [blk_1073891482_150663]

metadata

@@ -0,0 +1,249 @@
+--- !ruby/object:Gem::Specification
+name: logstash-input-oss
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: java
+authors:
+- Jinhu Wu
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2019-07-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  name: logstash-codec-plain
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: concurrent-ruby
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.10.0
+  name: nokogiri
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.10.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.0
+  name: rest-client
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.3.9
+  name: uuid
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.3.9
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.18
+  name: stud
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.18
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  name: logstash-codec-line
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  name: jar-dependencies
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gem-name. This
+  gem is not a stand-alone program
+email: jinhu.wu.nju@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CONTRIBUTORS
+- DEVELOPER.md
+- Gemfile
+- LICENSE
+- README.md
+- lib/com/aliyun/aliyun-java-sdk-core/3.4.0/aliyun-java-sdk-core-3.4.0.jar
+- lib/com/aliyun/aliyun-java-sdk-ecs/4.2.0/aliyun-java-sdk-ecs-4.2.0.jar
+- lib/com/aliyun/aliyun-java-sdk-ram/3.0.0/aliyun-java-sdk-ram-3.0.0.jar
+- lib/com/aliyun/aliyun-java-sdk-sts/3.0.0/aliyun-java-sdk-sts-3.0.0.jar
+- lib/com/aliyun/oss/aliyun-sdk-oss/3.4.0/aliyun-sdk-oss-3.4.0.jar
+- lib/com/sun/jersey/jersey-core/1.9/jersey-core-1.9.jar
+- lib/com/sun/jersey/jersey-json/1.9/jersey-json-1.9.jar
+- lib/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar
+- lib/commons-codec/commons-codec/1.9/commons-codec-1.9.jar
+- lib/commons-logging/commons-logging/1.2/commons-logging-1.2.jar
+- lib/javax/activation/activation/1.1/activation-1.1.jar
+- lib/javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2.jar
+- lib/javax/xml/stream/stax-api/1.0-2/stax-api-1.0-2.jar
+- lib/logstash-input-oss_jars.rb
+- lib/logstash/inputs/mns/message.rb
+- lib/logstash/inputs/mns/request.rb
+- lib/logstash/inputs/oss.rb
+- lib/logstash/inputs/version.rb
+- lib/org/apache/httpcomponents/httpclient/4.4.1/httpclient-4.4.1.jar
+- lib/org/apache/httpcomponents/httpcore/4.4.1/httpcore-4.4.1.jar
+- lib/org/codehaus/jackson/jackson-core-asl/1.8.3/jackson-core-asl-1.8.3.jar
+- lib/org/codehaus/jackson/jackson-jaxrs/1.8.3/jackson-jaxrs-1.8.3.jar
+- lib/org/codehaus/jackson/jackson-mapper-asl/1.8.3/jackson-mapper-asl-1.8.3.jar
+- lib/org/codehaus/jackson/jackson-xc/1.8.3/jackson-xc-1.8.3.jar
+- lib/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar
+- lib/org/jdom/jdom/1.1/jdom-1.1.jar
+- lib/org/json/json/20170516/json-20170516.jar
+- lib/stax/stax-api/1.0.1/stax-api-1.0.1.jar
+- logstash-input-oss.gemspec
+- spec/integration/common.rb
+- spec/integration/oss_spec.rb
+- spec/sample/uncompressed.log
+- spec/sample/uncompressed.log.1.gz
+homepage: http://www.elastic.co/guide/en/logstash/current/index.html
+licenses:
+- Apache-2.0
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: input
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements:
+- jar com.aliyun.oss:aliyun-sdk-oss, 3.4.0
+rubyforge_project:
+rubygems_version: 2.7.6
+signing_key:
+specification_version: 4
+summary: Streams events from files in an OSS(Aliyun Object Storage Service) bucket
+test_files:
+- spec/integration/common.rb
+- spec/integration/oss_spec.rb
+- spec/sample/uncompressed.log
+- spec/sample/uncompressed.log.1.gz