logstash-input-oss 0.0.1-java

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: f3623c612fe4e3a2267316232b1eb5af798ae26656d1299eb5508582a3982d9c
+  data.tar.gz: 07a9bfe7d7ae1ad99f300ec7020a612f1570d05ddfaed4daa2abb517eecfc2f2
+SHA512:
+  metadata.gz: 2fea65a2a8690cbce0d40a86843e2ff9457d8e042a28074d30238b5ffa01098264adcc5912669304ffb55b4dc49cb1ebba69f48691d18aa4bb64211dd18488e5
+  data.tar.gz: 55ba1325ca09e9a1f4fe51fce2bc976d30dcba1c1d7d6e70618e3406fc71249be34966843bf86fb4dc8e190ce2ac0316b19e12f95e02374228969d1c923cf79a

data/CHANGELOG.md

@@ -0,0 +1,2 @@
+## 0.0.1
+  - First release to support OSS

data/CONTRIBUTORS

@@ -0,0 +1,10 @@
+The following is a list of people who have contributed ideas, code, bug
+reports, or in general have helped logstash along its way.
+
+Contributors:
+* jinhu.wu - jinhu.wu.nju@gmail.com
+
+Note: If you've sent us patches, bug reports, or otherwise contributed to
+Logstash, and you aren't on the list above and want to be, please let us know
+and we'll make sure you're here. Contributions from folks like you are what make
+open source awesome.

data/DEVELOPER.md

@@ -0,0 +1,10 @@
+# Running the tests
+```bash
+bundle install
+bundle exec rspec
+```
+
+If you want to run the integration tests against a real bucket you need to pass your Aliyun credentials to the test runner or declare it in your environment.
+```bash
+OSS_ENDPOINT="Aliyun OSS endpoint to connect to" OSS_ACCESS_KEY="Your access key id" OSS_SECRET_KEY="Your access secret" OSS_BUCKET="Your bucket" MNS_ENDPOINT="MNS endpoint" MNS_QUEUE="MNS queue" BACKUP_DIR="/tmp/oss" BACKUP_BUCKET="Your backup bucket" bundle exec rspec spec/integration/oss_spec.rb --tag integration
+```

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,11 @@
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,143 @@
+# Logstash OSS Input Plugin
+
+This is a plugin for [Logstash](https://github.com/elastic/logstash).
+
+It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
+
+## Documentation
+
+This plugin reads data from OSS periodically.
+
+This plugin uses MNS on the same region of the OSS bucket. We must setup MNS and OSS event notification before using this plugin.
+
+[This document](https://help.aliyun.com/document_detail/52656.html) shows how to setup MNS and OSS event notification.
+
+This plugin will poll events from MNS queue and extract object keys from these events, and then will read those objects from OSS.
+
+### Usage:
+This is an example of logstash config:
+
+```
+input {
+  oss {
+    "endpoint" => "OSS endpoint to connect to"                     (required)
+    "bucket" => "Your bucket name"                                 (required)
+    "access_key_id" => "Your access key id"                        (required)
+    "access_key_secret" => "Your secret key"                       (required)
+    "backup_to_bucket" => "Your backup bucket name"                (optional, default = nil)
+    "prefix" => "sample"                                           (optional, default = nil)
+    "delete" => false                                              (optional, default = false)
+    "exclude_pattern" => "^sample-logstash"                        (optional, default = nil)
+    "backup_add_prefix" => "logstash-input/"                       (optional, default = nil)
+    "include_object_properties" => true                            (optional, default = false)
+    "additional_oss_settings" => {
+      "max_connections_to_oss" => 1024                             (optional, default = 1024)
+      "secure_connection_enabled" => false                         (optional, default = false)
+    }
+    "mns_settings" => {                                            (required)
+      "endpoint" => "MNS endpoint to connect to"                   (required)
+      "queue" => "MNS queue name"                                  (required)
+      "poll_interval_seconds" => 10                                (optional, default = 10)
+      "wait_seconds" => 3                                          (optional, default = nil)
+    }
+    codec => json {
+      charset => "UTF-8"
+    }
+  }
+}
+```
+### Logstash OSS Input Configuration Options
+This plugin supports the following configuration options
+
+Note: Files end with `.gz` or `.gzip` are handled as gzip'ed files, others are handled as plain files.
+
+|Configuration|Type|Required|Comments|
+|:---:|:---:|:---:|:---|
+|endpoint|string|Yes|OSS endpoint to connect|
+|bucket|string|Yes|Your OSS bucket name|
+|access_key_id|string|Yes|Your access key id|
+|access_key_secret|string|Yes|Your access secret key|
+|prefix|string|No|If specified, the prefix of filenames in the bucket must match (not a regexp)|
+|additional_oss_settings|hash|No|Additional oss client configurations, valid keys are: `secure_connection_enabled` and `max_connections_to_oss`|
+|delete|boolean|No|Whether to delete processed files from the original bucket|
+|backup_to_bucket|string|No|Name of an OSS bucket to backup processed files to|
+|backup_to_dir|string|No|Path of a local directory to backup processed files to|
+|backup_add_prefix|string|No|Append a prefix to the key (full path including file name in OSS) after processing(If backing up to another (or the same) bucket, this effectively lets you choose a new 'folder' to place the files in)|
+|include_object_properties|boolean|No|Whether or not to include the OSS object's properties (last_modified, content_type, metadata) into each Event at [@metadata][oss]. Regardless of this setting, [@metadata][oss][key] will always be present|
+|exclude_pattern|string|No|Ruby style regexp of keys to exclude from the bucket|
+|mns_settings|hash|Yes|MNS settings, valid keys are: `endpoint`, `queue`, `poll_interval_seconds`, `wait_seconds`|
+
+#### mns_settings
+
+[MNS consume messages](https://help.aliyun.com/document_detail/35136.html)
+
+* endpoint
+* queue
+* wait_seconds
+* poll_interval_seconds Poll messages interval from MNS if there is no message this time, default 10 seconds
+
+For more details about mns configurations, please view MNS documentation in the link above.
+
+## Need Help?
+
+Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/logstash discussion forum.
+
+## Developing
+
+### 1. Plugin Development and Testing
+
+#### Code
+- To get started, you'll need JRuby with the Bundler gem installed.
+
+- Install dependencies
+```sh
+bundle install
+```
+
+#### Test
+
+- Update your dependencies
+
+```sh
+bundle install
+```
+
+- Run tests
+
+```sh
+bundle exec rspec
+```
+
+### 2. Running your unpublished Plugin in Logstash
+
+#### 2.1 Run in an installed Logstash
+
+you can build the gem and install it using:
+
+- Build your plugin gem
+
+```sh
+gem build logstash-input-oss.gemspec
+```
+
+- Install the plugin from the Logstash home
+
+```sh
+bin/logstash-plugin install /path/to/logstash-input-oss-0.0.1-java.gem
+```
+
+- Start Logstash and proceed to test the plugin
+
+```bash
+./bin/logstash -f config/logstash-sample.conf
+```
+
+## Contributing
+
+All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin.
+
+Programming is not a required skill. Whatever you've seen about open source and maintainers or community members  saying "send patches or die" - you will not see that here.
+
+It is more important to the community that you are able to contribute.
+
+For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.

data/lib/logstash-input-oss_jars.rb

@@ -0,0 +1,54 @@
+# this is a generated file, to avoid over-writing it just delete this comment
+begin
+  require 'jar_dependencies'
+rescue LoadError
+  require 'org/apache/httpcomponents/httpcore/4.4.1/httpcore-4.4.1.jar'
+  require 'com/aliyun/oss/aliyun-sdk-oss/3.4.0/aliyun-sdk-oss-3.4.0.jar'
+  require 'commons-codec/commons-codec/1.9/commons-codec-1.9.jar'
+  require 'org/codehaus/jackson/jackson-xc/1.8.3/jackson-xc-1.8.3.jar'
+  require 'com/aliyun/aliyun-java-sdk-ram/3.0.0/aliyun-java-sdk-ram-3.0.0.jar'
+  require 'org/apache/httpcomponents/httpclient/4.4.1/httpclient-4.4.1.jar'
+  require 'javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2.jar'
+  require 'com/sun/jersey/jersey-json/1.9/jersey-json-1.9.jar'
+  require 'org/codehaus/jettison/jettison/1.1/jettison-1.1.jar'
+  require 'com/sun/jersey/jersey-core/1.9/jersey-core-1.9.jar'
+  require 'javax/xml/stream/stax-api/1.0-2/stax-api-1.0-2.jar'
+  require 'org/json/json/20170516/json-20170516.jar'
+  require 'com/aliyun/aliyun-java-sdk-core/3.4.0/aliyun-java-sdk-core-3.4.0.jar'
+  require 'commons-logging/commons-logging/1.2/commons-logging-1.2.jar'
+  require 'org/codehaus/jackson/jackson-jaxrs/1.8.3/jackson-jaxrs-1.8.3.jar'
+  require 'javax/activation/activation/1.1/activation-1.1.jar'
+  require 'com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar'
+  require 'org/jdom/jdom/1.1/jdom-1.1.jar'
+  require 'org/codehaus/jackson/jackson-mapper-asl/1.8.3/jackson-mapper-asl-1.8.3.jar'
+  require 'org/codehaus/jackson/jackson-core-asl/1.8.3/jackson-core-asl-1.8.3.jar'
+  require 'com/aliyun/aliyun-java-sdk-ecs/4.2.0/aliyun-java-sdk-ecs-4.2.0.jar'
+  require 'com/aliyun/aliyun-java-sdk-sts/3.0.0/aliyun-java-sdk-sts-3.0.0.jar'
+  require 'stax/stax-api/1.0.1/stax-api-1.0.1.jar'
+end
+
+if defined? Jars
+  require_jar 'org.apache.httpcomponents', 'httpcore', '4.4.1'
+  require_jar 'com.aliyun.oss', 'aliyun-sdk-oss', '3.4.0'
+  require_jar 'commons-codec', 'commons-codec', '1.9'
+  require_jar 'org.codehaus.jackson', 'jackson-xc', '1.8.3'
+  require_jar 'com.aliyun', 'aliyun-java-sdk-ram', '3.0.0'
+  require_jar 'org.apache.httpcomponents', 'httpclient', '4.4.1'
+  require_jar 'javax.xml.bind', 'jaxb-api', '2.2.2'
+  require_jar 'com.sun.jersey', 'jersey-json', '1.9'
+  require_jar 'org.codehaus.jettison', 'jettison', '1.1'
+  require_jar 'com.sun.jersey', 'jersey-core', '1.9'
+  require_jar 'javax.xml.stream', 'stax-api', '1.0-2'
+  require_jar 'org.json', 'json', '20170516'
+  require_jar 'com.aliyun', 'aliyun-java-sdk-core', '3.4.0'
+  require_jar 'commons-logging', 'commons-logging', '1.2'
+  require_jar 'org.codehaus.jackson', 'jackson-jaxrs', '1.8.3'
+  require_jar 'javax.activation', 'activation', '1.1'
+  require_jar 'com.sun.xml.bind', 'jaxb-impl', '2.2.3-1'
+  require_jar 'org.jdom', 'jdom', '1.1'
+  require_jar 'org.codehaus.jackson', 'jackson-mapper-asl', '1.8.3'
+  require_jar 'org.codehaus.jackson', 'jackson-core-asl', '1.8.3'
+  require_jar 'com.aliyun', 'aliyun-java-sdk-ecs', '4.2.0'
+  require_jar 'com.aliyun', 'aliyun-java-sdk-sts', '3.0.0'
+  require_jar 'stax', 'stax-api', '1.0.1'
+end

data/lib/logstash/inputs/mns/message.rb

@@ -0,0 +1,53 @@
+# encoding: utf-8
+require 'rexml/document'
+
+module LogStash
+  module Inputs
+    module MNS
+      # Class for Aliyun MNS Message.
+      class Message
+        include REXML
+
+        attr_reader :queue, :id, :body_md5, :body, :receipt_handle, :enqueue_at,
+                    :first_enqueue_at, :next_visible_at, :dequeue_count, :priority
+
+        def initialize(queue, content)
+          @queue = queue
+
+          doc = Document.new(content)
+          doc.elements[1].each do |e|
+            if e.node_type == :element
+              if e.name == 'MessageId'
+                @id = e.text
+              elsif e.name == 'MessageBodyMD5'
+                @body_md5 = e.text
+              elsif e.name == 'MessageBody'
+                @body = e.text
+              elsif e.name == 'EnqueueTime'
+                @enqueue_at = e.text.to_i
+              elsif e.name == 'FirstDequeueTime'
+                @first_enqueue_at = e.text.to_i
+              elsif e.name == 'DequeueCount'
+                @dequeue_count = e.text.to_i
+              elsif e.name == 'Priority'
+                @priority = e.text.to_i
+              elsif e.name == 'ReceiptHandle'
+                @receipt_handle = e.text
+              elsif e.name == 'NextVisibleTime'
+                @next_visible_at = e.text.to_i
+              end
+            end
+          end
+
+          # verify body
+          md5 = Digest::MD5.hexdigest(body).upcase
+          unless md5 == body_md5
+            raise Exception,
+                  'Invalid MNS Body, MD5 does not match, '\
+                  "MD5 #{body_md5}, expect MD5 #{md5}, Body: #{body}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/logstash/inputs/mns/request.rb

@@ -0,0 +1,85 @@
+# encoding: utf-8
+
+require 'base64'
+require 'uri'
+require 'rest-client'
+require 'rexml/document'
+
+module LogStash
+  module Inputs
+    module MNS
+      # Class for Aliyun MNS Request.
+      class Request
+        include REXML
+
+        attr_reader :log, :uri, :method, :body, :content_md5, :content_type,
+                    :content_length, :mns_headers, :access_key_id,
+                    :access_key_secret, :endpoint
+
+        def initialize(opts, headers, params)
+          @log = opts[:log]
+          conf = {
+              host: opts[:endpoint],
+              path: opts[:path]
+          }
+
+          conf[:query] = URI.encode_www_form(params) unless params.empty?
+          @uri = URI::HTTP.build(conf)
+          @method = opts[:method].to_s.downcase
+          @mns_headers = headers.merge('x-mns-version' => '2015-06-06')
+          @access_key_id = opts[:access_key_id]
+          @access_key_secret = opts[:access_key_secret]
+
+          log.info uri.to_s
+        end
+
+        def content(type, values = {})
+          ns = 'http://mns.aliyuncs.com/doc/v1/'
+          builder = Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml|
+            xml.send(type.to_sym, xmlns: ns) do |b|
+              values.each { |k, v| b.send k.to_sym, v }
+            end
+          end
+          @body = builder.to_xml
+          @content_md5 = Base64.encode64(Digest::MD5.hexdigest(body)).chop
+          @content_length = body.size
+          @content_type = 'text/xml;charset=utf-8'
+        end
+
+        def execute
+          date = DateTime.now.httpdate
+          headers = {
+              'Authorization' => authorization(date),
+              'Content-Length' => content_length || 0,
+              'Content-Type' => content_type,
+              'Content-MD5' => content_md5,
+              'Date' => date,
+              'Host' => uri.host
+          }.merge(@mns_headers).reject { |k, v| v.nil? }
+
+          begin
+            RestClient.send *[method, uri.to_s, headers, body].compact
+          rescue RestClient::Exception => e
+            doc = Document.new(e.response.to_s)
+            doc.elements[1].each do |e|
+              next unless e.node_type == :element
+              return nil if (e.name == 'Code') && (e.text == 'MessageNotExist')
+            end
+
+            log.error e.response
+
+            raise e
+          end
+        end
+
+        def authorization(date)
+          canonical_resource = [uri.path, uri.query].compact.join('?')
+          canonical_headers = mns_headers.sort.collect { |k, v| "#{k.downcase}:#{v}" }.join("\n")
+          signature = [method.to_s.upcase, content_md5 || '', content_type || '', date, canonical_headers, canonical_resource].join("\n")
+          sha1 = OpenSSL::HMAC.digest('sha1', access_key_secret, signature)
+          "MNS #{access_key_id}:#{Base64.encode64(sha1).chop}"
+        end
+      end
+    end
+  end
+end

data/lib/logstash/inputs/oss.rb

@@ -0,0 +1,382 @@
+# encoding: utf-8
+require 'json'
+require 'logstash/inputs/base'
+require 'logstash-input-oss_jars'
+
+java_import com.aliyun.oss.OSS
+java_import com.aliyun.oss.OSSClientBuilder
+java_import com.aliyun.oss.ClientBuilderConfiguration
+java_import com.aliyun.oss.model.GetObjectRequest
+java_import java.io.InputStream
+java_import java.io.InputStreamReader
+java_import java.io.FileInputStream
+java_import java.io.BufferedReader
+java_import java.util.zip.GZIPInputStream
+java_import java.util.zip.ZipException
+
+#
+# LogStash OSS Input Plugin
+# Stream events from files from an OSS bucket.
+# Each line from each file generates an event.
+# Files end with `.gz` or `.gzip` are handled as gzip'ed files.
+#
+class LogStash::Inputs::OSS < LogStash::Inputs::Base
+  require 'logstash/inputs/mns/message'
+  require 'logstash/inputs/mns/request'
+  require 'logstash/inputs/version'
+
+  MAX_CONNECTIONS_TO_OSS_KEY = "max_connections_to_oss"
+
+  SECURE_CONNECTION_ENABLED_KEY = "secure_connection_enabled"
+
+  MNS_ENDPOINT = "endpoint"
+
+  MNS_QUEUE = "queue"
+
+  MNS_WAIT_SECONDS = "wait_seconds"
+
+  MNS_POLL_INTERVAL_SECONDS = "poll_interval_seconds"
+
+  config_name "oss"
+
+  default :codec, "plain"
+
+  # The name of OSS bucket.
+  config :bucket, :validate => :string, :required => true
+
+  # OSS endpoint to connect
+  config :endpoint, :validate => :string, :required => true
+
+  # access key id
+  config :access_key_id, :validate => :string, :required => true
+
+  # access secret key
+  config :access_key_secret, :validate => :string, :required => true
+
+  # If specified, the prefix of filenames in the bucket must match (not a regexp)
+  config :prefix, :validate => :string, :default => nil
+
+  # additional oss client configurations, valid keys are:
+  # secure_connection_enabled(enable https or not)
+  # max_connections_to_oss(max connections to oss)
+  # TODO: add other oss configurations
+  config :additional_oss_settings, :validate => :hash, :default => nil
+
+  # Name of an OSS bucket to backup processed files to.
+  config :backup_to_bucket, :validate => :string, :default => nil
+
+  # Append a prefix to the key (full path including file name in OSS) after processing.
+  # If backing up to another (or the same) bucket, this effectively lets you
+  # choose a new 'folder' to place the files in
+  config :backup_add_prefix, :validate => :string, :default => nil
+
+  # Path of a local directory to backup processed files to.
+  config :backup_to_dir, :validate => :string, :default => nil
+
+  # Whether to delete processed files from the original bucket.
+  config :delete, :validate => :boolean, :default => false
+
+  # MNS configurations, valid keys are:
+  # endpoint: MNS endpoint to connect to
+  # queue: MNS queue to poll messages
+  # wait_seconds: MNS max waiting time to receive messages, not required
+  # poll_interval_seconds: Poll messages interval from MNS if there is no message this time, default 10 seconds
+  config :mns_settings, :validate => :hash, :required => true
+
+  # Ruby style regexp of keys to exclude from the bucket
+  config :exclude_pattern, :validate => :string, :default => nil
+
+  # Whether or not to include the OSS object's properties (last_modified, content_type, metadata)
+  # into each Event at [@metadata][oss]. Regardless of this setting, [@metadata][oss][key] will always
+  # be present.
+  config :include_object_properties, :validate => :boolean, :default => false
+
+  # For testing
+  config :stop_for_test, :validate => :boolean, :default => false
+
+  public
+  def register
+    @logger.info("Registering oss input", :bucket => @bucket)
+
+    # initialize oss client
+    @oss = initialize_oss_client
+
+    unless @backup_to_bucket.nil?
+      if @backup_to_bucket == @bucket
+        raise LogStash::ConfigurationError, "Logstash Input OSS Plugin: backup bucket and source bucket should be different"
+      end
+
+      unless @oss.doesBucketExist(@backup_to_bucket)
+        @oss.createBucket(@backup_to_bucket)
+      end
+    end
+
+    unless @backup_to_dir.nil?
+      Dir.mkdir(@backup_to_dir, 0700) unless File.exists?(@backup_to_dir)
+    end
+
+    if @mns_settings.include?(MNS_POLL_INTERVAL_SECONDS)
+      @interval = @mns_settings[MNS_POLL_INTERVAL_SECONDS]
+    else
+      @interval = 10
+    end
+
+    @mns_endpoint = @mns_settings[MNS_ENDPOINT]
+    @mns_queue = @mns_settings[MNS_QUEUE]
+    @mns_wait_seconds = @mns_settings[MNS_WAIT_SECONDS]
+  end
+
+  public
+  def run(queue)
+    @current_thread = Thread.current
+    Stud.interval(0.01) do
+      @logger.info "Start to poll message from MNS queue #{@mns_queue}"
+      process_objects(queue)
+      stop if stop?
+      if @stop_for_test and not @fetched
+        do_stop
+      end
+      sleep @interval unless @fetched
+    end
+  end
+
+  public
+  def stop
+    # @current_thread is initialized in the `#run` method,
+    # this variable is needed because the `#stop` is a called in another thread
+    # than the `#run` method and requiring us to call stop! with a explicit thread.
+    @logger.info("Logstash OSS Input Plugin is shutting down...")
+    Stud.stop!(@current_thread)
+  end
+
+  private
+  def process_objects(queue)
+    message = receive_message
+    if message.nil?
+      @fetched = false
+    else
+      @fetched = true
+      process(LogStash::Inputs::MNS::Message.new(@mns_queue, message), queue)
+    end
+  end
+
+  private
+  def receive_message
+    request_opts = {}
+    request_opts = { waitseconds: @mns_wait_seconds } if @mns_wait_seconds
+    opts = {
+      log: @logger,
+      method: 'GET',
+      endpoint: @mns_endpoint,
+      path: "/queues/#{@mns_queue}/messages",
+      access_key_id: @access_key_id,
+      access_key_secret: @access_key_secret
+    }
+    LogStash::Inputs::MNS::Request.new(opts, {}, request_opts).execute
+  end
+
+  private
+  def process(message, queue)
+    objects = get_objects(message)
+    objects.each do |object|
+      key = object.key
+      if key.end_with?("/")
+        @logger.info("Skip directory " + key)
+      elsif @prefix and @prefix != "" and not key.start_with?(@prefix)
+        @logger.info("Skip object " + key + " because object name does not match " + @prefix)
+      elsif @exclude_pattern and key =~ Regexp.new(@exclude_pattern)
+        @logger.info("Skip object " + key + " because object name matches exclude_pattern")
+      else
+        metadata = {}
+        begin
+          read_object(key) do |line, object_meta|
+            @codec.decode(line) do |event|
+              if event_is_metadata?(event)
+                @logger.debug('Event is metadata, updating the current cloudfront metadata', :event => event)
+                update_metadata(metadata, event)
+              else
+                decorate(event)
+
+                event.set("cloudfront_version", metadata[:cloudfront_version]) unless metadata[:cloudfront_version].nil?
+                event.set("cloudfront_fields", metadata[:cloudfront_fields]) unless metadata[:cloudfront_fields].nil?
+
+                if @include_object_properties
+                  object_meta.each do |key, value|
+                    event.set("[@metadata][oss][" + key + "]", value.to_s)
+                  end
+                else
+                  event.set("[@metadata][oss]", {})
+                end
+
+                event.set("[@metadata][oss][key]", key)
+                queue << event
+              end
+            end
+
+            @codec.flush do |event|
+              queue << event
+            end
+          end
+          backup_to_bucket(key)
+          backup_to_dir(key)
+          delete_file_from_bucket(key)
+        rescue Exception => e
+          # skip any broken file
+          @logger.error("Failed to read object. Skip processing.", :key => key, :exception => e.message)
+        end
+      end
+    end
+    delete_message(message)
+  end
+
+  private
+  def event_is_metadata?(event)
+    return false unless event.get("message").class == String
+    line = event.get("message")
+    version_metadata?(line) || fields_metadata?(line)
+  end
+
+  private
+  def version_metadata?(line)
+    line.start_with?('#Version: ')
+  end
+
+  private
+  def fields_metadata?(line)
+    line.start_with?('#Fields: ')
+  end
+
+  private
+  def update_metadata(metadata, event)
+    line = event.get('message').strip
+
+    if version_metadata?(line)
+      metadata[:cloudfront_version] = line.split(/#Version: (.+)/).last
+    end
+
+    if fields_metadata?(line)
+      metadata[:cloudfront_fields] = line.split(/#Fields: (.+)/).last
+    end
+  end
+
+  private
+  def read_object(key, &block)
+    @logger.info("Processing object " + key)
+    object = @oss.getObject(@bucket, key)
+    meta = object.getObjectMetadata.getRawMetadata
+    input = object.getObjectContent
+    content = input
+    if gzip?(key)
+      content = GZIPInputStream.new(input)
+    end
+
+    decoder = InputStreamReader.new(content, "UTF-8")
+    buffered = BufferedReader.new(decoder)
+
+    while (line = buffered.readLine)
+      block.call(line, meta)
+    end
+  ensure
+    buffered.close unless buffered.nil?
+    decoder.close unless decoder.nil?
+    content.close unless content.nil?
+    input.close unless input.nil?
+  end
+
+  private
+  def gzip?(filename)
+    filename.end_with?('.gz','.gzip')
+  end
+
+  private
+  def backup_to_bucket(key)
+    unless @backup_to_bucket.nil?
+      backup_key = "#{@backup_add_prefix}#{key}"
+      @oss.copyObject(@bucket, key, @backup_to_bucket, backup_key)
+    end
+  end
+
+  private
+  def backup_to_dir(key)
+    unless @backup_to_dir.nil?
+      file = @backup_to_dir + '/' + key
+
+      dirname = File.dirname(file)
+      unless File.directory?(dirname)
+        FileUtils.mkdir_p(dirname)
+      end
+
+      @oss.getObject(GetObjectRequest.new(@bucket, key), java.io.File.new(file))
+    end
+  end
+
+  private
+  def delete_file_from_bucket(key)
+    if @delete
+      @oss.deleteObject(@bucket, key)
+    end
+  end
+
+  private
+  def get_objects(message)
+    objects = []
+    events = JSON.parse(Base64.decode64(message.body))['events']
+    events.each do |event|
+      objects.push(OSSObject.new(event['eventName'],
+                                 @bucket,
+                                 event['oss']['object']['key'],
+                                 event['oss']['object']['size'],
+                                 event['oss']['object']['eTag']))
+    end
+    objects
+  end
+
+  private
+  def delete_message(message)
+    request_opts = { ReceiptHandle: message.receipt_handle }
+    opts = {
+      log: @logger,
+      method: 'DELETE',
+      endpoint: @mns_endpoint,
+      path: "/queues/#{@mns_queue}/messages",
+      access_key_id: @access_key_id,
+      access_key_secret: @access_key_secret
+    }
+    LogStash::Inputs::MNS::Request.new(opts, {}, request_opts).execute
+  end
+
+  # OSS Object class from MNS events
+  class OSSObject
+    attr_reader :event_name, :bucket, :key, :size, :etag
+    def initialize(event_name, bucket, key, size, etag)
+      @event_name = event_name
+      @bucket = bucket
+      @key = key
+      @size = size
+      @etag = etag
+    end
+  end
+
+  private
+  def initialize_oss_client
+    clientConf = ClientBuilderConfiguration.new
+    unless @additional_oss_settings.nil?
+      if @additional_oss_settings.include?(SECURE_CONNECTION_ENABLED_KEY)
+        clientConf.setProtocol(@additional_oss_settings[SECURE_CONNECTION_ENABLED_KEY] ?
+          com.aliyun.oss.common.comm.Protocol::HTTPS : com.aliyun.oss.common.comm.Protocol::HTTP)
+      end
+
+      if @additional_oss_settings.include?(MAX_CONNECTIONS_TO_OSS_KEY)
+        if @additional_oss_settings[MAX_CONNECTIONS_TO_OSS_KEY] <= 0
+          raise LogStash::ConfigurationError, "Logstash OSS Input plugin must have positive " + MAX_CONNECTIONS_TO_OSS_KEY
+        end
+        clientConf.setMaxConnections(@additional_oss_settings[MAX_CONNECTIONS_TO_OSS_KEY])
+      else
+        clientConf.setMaxConnections(1024)
+      end
+    end
+
+    clientConf.setUserAgent(clientConf.getUserAgent() + ", Logstash-oss/" + Version.version)
+    OSSClientBuilder.new().build(@endpoint, @access_key_id, @access_key_secret, clientConf)
+  end
+end