logstash-output-s3 2.0.7 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 80e75152fb124d5936f0fd110072509a69845f52
-  data.tar.gz: 407179b921a6f00a8ea5357d39db6f12525e87e7
+  metadata.gz: 6c4f5b3ad0556699be1a0cc4f99e3c7e96e350d8
+  data.tar.gz: 6274f98e488cc9545c510a974c994e0a71060d92
 SHA512:
-  metadata.gz: a7a5a417febef05d0f6591f93d1ce3a9192ed327729f1296736c43fb5570b41412c6cf0aefc3215a4fcd88e8faf4cfedb2e8cbbe8cd19d61e0038890c674dda9
-  data.tar.gz: f9446f247c8a768367a6df7c0829d5cd8471724f1a329b3c75297c03e76f0421b2155cb17401060a6dfc7adbf28dcca983ba936f3eda44eee1e68dca1363c6a1
+  metadata.gz: 21690767ece4c415ac215807faa58aa127e617368fde426a0fe25d5b78cad4b3467eca8879ffa6f269205640da2e5dddec0ee007bc086550cc3d33ed71bfd173
+  data.tar.gz: 318ab7c8009dd314e2b26f43746437027a14cf517abe5051faefb0c7e156fe59572578e9d7d9c21c2d431a30e0905bce49193bcf647d62dad02196ecfe258637

data/CHANGELOG.md

@@ -1,3 +1,5 @@
+## 3.0.0
+  - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
 # 2.0.7
   - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
 # 2.0.6

data/Gemfile

@@ -1,2 +1,4 @@
 source 'https://rubygems.org'
+
+# Specify your gem's dependencies in logstash-mass_effect.gemspec
 gemspec

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012–2015 Elasticsearch <http://www.elastic.co>
+Copyright (c) 2012–2016 Elasticsearch <http://www.elastic.co>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

data/README.md

@@ -1,7 +1,6 @@
 # Logstash Plugin
 
-[![Build
-Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Outputs/job/logstash-plugin-output-s3-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Outputs/job/logstash-plugin-output-s3-unit/)
+[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-output-s3.svg)](https://travis-ci.org/logstash-plugins/logstash-output-s3)
 
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
@@ -56,7 +55,12 @@ gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
 ```
 - Install plugin
 ```sh
+# Logstash 2.3 and higher
+bin/logstash-plugin install --no-verify
+
+# Prior to Logstash 2.3
 bin/plugin install --no-verify
+
 ```
 - Run Logstash with your plugin
 ```sh
@@ -74,7 +78,12 @@ gem build logstash-filter-awesome.gemspec
 ```
 - Install the plugin from the Logstash home
 ```sh
-bin/plugin install /your/local/plugin/logstash-filter-awesome.gem
+# Logstash 2.3 and higher
+bin/logstash-plugin install --no-verify
+
+# Prior to Logstash 2.3
+bin/plugin install --no-verify
+
 ```
 - Start Logstash and proceed to test the plugin
 

data/lib/logstash/outputs/s3.rb

@@ -12,13 +12,16 @@ require "fileutils"
 
 # INFORMATION:
 #
-# This plugin sends logstash events to Amazon Simple Storage Service (Amazon S3).
-# To use it you need to have the proper write premissions and a valid s3 bucket.
-# Make sure you have permissions to write files on S3's bucket.  Also be sure to run logstash as super user to establish a connection.
+# This plugin batches and uploads logstash events into Amazon Simple Storage Service (Amazon S3).
+# 
+# Requirements: 
+# * Amazon S3 Bucket and S3 Access Permissions (Typically access_key_id and secret_access_key)
+# * S3 PutObject permission
+# * Run logstash as superuser to establish connection
 #
+# S3 outputs create temporary files into "/opt/logstash/S3_temp/". If you want, you can change the path at the start of register method.
 #
-# This plugin outputs temporary files to "/opt/logstash/S3_temp/". If you want, you can change the path at the start of register method.
-# These files have a special name, for example:
+# S3 output files have the following format
 #
 # ls.s3.ip-10-228-27-95.2013-04-18T10.00.tag_hello.part0.txt
 #
@@ -28,23 +31,28 @@ require "fileutils"
 # "2013-04-18T10.00" : represents the time whenever you specify time_file.
 # "tag_hello" : this indicates the event's tag.
 # "part0" : this means if you indicate size_file then it will generate more parts if you file.size > size_file.
-#           When a file is full it will be pushed to a bucket and will be deleted from the temporary directory.
-#           If a file is empty is not pushed, it is not deleted.
+#           When a file is full it will be pushed to the bucket and then deleted from the temporary directory.
+#           If a file is empty, it is simply deleted.  Empty files will not be pushed
 #
-# This plugin have a system to restore the previous temporary files if something crash.
+# Crash Recovery:
+# * This plugin will recover and upload temporary log files after crash/abnormal termination
 #
-##[Note] :
+##[Note regarding time_file and size_file] :
 #
-## If you specify size_file and time_file then it will create file for each tag (if specified), when time_file or
-## their size > size_file, it will be triggered then they will be pushed on s3's bucket and will delete from local disk.
-## If you don't specify size_file, but time_file then it will create only one file for each tag (if specified).
-## When time_file it will be triggered then the files will be pushed on s3's bucket and delete from local disk.
+# Both time_file and size_file settings can trigger a log "file rotation"
+# A log rotation pushes the current log "part" to s3 and deleted from local temporary storage.
 #
-## If you don't specify time_file, but size_file  then it will create files for each tag (if specified),
-## that will be triggered when their size > size_file, then they will be pushed on s3's bucket and will delete from local disk.
+## If you specify BOTH size_file and time_file then it will create file for each tag (if specified). 
+## When EITHER time_file minutes have elapsed OR log file size > size_file, a log rotation is triggered.
+##
+## If you ONLY specify time_file but NOT file_size, one file for each tag (if specified) will be created..
+## When time_file minutes elapses, a log rotation will be triggered.
 #
-## If you don't specific size_file and time_file you have a curios mode. It will create only one file for each tag (if specified).
-## Then the file will be rest on temporary directory and don't will be pushed on bucket until we will restart logstash.
+## If you ONLY specify size_file, but NOT time_file, one files for each tag (if specified) will be created.
+## When size of log file part > size_file, a log rotation will be triggered.
+#
+## If NEITHER size_file nor time_file is specified, ONLY one file for each tag (if specified) will be created.
+## WARNING: Since no log rotation is triggered, S3 Upload will only occur when logstash restarts.
 #
 #
 # #### Usage:
@@ -54,10 +62,11 @@ require "fileutils"
 #    s3{
 #      access_key_id => "crazy_key"             (required)
 #      secret_access_key => "monkey_access_key" (required)
-#      endpoint_region => "eu-west-1"           (required)
+#      endpoint_region => "eu-west-1"           (required) - Deprecated
 #      bucket => "boss_please_open_your_bucket" (required)
-#      size_file => 2048                        (optional)
-#      time_file => 5                           (optional)
+#      size_file => 2048                        (optional) - Bytes
+#      time_file => 5                           (optional) - Minutes
+#      format => "plain"                        (optional)
 #      canned_acl => "private"                  (optional. Options are "private", "public_read", "public_read_write", "authenticated_read". Defaults to "private" )
 #    }
 #
@@ -83,7 +92,7 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   ##NOTE: define size of file is the better thing, because generate a local temporary file on disk and then put it in bucket.
   config :size_file, :validate => :number, :default => 0
 
-  # Set the time, in minutes, to close the current sub_time_section of bucket.
+  # Set the time, in MINUTES, to close the current sub_time_section of bucket.
   # If you define file_size you have a number of files in consideration of the section and the current tag.
   # 0 stay all time on listerner, beware if you specific 0 and size_file 0, because you will not put the file on bucket,
   # for now the only thing this plugin can do is to put the file when logstash restart.
@@ -99,11 +108,14 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   config :canned_acl, :validate => ["private", "public_read", "public_read_write", "authenticated_read"],
          :default => "private"
 
+  # Specifies wether or not to use S3's AES256 server side encryption. Defaults to false.
+  config :server_side_encryption, :validate => :boolean, :default => false
+
   # Set the directory where logstash will store the tmp files before sending it to S3
   # default to the current OS temporary directory in linux /tmp/logstash
   config :temporary_directory, :validate => :string, :default => File.join(Dir.tmpdir, "logstash")
 
-  # Specify a prefix to the uploaded filename, this can simulate directories on S3
+  # Specify a prefix to the uploaded filename, this can simulate directories on S3.  Prefix does not require leading slash.
   config :prefix, :validate => :string, :default => ''
 
   # Specify how many workers to use to upload the files to S3
@@ -173,7 +185,7 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
       begin
         # prepare for write the file
         object = bucket.objects[remote_filename]
-        object.write(fileIO, :acl => @canned_acl)
+        object.write(fileIO, :acl => @canned_acl, :server_side_encryption => @server_side_encryption ? :aes256 : nil)
       rescue AWS::Errors::Base => error
         @logger.error("S3: AWS error", :error => error)
         raise LogStash::Error, "AWS Configuration Error, #{error}"
@@ -258,11 +270,11 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
 
   public
   def restore_from_crashes
-    @logger.debug("S3: is attempting to verify previous crashes...")
+    @logger.debug("S3: Checking for temp files from a previoius crash...")
 
     Dir[File.join(@temporary_directory, "*.#{TEMPFILE_EXTENSION}")].each do |file|
       name_file = File.basename(file)
-      @logger.warn("S3: have found temporary file the upload process crashed, uploading file to S3.", :filename => name_file)
+      @logger.warn("S3: Found temporary file from crash.  Uploading file to S3.", :filename => name_file)
       move_file_to_bucket_async(file)
     end
   end
@@ -271,7 +283,7 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   def move_file_to_bucket(file)
     if !File.zero?(file)
       write_on_bucket(file)
-      @logger.debug("S3: file was put on the upload thread", :filename => File.basename(file), :bucket => @bucket)
+      @logger.debug("S3: File was put on the upload thread", :filename => File.basename(file), :bucket => @bucket)
     end
 
     begin

data/logstash-output-s3.gemspec

@@ -1,10 +1,10 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-s3'
-  s.version         = '2.0.7'
+  s.version         = '3.0.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This plugin was created for store the logstash's events into Amazon Simple Storage Service (Amazon S3)"
-  s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors         = ["Elastic"]
   s.email           = 'info@elastic.co'
   s.homepage        = "http://www.elastic.co/guide/en/logstash/current/index.html"
@@ -20,7 +20,7 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
   s.add_runtime_dependency 'logstash-mixin-aws'
   s.add_runtime_dependency 'stud', '~> 0.0.22'
   s.add_development_dependency 'logstash-devutils'

metadata

@@ -1,100 +1,102 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-s3
 version: !ruby/object:Gem::Version
-  version: 2.0.7
+  version: 3.0.0
 platform: ruby
 authors:
 - Elastic
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-24 00:00:00.000000000 Z
+date: 2016-05-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
-  name: logstash-core-plugin-api
-  prerelease: false
+        version: '2.0'
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
+  name: logstash-mixin-aws
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-mixin-aws
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: stud
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.0.22
-  name: stud
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.0.22
 - !ruby/object:Gem::Dependency
+  name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: logstash-input-generator
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-input-generator
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: logstash-codec-line
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-codec-line
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program
 email: info@elastic.co
 executables: []
 extensions: []
@@ -118,7 +120,7 @@ licenses:
 metadata:
   logstash_plugin: 'true'
   logstash_group: output
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -133,11 +135,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.4.8
-signing_key:
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
 specification_version: 4
-summary: This plugin was created for store the logstash's events into Amazon Simple Storage Service (Amazon S3)
+summary: This plugin was created for store the logstash's events into Amazon Simple
+  Storage Service (Amazon S3)
 test_files:
 - spec/integration/s3_spec.rb
 - spec/outputs/s3_spec.rb