logstash-output-s3 4.0.7 → 4.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4101484ab3c3ed89b22088de339be1cf010b3753
-  data.tar.gz: 4f31784de1daf62a7673a3e756bff48876c3874f
+  metadata.gz: 0c5e0dea828a0337ddbc6f6a16c066d96957d1e9
+  data.tar.gz: 48349191b34fee81b5e5270bdfaa859679e0f6dc
 SHA512:
-  metadata.gz: fd14b0db9ce158b3c0d7a00182f7aa6adf7eb5d5c178d958d6103204864fd1dc914929a2f214e67b1b4f0c931abef23f57fde968f423fb4ce62f31f52d79952a
-  data.tar.gz: f56cd39938dbf9191e8d4190b5e38fa60072a8ffb2e876776bcd82a7b84436d349d7056ee81372247ccdc7a88cc5b0635e2f81251c1d560f81777b52a66ca527
+  metadata.gz: ed282270f16481347356ee28d24905cc464dbcd5854370a351f95744d3eb6520e67079a28aaa6f736f2a23a1f1be3b015d6233101bf99fee7c34ed17f821d8df
+  data.tar.gz: 556d6cbe0469cd0c1ca7de0ee2c6e536a15aa841444774dc9a172647f10881c6bde9c1f1b9fd67719aa09c38f65fda59cff98c7239d9f584ca28109808a4beb8

data/Gemfile

@@ -1,4 +1,11 @@
 source 'https://rubygems.org'
 
-# Specify your gem's dependencies in logstash-mass_effect.gemspec
 gemspec
+
+logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
+use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
+
+if Dir.exist?(logstash_path) && use_logstash_source
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+end

data/docs/index.asciidoc

@@ -0,0 +1,327 @@
+:plugin: s3
+:type: output
+
+///////////////////////////////////////////
+START - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+:version: %VERSION%
+:release_date: %RELEASE_DATE%
+:changelog_url: %CHANGELOG_URL%
+:include_path: ../../../../logstash/docs/include
+///////////////////////////////////////////
+END - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+
+[id="plugins-{type}-{plugin}"]
+
+=== S3 output plugin
+
+include::{include_path}/plugin_header.asciidoc[]
+
+==== Description
+
+INFORMATION:
+
+This plugin batches and uploads logstash events into Amazon Simple Storage Service (Amazon S3).
+
+Requirements:
+* Amazon S3 Bucket and S3 Access Permissions (Typically access_key_id and secret_access_key)
+* S3 PutObject permission
+
+S3 outputs create temporary files into the OS' temporary directory, you can specify where to save them using the `temporary_directory` option.
+
+S3 output files have the following format
+
+ls.s3.312bc026-2f5d-49bc-ae9f-5940cf4ad9a6.2013-04-18T10.00.tag_hello.part0.txt
+
+
+|=======
+| ls.s3 | indicate logstash plugin s3 |
+| 312bc026-2f5d-49bc-ae9f-5940cf4ad9a6 | a new, random uuid per file. |
+| 2013-04-18T10.00 | represents the time whenever you specify time_file. |
+| tag_hello | this indicates the event's tag. |
+| part0 | this means if you indicate size_file then it will generate more parts if you file.size > size_file. When a file is full it will be pushed to the bucket and then deleted from the temporary directory. If a file is empty, it is simply deleted.  Empty files will not be pushed |
+|=======
+
+Crash Recovery:
+* This plugin will recover and upload temporary log files after crash/abnormal termination when using `restore` set to true
+
+
+
+
+
+
+
+#### Usage:
+This is an example of logstash config:
+[source,ruby]
+output {
+   s3{
+     access_key_id => "crazy_key"             (required)
+     secret_access_key => "monkey_access_key" (required)
+     region => "eu-west-1"                    (optional, default = "us-east-1")
+     bucket => "your_bucket"                  (required)
+     size_file => 2048                        (optional) - Bytes
+     time_file => 5                           (optional) - Minutes
+     codec => "plain"                         (optional)
+     canned_acl => "private"                  (optional. Options are "private", "public-read", "public-read-write", "authenticated-read". Defaults to "private" )
+   }
+
+
+[id="plugins-{type}s-{plugin}-options"]
+==== S3 Output Configuration Options
+
+This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+|Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-access_key_id>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-aws_credentials_file>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-bucket>> |<<string,string>>|Yes
+| <<plugins-{type}s-{plugin}-canned_acl>> |<<string,string>>, one of `["private", "public-read", "public-read-write", "authenticated-read"]`|No
+| <<plugins-{type}s-{plugin}-encoding>> |<<string,string>>, one of `["none", "gzip"]`|No
+| <<plugins-{type}s-{plugin}-prefix>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-proxy_uri>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-region>> |<<string,string>>, one of `["us-east-1", "us-east-2", "us-west-1", "us-west-2", "eu-central-1", "eu-west-1", "eu-west-2", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1", "ap-northeast-2", "sa-east-1", "us-gov-west-1", "cn-north-1", "ap-south-1", "ca-central-1"]`|No
+| <<plugins-{type}s-{plugin}-restore>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-rotation_strategy>> |<<string,string>>, one of `["size_and_time", "size", "time"]`|No
+| <<plugins-{type}s-{plugin}-secret_access_key>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-server_side_encryption>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-server_side_encryption_algorithm>> |<<string,string>>, one of `["AES256", "aws:kms"]`|No
+| <<plugins-{type}s-{plugin}-session_token>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-signature_version>> |<<string,string>>, one of `["v2", "v4"]`|No
+| <<plugins-{type}s-{plugin}-size_file>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-ssekms_key_id>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-storage_class>> |<<string,string>>, one of `["STANDARD", "REDUCED_REDUNDANCY", "STANDARD_IA"]`|No
+| <<plugins-{type}s-{plugin}-temporary_directory>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-time_file>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-upload_queue_size>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-upload_workers_count>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-validate_credentials_on_root_bucket>> |<<boolean,boolean>>|No
+|=======================================================================
+
+Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
+output plugins.
+
+&nbsp;
+
+[id="plugins-{type}s-{plugin}-access_key_id"]
+===== `access_key_id` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order:
+
+1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config
+2. External credentials file specified by `aws_credentials_file`
+3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
+4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY`
+5. IAM Instance Profile (available when running inside EC2)
+
+[id="plugins-{type}s-{plugin}-aws_credentials_file"]
+===== `aws_credentials_file` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Path to YAML file containing a hash of AWS credentials.
+This file will only be loaded if `access_key_id` and
+`secret_access_key` aren't set. The contents of the
+file should look like this:
+
+[source,ruby]
+----------------------------------
+    :access_key_id: "12345"
+    :secret_access_key: "54321"
+----------------------------------
+
+
+[id="plugins-{type}s-{plugin}-bucket"]
+===== `bucket` 
+
+  * This is a required setting.
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+S3 bucket
+
+[id="plugins-{type}s-{plugin}-canned_acl"]
+===== `canned_acl` 
+
+  * Value can be any of: `private`, `public-read`, `public-read-write`, `authenticated-read`
+  * Default value is `"private"`
+
+The S3 canned ACL to use when putting the file. Defaults to "private".
+
+[id="plugins-{type}s-{plugin}-encoding"]
+===== `encoding` 
+
+  * Value can be any of: `none`, `gzip`
+  * Default value is `"none"`
+
+Specify the content encoding. Supports ("gzip"). Defaults to "none"
+
+[id="plugins-{type}s-{plugin}-prefix"]
+===== `prefix` 
+
+  * Value type is <<string,string>>
+  * Default value is `""`
+
+Specify a prefix to the uploaded filename, this can simulate directories on S3.  Prefix does not require leading slash.
+This option support string interpolation, be warned this can created a lot of temporary local files.
+
+[id="plugins-{type}s-{plugin}-proxy_uri"]
+===== `proxy_uri` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+URI to proxy server if required
+
+[id="plugins-{type}s-{plugin}-region"]
+===== `region` 
+
+  * Value can be any of: `us-east-1`, `us-east-2`, `us-west-1`, `us-west-2`, `eu-central-1`, `eu-west-1`, `eu-west-2`, `ap-southeast-1`, `ap-southeast-2`, `ap-northeast-1`, `ap-northeast-2`, `sa-east-1`, `us-gov-west-1`, `cn-north-1`, `ap-south-1`, `ca-central-1`
+  * Default value is `"us-east-1"`
+
+The AWS Region
+
+[id="plugins-{type}s-{plugin}-restore"]
+===== `restore` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `true`
+
+
+
+[id="plugins-{type}s-{plugin}-rotation_strategy"]
+===== `rotation_strategy` 
+
+  * Value can be any of: `size_and_time`, `size`, `time`
+  * Default value is `"size_and_time"`
+
+Define the strategy to use to decide when we need to rotate the file and push it to S3,
+The default strategy is to check for both size and time, the first one to match will rotate the file.
+
+[id="plugins-{type}s-{plugin}-secret_access_key"]
+===== `secret_access_key` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The AWS Secret Access Key
+
+[id="plugins-{type}s-{plugin}-server_side_encryption"]
+===== `server_side_encryption` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+Specifies wether or not to use S3's server side encryption. Defaults to no encryption.
+
+[id="plugins-{type}s-{plugin}-server_side_encryption_algorithm"]
+===== `server_side_encryption_algorithm` 
+
+  * Value can be any of: `AES256`, `aws:kms`
+  * Default value is `"AES256"`
+
+Specifies what type of encryption to use when SSE is enabled.
+
+[id="plugins-{type}s-{plugin}-session_token"]
+===== `session_token` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The AWS Session token for temporary credential
+
+[id="plugins-{type}s-{plugin}-signature_version"]
+===== `signature_version` 
+
+  * Value can be any of: `v2`, `v4`
+  * There is no default value for this setting.
+
+The version of the S3 signature hash to use. Normally uses the internal client default, can be explicitly
+specified here
+
+[id="plugins-{type}s-{plugin}-size_file"]
+===== `size_file` 
+
+  * Value type is <<number,number>>
+  * Default value is `5242880`
+
+Set the size of file in bytes, this means that files on bucket when have dimension > file_size, they are stored in two or more file.
+If you have tags then it will generate a specific size file for every tags
+
+[id="plugins-{type}s-{plugin}-ssekms_key_id"]
+===== `ssekms_key_id` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The key to use when specified along with server_side_encryption => aws:kms.
+If server_side_encryption => aws:kms is set but this is not default KMS key is used.
+http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html
+
+[id="plugins-{type}s-{plugin}-storage_class"]
+===== `storage_class` 
+
+  * Value can be any of: `STANDARD`, `REDUCED_REDUNDANCY`, `STANDARD_IA`
+  * Default value is `"STANDARD"`
+
+Specifies what S3 storage class to use when uploading the file.
+More information about the different storage classes can be found:
+http://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html
+Defaults to STANDARD.
+
+[id="plugins-{type}s-{plugin}-temporary_directory"]
+===== `temporary_directory` 
+
+  * Value type is <<string,string>>
+  * Default value is `"/tmp/logstash"`
+
+Set the directory where logstash will store the tmp files before sending it to S3
+default to the current OS temporary directory in linux /tmp/logstash
+
+[id="plugins-{type}s-{plugin}-time_file"]
+===== `time_file` 
+
+  * Value type is <<number,number>>
+  * Default value is `15`
+
+Set the time, in MINUTES, to close the current sub_time_section of bucket.
+If you define file_size you have a number of files in consideration of the section and the current tag.
+0 stay all time on listerner, beware if you specific 0 and size_file 0, because you will not put the file on bucket,
+for now the only thing this plugin can do is to put the file when logstash restart.
+
+[id="plugins-{type}s-{plugin}-upload_queue_size"]
+===== `upload_queue_size` 
+
+  * Value type is <<number,number>>
+  * Default value is `4`
+
+Number of items we can keep in the local queue before uploading them
+
+[id="plugins-{type}s-{plugin}-upload_workers_count"]
+===== `upload_workers_count` 
+
+  * Value type is <<number,number>>
+  * Default value is `4`
+
+Specify how many workers to use to upload the files to S3
+
+[id="plugins-{type}s-{plugin}-validate_credentials_on_root_bucket"]
+===== `validate_credentials_on_root_bucket` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `true`
+
+The common use case is to define permission on the root bucket and give Logstash full access to write its logs.
+In some circonstances you need finer grained permission on subfolder, this allow you to disable the check at startup.
+
+
+
+[id="plugins-{type}s-{plugin}-common-options"]
+include::{include_path}/{type}.asciidoc[]

data/logstash-output-s3.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-s3'
-  s.version         = '4.0.7'
+  s.version         = '4.0.8'
   s.licenses        = ['Apache-2.0']
   s.summary         = "This plugin was created for store the logstash's events into Amazon Simple Storage Service (Amazon S3)"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -10,7 +10,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Files
-  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-s3
 version: !ruby/object:Gem::Version
-  version: 4.0.7
+  version: 4.0.8
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-04-28 00:00:00.000000000 Z
+date: 2017-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -127,6 +127,7 @@ files:
 - LICENSE
 - NOTICE.TXT
 - README.md
+- docs/index.asciidoc
 - lib/logstash/outputs/s3.rb
 - lib/logstash/outputs/s3/file_repository.rb
 - lib/logstash/outputs/s3/patch.rb