logstash-input-s3 3.1.4 → 3.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 939d76e37b99e18082ac1ce254e2f3cb985badcf
-  data.tar.gz: c343885300f82500f0c2afa414336a7c56d1927f
+  metadata.gz: f4ddd73be7f58d106502b3f584151beaf1dcf5ff
+  data.tar.gz: 3808a1c7dbbb40f6ecdcc3c69dfb55ad139ed085
 SHA512:
-  metadata.gz: e5126a3db583f404d16bdf1d1e447dd574fa18d4d31e04d2e447e0345d3a9d8f02bbc436ae6a701c049be4b1c53619e6ed58279e723851d6fe6c49825d354a49
-  data.tar.gz: 01e4aebedbca23c9a4133a6c8886450081594cf72803bd90d19a2d542ff815c592362b50539933b04a58247edee8439a6e32f77adf99f5df7eb06ac7555b1f16
+  metadata.gz: ac07cf8020fdf92b1b8e814bf0565c2ef9c5fbe2610b1cfe71ab2a0f37d60613da3ad751ccd7ce77b6c566d71da27ff6a052223c6f1078065a3ec2a24fcf0ed4
+  data.tar.gz: e1f877839588b264ad73068a6197415d142c7a40b2e24abb5ecda68ac15cfea846c4c570052fa1d73108455f50ac0793d22123b6a949dbec33583c0549f0fc11

data/Gemfile

@@ -1,4 +1,11 @@
 source 'https://rubygems.org'
 
-# Specify your gem's dependencies in logstash-mass_effect.gemspec
 gemspec
+
+logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
+use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
+
+if Dir.exist?(logstash_path) && use_logstash_source
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+end

data/docs/index.asciidoc

@@ -0,0 +1,214 @@
+:plugin: s3
+:type: input
+
+///////////////////////////////////////////
+START - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+:version: %VERSION%
+:release_date: %RELEASE_DATE%
+:changelog_url: %CHANGELOG_URL%
+:include_path: ../../../../logstash/docs/include
+///////////////////////////////////////////
+END - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+
+[id="plugins-{type}-{plugin}"]
+
+=== S3 input plugin
+
+include::{include_path}/plugin_header.asciidoc[]
+
+==== Description
+
+Stream events from files from a S3 bucket.
+
+Each line from each file generates an event.
+Files ending in `.gz` are handled as gzip'ed files.
+
+[id="plugins-{type}s-{plugin}-options"]
+==== S3 Input Configuration Options
+
+This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+|Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-access_key_id>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-aws_credentials_file>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-backup_add_prefix>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-backup_to_bucket>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-backup_to_dir>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-bucket>> |<<string,string>>|Yes
+| <<plugins-{type}s-{plugin}-delete>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-exclude_pattern>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-interval>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-prefix>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-proxy_uri>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-region>> |<<string,string>>, one of `["us-east-1", "us-east-2", "us-west-1", "us-west-2", "eu-central-1", "eu-west-1", "eu-west-2", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1", "ap-northeast-2", "sa-east-1", "us-gov-west-1", "cn-north-1", "ap-south-1", "ca-central-1"]`|No
+| <<plugins-{type}s-{plugin}-secret_access_key>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-session_token>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-sincedb_path>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-temporary_directory>> |<<string,string>>|No
+|=======================================================================
+
+Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
+input plugins.
+
+&nbsp;
+
+[id="plugins-{type}s-{plugin}-access_key_id"]
+===== `access_key_id` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order:
+
+1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config
+2. External credentials file specified by `aws_credentials_file`
+3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
+4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY`
+5. IAM Instance Profile (available when running inside EC2)
+
+[id="plugins-{type}s-{plugin}-aws_credentials_file"]
+===== `aws_credentials_file` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Path to YAML file containing a hash of AWS credentials.
+This file will only be loaded if `access_key_id` and
+`secret_access_key` aren't set. The contents of the
+file should look like this:
+
+[source,ruby]
+----------------------------------
+    :access_key_id: "12345"
+    :secret_access_key: "54321"
+----------------------------------
+
+
+[id="plugins-{type}s-{plugin}-backup_add_prefix"]
+===== `backup_add_prefix` 
+
+  * Value type is <<string,string>>
+  * Default value is `nil`
+
+Append a prefix to the key (full path including file name in s3) after processing.
+If backing up to another (or the same) bucket, this effectively lets you
+choose a new 'folder' to place the files in
+
+[id="plugins-{type}s-{plugin}-backup_to_bucket"]
+===== `backup_to_bucket` 
+
+  * Value type is <<string,string>>
+  * Default value is `nil`
+
+Name of a S3 bucket to backup processed files to.
+
+[id="plugins-{type}s-{plugin}-backup_to_dir"]
+===== `backup_to_dir` 
+
+  * Value type is <<string,string>>
+  * Default value is `nil`
+
+Path of a local directory to backup processed files to.
+
+[id="plugins-{type}s-{plugin}-bucket"]
+===== `bucket` 
+
+  * This is a required setting.
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The name of the S3 bucket.
+
+[id="plugins-{type}s-{plugin}-delete"]
+===== `delete` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+Whether to delete processed files from the original bucket.
+
+[id="plugins-{type}s-{plugin}-exclude_pattern"]
+===== `exclude_pattern` 
+
+  * Value type is <<string,string>>
+  * Default value is `nil`
+
+Ruby style regexp of keys to exclude from the bucket
+
+[id="plugins-{type}s-{plugin}-interval"]
+===== `interval` 
+
+  * Value type is <<number,number>>
+  * Default value is `60`
+
+Interval to wait between to check the file list again after a run is finished.
+Value is in seconds.
+
+[id="plugins-{type}s-{plugin}-prefix"]
+===== `prefix` 
+
+  * Value type is <<string,string>>
+  * Default value is `nil`
+
+If specified, the prefix of filenames in the bucket must match (not a regexp)
+
+[id="plugins-{type}s-{plugin}-proxy_uri"]
+===== `proxy_uri` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+URI to proxy server if required
+
+[id="plugins-{type}s-{plugin}-region"]
+===== `region` 
+
+  * Value can be any of: `us-east-1`, `us-east-2`, `us-west-1`, `us-west-2`, `eu-central-1`, `eu-west-1`, `eu-west-2`, `ap-southeast-1`, `ap-southeast-2`, `ap-northeast-1`, `ap-northeast-2`, `sa-east-1`, `us-gov-west-1`, `cn-north-1`, `ap-south-1`, `ca-central-1`
+  * Default value is `"us-east-1"`
+
+The AWS Region
+
+[id="plugins-{type}s-{plugin}-secret_access_key"]
+===== `secret_access_key` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The AWS Secret Access Key
+
+[id="plugins-{type}s-{plugin}-session_token"]
+===== `session_token` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The AWS Session token for temporary credential
+
+[id="plugins-{type}s-{plugin}-sincedb_path"]
+===== `sincedb_path` 
+
+  * Value type is <<string,string>>
+  * Default value is `nil`
+
+Where to write the since database (keeps track of the date
+the last handled file was added to S3). The default will write
+sincedb files to some path matching "$HOME/.sincedb*"
+Should be a path with filename not just a directory.
+
+[id="plugins-{type}s-{plugin}-temporary_directory"]
+===== `temporary_directory` 
+
+  * Value type is <<string,string>>
+  * Default value is `"/tmp/logstash"`
+
+Set the directory where logstash will store the tmp files before processing them.
+default to the current OS temporary directory in linux /tmp/logstash
+
+
+
+[id="plugins-{type}s-{plugin}-common-options"]
+include::{include_path}/{type}.asciidoc[]

data/logstash-input-s3.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-s3'
-  s.version         = '3.1.4'
+  s.version         = '3.1.5'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Stream events from files from a S3 bucket."
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Files
-  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3
 version: !ruby/object:Gem::Version
-  version: 3.1.4
+  version: 3.1.5
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-04-28 00:00:00.000000000 Z
+date: 2017-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -98,6 +98,7 @@ files:
 - LICENSE
 - NOTICE.TXT
 - README.md
+- docs/index.asciidoc
 - lib/logstash/inputs/s3.rb
 - lib/logstash/inputs/s3/patch.rb
 - logstash-input-s3.gemspec