logstash-output-s3-zst 1.0.0

data/docs/index.asciidoc

@@ -0,0 +1,451 @@
+:plugin: s3
+:type: output
+:default_codec: line
+
+///////////////////////////////////////////
+START - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+:version: %VERSION%
+:release_date: %RELEASE_DATE%
+:changelog_url: %CHANGELOG_URL%
+:include_path: ../../../../logstash/docs/include
+///////////////////////////////////////////
+END - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+
+[id="plugins-{type}s-{plugin}"]
+
+=== S3 output plugin
+
+include::{include_path}/plugin_header.asciidoc[]
+
+==== Description
+
+This plugin batches and uploads logstash events into Amazon Simple Storage
+Service (Amazon S3).
+
+IMPORTANT: The S3 output plugin only supports AWS S3. 
+Other S3 compatible storage solutions are not supported.
+
+S3 outputs create temporary files into the OS' temporary directory. 
+You can specify where to save them using the `temporary_directory` option.
+
+IMPORTANT: For configurations containing multiple s3 outputs with the `restore`
+option enabled, each output should define its own `temporary_directory`.
+Shared or nested directories can cause data loss upon recovery.
+
+===== Requirements
+
+* Amazon S3 Bucket and S3 Access Permissions (Typically access_key_id and secret_access_key)
+* S3 PutObject permission
+
+===== S3 output file
+ 
+[source,txt]
+-----
+`ls.s3.312bc026-2f5d-49bc-ae9f-5940cf4ad9a6.2013-04-18T10.00.tag_hello.part0.txt`
+-----
+
+|=======
+| ls.s3 | indicates logstash plugin s3 |
+| 312bc026-2f5d-49bc-ae9f-5940cf4ad9a6 | a new, random uuid per file. |
+| 2013-04-18T10.00 | represents the time whenever you specify time_file. |
+| tag_hello | indicates the event's tag. |
+| part0 | If you indicate size_file, it will generate more parts if your file.size > size_file. 
+When a file is full, it gets pushed to the bucket and then deleted from the temporary directory. 
+If a file is empty, it is simply deleted.  Empty files will not be pushed. |
+|=======
+
+===== Crash Recovery
+
+This plugin will recover and upload temporary log files after crash/abnormal termination when using `restore` set to true
+
+===== Usage
+This is an example of logstash config:
+[source,ruby]
+output {
+   s3{
+     access_key_id => "crazy_key"             (optional)
+     secret_access_key => "monkey_access_key" (optional)
+     region => "eu-west-1"                    (optional, default = "us-east-1")
+     bucket => "your_bucket"                  (required)
+     size_file => 2048                        (optional) - Bytes
+     time_file => 5                           (optional) - Minutes
+     codec => "plain"                         (optional)
+     canned_acl => "private"                  (optional. Options are "private", "public-read", "public-read-write", "authenticated-read", "aws-exec-read", "bucket-owner-read", "bucket-owner-full-control", "log-delivery-write". Defaults to "private" )
+   }
+
+
+[id="plugins-{type}s-{plugin}-options"]
+==== S3 Output Configuration Options
+
+This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+|Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-access_key_id>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-additional_settings>> |<<hash,hash>>|No
+| <<plugins-{type}s-{plugin}-aws_credentials_file>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-bucket>> |<<string,string>>|Yes
+| <<plugins-{type}s-{plugin}-canned_acl>> |<<string,string>>, one of `["private", "public-read", "public-read-write", "authenticated-read", "aws-exec-read", "bucket-owner-read", "bucket-owner-full-control", "log-delivery-write"]`|No
+| <<plugins-{type}s-{plugin}-encoding>> |<<string,string>>, one of `["none", "gzip"]`|No
+| <<plugins-{type}s-{plugin}-endpoint>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-prefix>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-proxy_uri>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-region>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-restore>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-retry_count>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-retry_delay>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-role_arn>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-role_session_name>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-rotation_strategy>> |<<string,string>>, one of `["size_and_time", "size", "time"]`|No
+| <<plugins-{type}s-{plugin}-secret_access_key>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-server_side_encryption>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-server_side_encryption_algorithm>> |<<string,string>>, one of `["AES256", "aws:kms"]`|No
+| <<plugins-{type}s-{plugin}-session_token>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-signature_version>> |<<string,string>>, one of `["v2", "v4"]`|No
+| <<plugins-{type}s-{plugin}-size_file>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-ssekms_key_id>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-storage_class>> |<<string,string>>, one of `["STANDARD", "REDUCED_REDUNDANCY", "STANDARD_IA", "ONEZONE_IA"]`|No
+| <<plugins-{type}s-{plugin}-temporary_directory>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-time_file>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-upload_multipart_threshold>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-upload_queue_size>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-upload_workers_count>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-validate_credentials_on_root_bucket>> |<<boolean,boolean>>|No
+|=======================================================================
+
+Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
+output plugins.
+
+&nbsp;
+
+[id="plugins-{type}s-{plugin}-access_key_id"]
+===== `access_key_id` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order:
+
+. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config
+. External credentials file specified by `aws_credentials_file`
+. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
+. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY`
+. IAM Instance Profile (available when running inside EC2)
+
+[id="plugins-{type}s-{plugin}-additional_settings"]
+===== `additional_settings`
+
+  * Value type is <<hash,hash>>
+  * Default value is `{}`
+
+Key-value pairs of settings and corresponding values used to parametrize
+the connection to S3. See full list in https://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html[the AWS SDK documentation]. Example:
+
+[source,ruby]
+    output {
+      s3 {
+        access_key_id => "1234",
+        secret_access_key => "secret",
+        region => "eu-west-1",
+        bucket => "logstash-test",
+        additional_settings => {
+          "force_path_style" => true,
+          "follow_redirects" => false
+        }
+      }
+    }
+
+[id="plugins-{type}s-{plugin}-aws_credentials_file"]
+===== `aws_credentials_file` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Path to YAML file containing a hash of AWS credentials.
+This file will only be loaded if `access_key_id` and
+`secret_access_key` aren't set. The contents of the
+file should look like this:
+
+[source,ruby]
+----------------------------------
+    :access_key_id: "12345"
+    :secret_access_key: "54321"
+----------------------------------
+
+
+[id="plugins-{type}s-{plugin}-bucket"]
+===== `bucket` 
+
+  * This is a required setting.
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+S3 bucket
+
+[id="plugins-{type}s-{plugin}-canned_acl"]
+===== `canned_acl` 
+
+  * Value can be any of: `private`, `public-read`, `public-read-write`, `authenticated-read`, `aws-exec-read`, `bucket-owner-read`, `bucket-owner-full-control`, `log-delivery-write`
+  * Default value is `"private"`
+
+The S3 canned ACL to use when putting the file. Defaults to "private".
+
+[id="plugins-{type}s-{plugin}-encoding"]
+===== `encoding` 
+
+  * Value can be any of: `none`, `gzip`
+  * Default value is `"none"`
+
+Specify the content encoding. Supports ("gzip"). Defaults to "none"
+
+[id="plugins-{type}s-{plugin}-endpoint"]
+===== `endpoint`
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The endpoint to connect to. By default it is constructed using the value of `region`.
+This is useful when connecting to S3 compatible services, but beware that these aren't
+guaranteed to work correctly with the AWS SDK.
+The endpoint should be an HTTP or HTTPS URL, e.g. https://example.com
+
+[id="plugins-{type}s-{plugin}-prefix"]
+===== `prefix` 
+
+  * Value type is <<string,string>>
+  * Default value is `""`
+
+Specify a prefix to the uploaded filename to simulate directories on S3.
+Prefix does not require leading slash.
+This option supports
+{logstash-ref}/event-dependent-configuration.html#sprintf[Logstash
+interpolation]. For example, files can be prefixed with the event date using
+`prefix = "%{+YYYY}/%{+MM}/%{+dd}"`.
+
+IMPORTANT: Take care when you are using interpolated strings in prefixes. This
+has the potential to create large numbers of unique prefixes, causing large
+numbers of in-progress uploads. This scenario may result in performance and
+stability issues, which can be further exacerbated when you use a
+rotation_strategy that delays uploads.
+
+[id="plugins-{type}s-{plugin}-proxy_uri"]
+===== `proxy_uri` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+URI to proxy server if required
+
+[id="plugins-{type}s-{plugin}-region"]
+===== `region` 
+
+  * Value type is <<string,string>>
+  * Default value is `"us-east-1"`
+
+The AWS Region
+
+[id="plugins-{type}s-{plugin}-restore"]
+===== `restore` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `true`
+
+Used to enable recovery after crash/abnormal termination.
+Temporary log files will be recovered and uploaded.
+
+NOTE: If you're using multiple S3 outputs, always set
+<<plugins-{type}s-{plugin}-temporary_directory>> to a
+unique directory. Otherwise the recovery mechanism won't work correctly.
+
+[id="plugins-{type}s-{plugin}-retry_count"]
+===== `retry_count`
+
+* Value type is <<number,number>>
+* Default value is `Infinity`
+
+Allows to limit number of retries when S3 uploading fails.
+
+[id="plugins-{type}s-{plugin}-retry_delay"]
+===== `retry_delay`
+
+* Value type is <<number,number>>
+* Default value is `1`
+
+Delay (in seconds) to wait between consecutive retries on upload failures.
+
+[id="plugins-{type}s-{plugin}-role_arn"]
+===== `role_arn`
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The AWS IAM Role to assume, if any.
+This is used to generate temporary credentials, typically for cross-account access.
+See the https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html[AssumeRole API documentation] for more information.
+
+[id="plugins-{type}s-{plugin}-role_session_name"]
+===== `role_session_name`
+
+  * Value type is <<string,string>>
+  * Default value is `"logstash"`
+
+Session name to use when assuming an IAM role.
+
+[id="plugins-{type}s-{plugin}-rotation_strategy"]
+===== `rotation_strategy` 
+
+  * Value can be any of: `size_and_time`, `size`, `time`
+  * Default value is `"size_and_time"`
+
+Controls when to close the file and push it to S3. 
+
+If you set this value to `size`, it uses the value set in
+<<plugins-{type}s-{plugin}-size_file,`size_file`>>.
+If you set this value to `time`, it uses the value set in
+<<plugins-{type}s-{plugin}-time_file,`time_file`>>.
+If you set this value to `size_and_time`, it uses the values from
+<<plugins-{type}s-{plugin}-size_file,`size_file`>> and
+<<plugins-{type}s-{plugin}-time_file,`time_file`>>, and splits the file when
+either one matches. 
+
+The default strategy checks both size and time. The first value to
+match triggers file rotation.
+
+[id="plugins-{type}s-{plugin}-secret_access_key"]
+===== `secret_access_key` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The AWS Secret Access Key
+
+[id="plugins-{type}s-{plugin}-server_side_encryption"]
+===== `server_side_encryption` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+Specifies whether or not to use S3's server side encryption. Defaults to no encryption.
+
+[id="plugins-{type}s-{plugin}-server_side_encryption_algorithm"]
+===== `server_side_encryption_algorithm` 
+
+  * Value can be any of: `AES256`, `aws:kms`
+  * Default value is `"AES256"`
+
+Specifies what type of encryption to use when SSE is enabled.
+
+[id="plugins-{type}s-{plugin}-session_token"]
+===== `session_token` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The AWS Session token for temporary credential
+
+[id="plugins-{type}s-{plugin}-signature_version"]
+===== `signature_version` 
+
+  * Value can be any of: `v2`, `v4`
+  * There is no default value for this setting.
+
+The version of the S3 signature hash to use. Normally uses the internal client default, can be explicitly
+specified here
+
+[id="plugins-{type}s-{plugin}-size_file"]
+===== `size_file` 
+
+  * Value type is <<number,number>>
+  * Default value is `5242880`
+
+Set the file size in bytes. When the number of bytes exceeds the `size_file`
+value, a new file is created. If you use tags, Logstash generates a specific size
+file for every tag.
+
+[id="plugins-{type}s-{plugin}-ssekms_key_id"]
+===== `ssekms_key_id` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The key to use when specified along with server_side_encryption => aws:kms.
+If server_side_encryption => aws:kms is set but this is not default KMS key is used.
+http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html
+
+[id="plugins-{type}s-{plugin}-storage_class"]
+===== `storage_class` 
+
+  * Value can be any of: `STANDARD`, `REDUCED_REDUNDANCY`, `STANDARD_IA`, `ONEZONE_IA`
+  * Default value is `"STANDARD"`
+
+Specifies what S3 storage class to use when uploading the file.
+More information about the different storage classes can be found:
+http://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html
+Defaults to STANDARD.
+
+[id="plugins-{type}s-{plugin}-temporary_directory"]
+===== `temporary_directory` 
+
+  * Value type is <<string,string>>
+  * Default value is `"/tmp/logstash"`
+
+Set the directory where logstash will store the tmp files before sending it to S3
+default to the current OS temporary directory in linux `/tmp/logstash`.
+
+WARNING: Using multiple S3 outputs with `restore => true` requires unique directories
+per output. All of the directory's contents are processed and deleted upon recovery, and shared or nested directories can cause data loss.
+For example, an output using `/tmp/s3` and a second configured with `/tmp/s3/sub` would
+cause issues. Having temporary directories `/tmp/s3/sub1` and `/tmp/s3/sub2` is fine.
+
+[id="plugins-{type}s-{plugin}-time_file"]
+===== `time_file` 
+
+  * Value type is <<number,number>>
+  * Default value is `15`
+
+Set the time, in MINUTES, to close the current sub_time_section of bucket.
+If <<plugins-{type}s-{plugin}-rotation_strategy,`rotation_strategy`>> is set to `time` or `size_and_time`, then `time_file` cannot be set to 0. 
+Otherwise, the plugin raises a configuration error.
+
+[id="plugins-{type}s-{plugin}-upload_multipart_threshold"]
+===== `upload_multipart_threshold`
+
+  * Value type is <<number,number>>
+  * Default value is `15728640`
+
+Files larger than this number are uploaded using the S3 multipart APIs
+
+[id="plugins-{type}s-{plugin}-upload_queue_size"]
+===== `upload_queue_size` 
+
+  * Value type is <<number,number>>
+  * Default value is `4`
+
+Number of items we can keep in the local queue before uploading them
+
+[id="plugins-{type}s-{plugin}-upload_workers_count"]
+===== `upload_workers_count` 
+
+  * Value type is <<number,number>>
+  * Default value is `4`
+
+Specify how many workers to use to upload the files to S3
+
+[id="plugins-{type}s-{plugin}-validate_credentials_on_root_bucket"]
+===== `validate_credentials_on_root_bucket` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `true`
+
+The common use case is to define permissions on the root bucket and give Logstash
+full access to write logs. 
+In some circumstances, you need more granular permissions on the subfolder. This
+allows you to disable the check at startup.
+
+[id="plugins-{type}s-{plugin}-common-options"]
+include::{include_path}/{type}.asciidoc[]
+
+:default_codec!:

data/lib/logstash/outputs/s3/file_repository.rb

@@ -0,0 +1,177 @@
+# encoding: utf-8
+require "java"
+require "concurrent/map"
+require "concurrent/timer_task"
+require "logstash/util"
+
+module LogStash
+  module Outputs
+    class S3
+      class FileRepository
+        DEFAULT_STATE_SWEEPER_INTERVAL_SECS = 60
+        DEFAULT_STALE_TIME_SECS = 15 * 60
+        # Ensure that all access or work done
+        # on a factory is threadsafe
+        class PrefixedValue
+          def initialize(file_factory, stale_time)
+            @file_factory = file_factory
+            @lock = Monitor.new
+            @stale_time = stale_time
+            @is_deleted = false
+          end
+
+          def with_lock
+            @lock.synchronize {
+              yield @file_factory
+            }
+          end
+
+          def stale?
+            with_lock { |factory| factory.current.size == 0 && (Time.now - factory.current.ctime > @stale_time) }
+          end
+
+          def apply(prefix)
+            return self
+          end
+
+          def delete!
+            with_lock do |factory|
+              factory.current.delete!
+              @is_deleted = true
+            end
+          end
+
+          def deleted?
+            with_lock { |_| @is_deleted }
+          end
+        end
+
+        class FactoryInitializer
+
+          def initialize(tags, encoding, temporary_directory, stale_time)
+            @tags = tags
+            @encoding = encoding
+            @temporary_directory = temporary_directory
+            @stale_time = stale_time
+          end
+
+          def create_value(prefix_key)
+            PrefixedValue.new(TemporaryFileFactory.new(prefix_key, @tags, @encoding, @temporary_directory), @stale_time)
+          end
+
+        end
+
+        def initialize(tags, encoding, temporary_directory,
+                       stale_time = DEFAULT_STALE_TIME_SECS,
+                       sweeper_interval = DEFAULT_STATE_SWEEPER_INTERVAL_SECS)
+          # The path need to contains the prefix so when we start
+          # logtash after a crash we keep the remote structure
+          @prefixed_factories = Concurrent::Map.new
+
+          @sweeper_interval = sweeper_interval
+
+          @factory_initializer = FactoryInitializer.new(tags, encoding, temporary_directory, stale_time)
+
+          start_stale_sweeper
+        end
+
+        def keys
+          @prefixed_factories.keys
+        end
+
+        def each_files
+          each_factory(keys) do |factory|
+            yield factory.current
+          end
+        end
+
+        ##
+        # Yields the file factory while the current thread has exclusive access to it, creating a new
+        # one if one does not exist or if the current one is being reaped by the stale watcher.
+        # @param prefix_key [String]: the prefix key
+        # @yieldparam factory [TemporaryFileFactory]: a temporary file factory that this thread has exclusive access to
+        # @yieldreturn [Object]: a value to return; should NOT be the factory, which should be contained by the exclusive access scope.
+        # @return [Object]: the value returned by the provided block
+        def get_factory(prefix_key)
+
+          # fast-path: if factory exists and is not deleted, yield it with exclusive access and return
+          prefix_val = @prefixed_factories.get(prefix_key)
+          prefix_val&.with_lock do |factory|
+            # intentional local-jump to ensure deletion detection
+            # is done inside the exclusive access.
+            return yield(factory) unless prefix_val.deleted?
+          end
+
+          # slow-path:
+          # the Concurrent::Map#get operation is lock-free, but may have returned an entry that was being deleted by
+          # another thread (such as via stale detection). If we failed to retrieve a value, or retrieved one that had
+          # been marked deleted, use the atomic Concurrent::Map#compute to retrieve a non-deleted entry.
+          prefix_val = @prefixed_factories.compute(prefix_key) do |existing|
+            existing && !existing.deleted? ? existing : @factory_initializer.create_value(prefix_key)
+          end
+          prefix_val.with_lock { |factory| yield factory }
+        end
+
+        ##
+        # Yields each non-deleted file factory while the current thread has exclusive access to it.
+        # @param prefixes [Array<String>]: the prefix keys
+        # @yieldparam factory [TemporaryFileFactory]
+        # @return [void]
+        def each_factory(prefixes)
+          prefixes.each do |prefix_key|
+            prefix_val = @prefixed_factories.get(prefix_key)
+            prefix_val&.with_lock do |factory|
+              yield factory unless prefix_val.deleted?
+            end
+          end
+        end
+
+        def get_file(prefix_key)
+          get_factory(prefix_key) { |factory| yield factory.current }
+        end
+
+        def shutdown
+          stop_stale_sweeper
+        end
+
+        def size
+          @prefixed_factories.size
+        end
+
+        def remove_if_stale(prefix_key)
+          # we use the ATOMIC `Concurrent::Map#compute_if_present` to atomically
+          # detect the staleness, mark a stale prefixed factory as deleted, and delete from the map.
+          @prefixed_factories.compute_if_present(prefix_key) do |prefixed_factory|
+            # once we have retrieved an instance, we acquire exclusive access to it
+            # for stale detection, marking it as deleted before releasing the lock
+            # and causing it to become deleted from the map.
+            prefixed_factory.with_lock do |_|
+              if prefixed_factory.stale?
+                prefixed_factory.delete! # mark deleted to prevent reuse
+                nil # cause deletion
+              else
+                prefixed_factory # keep existing
+              end
+            end
+          end
+        end
+
+        def start_stale_sweeper
+          @stale_sweeper = Concurrent::TimerTask.new(:execution_interval => @sweeper_interval) do
+            LogStash::Util.set_thread_name("S3, Stale factory sweeper")
+
+            @prefixed_factories.keys.each do |prefix|
+              remove_if_stale(prefix)
+            end
+          end
+
+          @stale_sweeper.execute
+        end
+
+        def stop_stale_sweeper
+          @stale_sweeper.shutdown
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/s3/patch.rb

@@ -0,0 +1,22 @@
+# This is patch related to the autoloading and ruby
+#
+# The fix exist in jruby 9k but not in the current jruby, not sure when or it will be backported
+# https://github.com/jruby/jruby/issues/3645
+#
+# AWS is doing tricky name discovery in the module to generate the correct error class and
+# this strategy is bogus in jruby and `eager_autoload` don't fix this issue.
+#
+# This will be a short lived patch since AWS is removing the need.
+# see: https://github.com/aws/aws-sdk-ruby/issues/1301#issuecomment-261115960
+old_stderr = $stderr
+
+$stderr = StringIO.new
+begin
+  module Aws
+    const_set(:S3, Aws::S3)
+  end
+ensure
+  $stderr = old_stderr
+end
+
+

data/lib/logstash/outputs/s3/path_validator.rb

@@ -0,0 +1,18 @@
+# encoding: utf-8
+module LogStash
+  module Outputs
+    class S3
+      class PathValidator
+        INVALID_CHARACTERS = "\^`><"
+
+        def self.valid?(name)
+          name.match(matches_re).nil?
+        end
+
+        def self.matches_re
+          /[#{Regexp.escape(INVALID_CHARACTERS)}]/
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/s3/size_and_time_rotation_policy.rb

@@ -0,0 +1,24 @@
+# encoding: utf-8
+require "logstash/outputs/s3/size_rotation_policy"
+require "logstash/outputs/s3/time_rotation_policy"
+
+module LogStash
+  module Outputs
+    class S3
+      class SizeAndTimeRotationPolicy
+        def initialize(file_size, time_file)
+          @size_strategy = SizeRotationPolicy.new(file_size)
+          @time_strategy = TimeRotationPolicy.new(time_file)
+        end
+
+        def rotate?(file)
+          @size_strategy.rotate?(file) || @time_strategy.rotate?(file)
+        end 
+
+        def needs_periodic?
+          true
+        end
+      end
+    end
+  end
+end