fluent-plugin-s3 0.7.2 → 0.8.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 55e080d8a3659c9031a55c444803d30f9fc748de
-  data.tar.gz: 09d5d7227c24f4f997e88d7a5d8ea41fb24aaf1a
+  metadata.gz: 396e2f615eeaf603b9240f364dea07db8a7e92ef
+  data.tar.gz: 64b0a5e72ac0ee484e676d15417c8c44536001c9
 SHA512:
-  metadata.gz: 49e4ba0663061d8443a9603a0d465b491a23f4e286d1e620d8f1cb8a5bbde204c0a274d60c088181e22b181a702881ca39664764b88fc02ad75d73615a604ebc
-  data.tar.gz: 13da940d95c312cf99f7fc632493a2d1c5b36e397528212a45de930520c61250e7f798a2d6f498d9d7e6825f5b660e7cedfabb6462b33d53782eafd66d600ec5
+  metadata.gz: 3bc5c5e87a48959943863e011ea8d12532b348602cb94f76c6c050af4e07bafd7d6f175e7a1c5f111187a8c7d63e994a98983f50a3af168550c7dc3f6644e137
+  data.tar.gz: 4763fb32496ccb0cea8565aea1b2b553fefa015cfaf952f31f7eb7d15a5361ba46a2545c8fbe4296c2d63e6de1290263c07d9cce265935de5bca5236e0939136

data/Gemfile

@@ -1,5 +1,7 @@
 source "http://rubygems.org"
 
 gem 'json', '= 1.8.3'
+gem 'uuidtools'
+gem 'fluentd', '= 0.14.8'
 
 gemspec

data/README.md

@@ -85,8 +85,9 @@ The format of S3 object keys. You can use several built-in variables:
 *   %{time_slice}
 *   %{index}
 *   %{file_extension}
-*   %{uuid_flush}
 *   %{hex_random}
+*   %{uuid_flush}
+*   %{hostname}
 
 to decide keys dynamically.
 
@@ -98,7 +99,8 @@ the sequential number starts from 0, increments when multiple files are
 uploaded to S3 in the same time slice.
 * %{file_extention} is always "gz" for
 now.
-* %{uuid_flush} a uuid that is replaced everytime the buffer will be flushed
+* %{uuid_flush} a uuid that is replaced everytime the buffer will be flushed. If you want to use this placeholder, install `uuidtools` gem first.
+* %{hostname} is replaced with `Socket.gethostname` result. This is same as "#{Socket.gethostname}".
 * %{hex_random} a random hex string that is replaced for each buffer chunk, not
 assured to be unique. This is used to follow a way of peformance tuning, `Add
 a Hex Hash Prefix to Key Name`, written in [Request Rate and Performance
@@ -130,14 +132,6 @@ You get:
     "log/events/ts=20130111-23/events_1.gz"
     "log/events/ts=20130112-00/events_0.gz"
 
-The
-[fluent-mixin-config-placeholders](https://github.com/tagomoris/fluent-mixin-config-placeholders) mixin is also incorporated, so additional variables such
-as %{hostname}, %{uuid}, etc. can be used in the s3_object_key_format. This
-could prove useful in preventing filename conflicts when writing from multiple
-servers.
-
-    s3_object_key_format %{path}/events/ts=%{time_slice}/events_%{index}-%{hostname}.%{file_extension}
-
 **force_path_style**
 
 :force_path_style (Boolean) — default: false — When set to true, the

data/VERSION

@@ -1 +1 @@
-0.7.2
+0.8.0.rc1

data/fluent-plugin-s3.gemspec

@@ -19,8 +19,6 @@ Gem::Specification.new do |gem|
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
   gem.add_dependency "aws-sdk", [">= 2.3.22", "< 3"]
-  gem.add_dependency "yajl-ruby", "~> 1.0"
-  gem.add_dependency "fluent-mixin-config-placeholders", ">= 0.3.0"
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "rr", "= 1.1.2"
   gem.add_development_dependency "test-unit", ">= 3.0.8"

data/lib/fluent/plugin/out_s3.rb

@@ -1,8 +1,6 @@
 require 'fluent/output'
 
 module Fluent
-  require 'fluent/mixin/config_placeholders'
-
   class S3Output < Fluent::TimeSlicedOutput
     Fluent::Plugin.register_output('s3', self)
 
@@ -14,6 +12,7 @@ module Fluent
       require 'tempfile'
 
       @compressor = nil
+      @uuid_flush_enabled = false
     end
 
     # For fluentd v0.12.16 or earlier
@@ -119,13 +118,8 @@ module Fluent
 
     attr_reader :bucket
 
-    include Fluent::Mixin::ConfigPlaceholders
     MAX_HEX_RANDOM_LENGTH = 16
 
-    def placeholders
-      [:percent]
-    end
-
     def configure(conf)
       super
 
@@ -162,6 +156,8 @@ module Fluent
         $log.warn "reduced_redundancy parameter is deprecated. Use storage_class parameter instead"
         @storage_class = "REDUCED_REDUNDANCY"
       end
+
+      @s3_object_key_format = process_s3_object_key_format
       @values_for_s3_object_chunk = {}
     end
 
@@ -203,8 +199,8 @@ module Fluent
           "time_slice" => chunk.key,
           "file_extension" => @compressor.ext,
           "index" => i,
-          "uuid_flush" => uuid_random,
         }.merge!(@values_for_s3_object_chunk[chunk.unique_id])
+        values_for_s3_object_key['uuid_flush'.freeze] = uuid_random if @uuid_flush_enabled
 
         s3path = @s3_object_key_format.gsub(%r(%{[^}]+})) { |expr|
           values_for_s3_object_key[expr[2...expr.size-1]]
@@ -268,6 +264,10 @@ module Fluent
       unique_hex[0...@hex_random_length]
     end
 
+    def uuid_random
+      ::UUIDTools::UUID.random_create.to_s
+    end
+
     def ensure_bucket
       if !@bucket.exists?
         if @auto_create_bucket
@@ -279,6 +279,34 @@ module Fluent
       end
     end
 
+    def process_s3_object_key_format
+      %W(%{uuid} %{uuid:random} %{uuid:hostname} %{uuid:timestamp}).each { |ph|
+        if @s3_object_key_format.include?(ph)
+          raise ConfigError, %!#{ph} placeholder in s3_object_key_format is removed!
+        end
+      }
+
+      if @s3_object_key_format.include?('%{uuid_flush}')
+        # test uuidtools works or not
+        begin
+          require 'uuidtools'
+        rescue LoadError
+          raise ConfigError, "uuidtools gem not found. Install uuidtools gem first"
+        end
+        begin
+          uuid_random
+        rescue => e
+          raise ConfigError, "Generating uuid doesn't work. Can't use %{uuid_flush} on this environment. #{e}"
+        end
+        @uuid_flush_enabled = true
+      end
+
+      @s3_object_key_format.gsub('%{hostname}') { |expr|
+        log.warn "%{hostname} will be removed in the future. Use \"\#{Socket.gethostname}\" instead"
+        Socket.gethostname
+      }
+    end
+
     def check_apikeys
       @bucket.objects(:prefix => @path).first
     rescue Aws::S3::Errors::NoSuchBucket

data/test/test_out_s3.rb

@@ -251,7 +251,6 @@ class S3OutputTest < Test::Unit::TestCase
   end
 
   CONFIG_TIME_SLICE = %[
-    hostname testing.node.local
     aws_key_id test_key_id
     aws_sec_key test_sec_key
     s3_bucket test_bucket
@@ -299,11 +298,18 @@ class S3OutputTest < Test::Unit::TestCase
   end
 
   def test_write_with_custom_s3_object_key_format_containing_uuid_flush_placeholder
+
+    begin
+      require 'uuidtools'
+    rescue LoadError
+      pend("uuidtools not found. skip this test")
+    end
+
     # Partial mock the S3Bucket, not to make an actual connection to Amazon S3
     setup_mocks(true)
 
     uuid = "5755e23f-9b54-42d8-8818-2ea38c6f279e"
-    stub(UUIDTools::UUID).random_create{ uuid }
+    stub(::UUIDTools::UUID).random_create{ uuid }
 
     s3_local_file_path = "/tmp/s3-test.txt"
     s3path = "log/events/ts=20110102-13/events_0-#{uuid}.gz"
@@ -380,7 +386,7 @@ class S3OutputTest < Test::Unit::TestCase
   end
 
   def setup_s3_object_mocks(params = {})
-    s3path = params[:s3path] || "log/events/ts=20110102-13/events_0-testing.node.local.gz"
+    s3path = params[:s3path] || "log/events/ts=20110102-13/events_0-#{Socket.gethostname}.gz"
     s3_local_file_path = params[:s3_local_file_path] || "/tmp/s3-test.txt"
 
     # Assert content of event logs which are being sent to S3

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 0.7.2
+  version: 0.8.0.rc1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-20 00:00:00.000000000 Z
+date: 2016-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -51,34 +51,6 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '3'
-- !ruby/object:Gem::Dependency
-  name: yajl-ruby
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
-- !ruby/object:Gem::Dependency
-  name: fluent-mixin-config-placeholders
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.3.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.3.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -191,9 +163,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.5.1