fluent-plugin-gcs 0.3.0 → 0.4.0.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c9164a1b5a8a40d60121894c7d13a77a85dacb57
-  data.tar.gz: 1557660c042ee21cfd6885546c8ca9aefb52719a
+  metadata.gz: d4e0e0d0018992e06145a4982a26182240998f12
+  data.tar.gz: 5c96a75ae786b74be8f8092ed210ebae68be3ad6
 SHA512:
-  metadata.gz: 73b599c4b9519adbf1b8f67164c2e24151e1305df2da5cceecce59df7ec80cc22657e397a0b66c8a2cbc26aaa60fa7f60eabac61eae22e644382dd5bc14e1642
-  data.tar.gz: 95b29243808c6c09f15b443fb2a6ac6386a0f596e90fff71c21c87a9d2502ecd66f38efaaa8c3c12e5732172215af3098aaf8f4104d5c2c8fc25be5432a2d750
+  metadata.gz: a6d11e853d46efff711d2a0e2f2aeab878ad0923cd7230923bdf90da33ae98d11f7789a76c042faa01151473505e56a98d477904d246a876ca593582491e7f4a
+  data.tar.gz: 0fcffefaf44dc28f0d29d97c6f26e2cd6047ebaa8092eea65726c46637bb9073d31f1d0998ad94de502b1bdd8225f89429a89a9c93b6ed35338495474e0ab499

data/CHANGELOG.md

@@ -1,16 +1,32 @@
-## ChangeLog
+## [Unreleased]
 
-### Release 0.3.0 - 2017/02/28
+## [0.4.0] - xxxx/xx/xx
 
-- Add support for setting a File's storage_class on file creation
+New features / Enhancements
+
+- [Support v0.14 (by @cosmo0920)](https://github.com/daichirata/fluent-plugin-gcs/pull/6)
+
+## [0.3.0] - 2017/02/28
+
+New features / Enhancements
+
+- [Add support for setting a File's storage_class on file creation](https://github.com/daichirata/fluent-plugin-gcs/pull/4)
   - see also https://cloud.google.com/storage/docs/storage-classes
 
-### Release 0.2.0 - 2017/01/16
+## [0.2.0] - 2017/01/16
+
+Bug fixes
 
-- Remove encryption_key_sha256 parameter.
+- [Remove encryption_key_sha256 parameter.](https://github.com/daichirata/fluent-plugin-gcs/pull/2)
   - see also. https://github.com/GoogleCloudPlatform/google-cloud-ruby/blob/master/google-cloud-storage/CHANGELOG.md#0230--2016-12-8
 
-### Release 0.1.1 - 2016/11/28
+## [0.1.1] - 2016/11/28
+
+New features / Enhancements
 
 - Add support for `%{hostname}` of object_key_format
 
+[Unreleased]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.3.0...HEAD
+[0.3.0]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.2.0...v0.3.0
+[0.2.0]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.1.a...v0.2.0
+[0.1.1]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.1.0...v0.1.1

data/README.md

@@ -3,14 +3,54 @@
 
 Google Cloud Storage output plugin for [Fluentd](https://github.com/fluent/fluentd).
 
+## Requirements
+
+| fluent-plugin-gcs  | fluentd    | ruby   |
+|--------------------|------------|--------|
+| >= 0.4.0           | >= v0.14.0 | >= 2.1 |
+|  < 0.4.0           | >= v0.12.0 | >= 1.9 |
+
+NOTE: fluent-plugin-gcs v0.4.0 is now beta.
+
 ## Installation
 
 ``` shell
-gem install fluent-plugin-gcs
+$ gem install fluent-plugin-gcs -v "~> 0.3"  --no-document # for fluentd v0.12 or later
+$ gem install fluent-plugin-gcs -v "0.4.0.beta1" --no-document # for fluentd v0.14 or later
 ```
 
 ## Examples
 
+### For v0.14 style
+
+```
+<match pattern>
+  @type gcs
+
+  project YOUR_PROJECT
+  keyfile YOUR_KEYFILE_PATH
+  bucket YOUR_GCS_BUCKET_NAME
+  object_key_format %{path}%{time_slice}_%{index}.%{file_extension}
+  path logs/${tag}/%Y/%m/%d/
+
+  # if you want to use ${tag} or %Y/%m/%d/ like syntax in path / object_key_format,
+  # need to specify tag for ${tag} and time for %Y/%m/%d in <buffer> argument.
+  <buffer tag,time>
+    @type file
+    path /var/log/fluent/gcs
+    timekey 1h # 1 hour partition
+    timekey_wait 10m
+    timekey_use_utc true # use utc
+  </buffer>
+
+  <format>
+    @type json
+  </format>
+</match>
+```
+
+### For v0.12 style
+
 ```
 <match pattern>
   @type gcs

data/fluent-plugin-gcs.gemspec

@@ -20,8 +20,8 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "fluentd", "~> 0.12.0"
-  spec.add_runtime_dependency "google-cloud-storage", "~> 0.23.2"
+  spec.add_runtime_dependency "fluentd", [">= 0.14.0", "< 2"]
+  spec.add_runtime_dependency "google-cloud-storage", "~> 1.1.0"
 
   spec.add_development_dependency "bundler", "~> 1.13"
   spec.add_development_dependency "rake", "~> 10.0"

data/lib/fluent/plugin/gcs/version.rb

@@ -1,5 +1,5 @@
 module Fluent
   module GCSPlugin
-    VERSION = "0.3.0"
+    VERSION = "0.4.0.beta1"
   end
 end

data/lib/fluent/plugin/out_gcs.rb

@@ -4,11 +4,14 @@ require "socket"
 
 require "fluent/plugin/gcs/object_creator"
 require "fluent/plugin/gcs/version"
+require "fluent/plugin/output"
 
-module Fluent
-  class GCSOutput < TimeSlicedOutput
+module Fluent::Plugin
+  class GCSOutput < Output
     Fluent::Plugin.register_output("gcs", self)
 
+    helpers :compat_parameters, :formatter, :inject
+
     def initialize
       super
       require "google/cloud/storage"
@@ -51,9 +54,21 @@ module Fluent
       config_param :value, :string, default: ""
     end
 
+    DEFAULT_FORMAT_TYPE = "out_file"
+
+    config_section :format do
+      config_set_default :@type, DEFAULT_FORMAT_TYPE
+    end
+
+    config_section :buffer do
+      config_set_default :chunk_keys, ['time']
+      config_set_default :timekey, (60 * 60 * 24)
+    end
+
     MAX_HEX_RANDOM_LENGTH = 32
 
     def configure(conf)
+      compat_parameters_convert(conf, :buffer, :formatter, :inject)
       super
 
       if @hex_random_length > MAX_HEX_RANDOM_LENGTH
@@ -69,10 +84,13 @@ module Fluent
         @object_metadata_hash = @object_metadata.map {|m| [m.key, m.value] }.to_h
       end
 
-      @formatter = Fluent::Plugin.new_formatter(@format)
-      @formatter.configure(conf)
+      @formatter = formatter_create
 
       @object_creator = Fluent::GCS.discovered_object_creator(@store_as, transcoding: @transcoding)
+      # For backward compatibility
+      # TODO: Remove time_slice_format when end of support compat_parameters
+      @configured_time_slice_format = conf['time_slice_format']
+      @time_slice_with_tz = Fluent::Timezone.formatter(@timekey_zone, @configured_time_slice_format || timekey_to_timeformat(@buffer_config['timekey']))
     end
 
     def start
@@ -89,7 +107,12 @@ module Fluent
     end
 
     def format(tag, time, record)
-      @formatter.format(tag, time, record)
+      r = inject_values_to_record(tag, time, record)
+      @formatter.format(tag, time, r)
+    end
+
+    def multi_workers_ready?
+      true
     end
 
     def write(chunk)
@@ -126,22 +149,24 @@ module Fluent
       Digest::MD5.hexdigest(chunk.unique_id)[0...@hex_random_length]
     end
 
-    def format_path(chunk)
-      now = Time.strptime(chunk.key, @time_slice_format)
-      (@localtime ? now : now.utc).strftime(@path)
-    end
-
     def generate_path(chunk, i = 0, prev = nil)
+      metadata = chunk.metadata
+      time_slice = if metadata.timekey.nil?
+                     ''.freeze
+                   else
+                     @time_slice_with_tz.call(metadata.timekey)
+                   end
       tags = {
         "%{file_extension}" => @object_creator.file_extension,
         "%{hex_random}" => hex_random(chunk),
         "%{hostname}" => Socket.gethostname,
         "%{index}" => i,
-        "%{path}" => format_path(chunk),
-        "%{time_slice}" => chunk.key,
+        "%{path}" => @path,
+        "%{time_slice}" => time_slice,
         "%{uuid_flush}" => SecureRandom.uuid,
       }
       path = @object_key_format.gsub(Regexp.union(tags.keys), tags)
+      path = extract_placeholders(path, metadata)
       return path unless @gcs_bucket.find_file(path, @encryption_opts)
 
       if path == prev
@@ -153,5 +178,16 @@ module Fluent
       end
       generate_path(chunk, i + 1, path)
     end
+
+    # This is stolen from Fluentd
+    def timekey_to_timeformat(timekey)
+      case timekey
+      when nil          then ''
+      when 0...60       then '%Y%m%d%H%M%S' # 60 exclusive
+      when 60...3600    then '%Y%m%d%H%M'
+      when 3600...86400 then '%Y%m%d%H'
+      else                   '%Y%m%d'
+      end
+    end
   end
 end

metadata

@@ -1,43 +1,49 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-gcs
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0.beta1
 platform: ruby
 authors:
 - Daichi HIRATA
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-02-28 00:00:00.000000000 Z
+date: 2017-06-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.12.0
+        version: 0.14.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 0.12.0
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: google-cloud-storage
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.23.2
+        version: 1.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.23.2
+        version: 1.1.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -157,9 +163,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.5.1