fluent-plugin-gcs 0.1.1 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 712de7c582c00ef25cd4c781daee6cfc537d9437
-  data.tar.gz: 40d2803153868f6792d1f157e669ea9ab8b757b4
+SHA256:
+  metadata.gz: 85b1b9b0ed8cd2b264dacc7f54fb1bf9e15b2f1e6f28871b17e0bb7e844e1669
+  data.tar.gz: a90fec1f3d5509be2df1a72d2479e9db6ba7f133ad67c60789abbdbb1360e519
 SHA512:
-  metadata.gz: 8685cb9091a416bf6573c52fb675253eff9ed54a588a34af45baf9ca2f3556945f6b7ec1117f90fa2e54947c421626e2f44e3c1b08d9d873f72e329cf60a6f83
-  data.tar.gz: 687066ae6f1c8201bfe8e9b5a61c7ed9d1b765bc10b95ec5aa2305a272eaf6bb9d5fc4f606804737f51d753ba6567e554ed68fd20479c1ef69f411000dff3744
+  metadata.gz: 8bcb9225215a56ea5ad0d34ac2447e0b39074ae9269e885c2ac70ba2679213340653b47941ea2c5c395861535fd565ae81a6e153e48adef175cf6966b1849a20
+  data.tar.gz: dfe13e0a8ea31f3d80d1174aa4c35385cf6206820d5574f31d501767e972b144f646a48711e64c63b10261045385aed4af9a2e7516aeeac0f6aa460cb37cf807

data/.travis.yml

@@ -1,9 +1,9 @@
 language: ruby
 
 rvm:
-  - 2.1
-  - 2.2
-  - 2.3.0
+  - 2.6
+  - 2.7
+  - ruby-head
 
 before_install:
   - gem update bundler

data/CHANGELOG.md

@@ -0,0 +1,40 @@
+## [Unreleased]
+
+New features / Enhancements
+
+## [0.4.1] - 2020/04/17
+
+New features
+- [Support blind write to GSC](https://github.com/daichirata/fluent-plugin-gcs/pull/14)
+
+## [0.4.0] - 2019/04/01
+
+New features / Enhancements
+
+- [Support v0.14 (by @cosmo0920)](https://github.com/daichirata/fluent-plugin-gcs/pull/6)
+
+## [0.3.0] - 2017/02/28
+
+New features / Enhancements
+
+- [Add support for setting a File's storage_class on file creation](https://github.com/daichirata/fluent-plugin-gcs/pull/4)
+  - see also https://cloud.google.com/storage/docs/storage-classes
+
+## [0.2.0] - 2017/01/16
+
+Bug fixes
+
+- [Remove encryption_key_sha256 parameter.](https://github.com/daichirata/fluent-plugin-gcs/pull/2)
+  - see also. https://github.com/GoogleCloudPlatform/google-cloud-ruby/blob/master/google-cloud-storage/CHANGELOG.md#0230--2016-12-8
+
+## [0.1.1] - 2016/11/28
+
+New features / Enhancements
+
+- Add support for `%{hostname}` of object_key_format
+
+[Unreleased]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.4.0...HEAD
+[0.4.0]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.3.0...v0.4.0
+[0.3.0]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.2.0...v0.3.0
+[0.2.0]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.1.0...v0.2.0
+[0.1.1]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.1.0...v0.1.1

data/README.md

@@ -3,14 +3,52 @@
 
 Google Cloud Storage output plugin for [Fluentd](https://github.com/fluent/fluentd).
 
+## Requirements
+
+| fluent-plugin-gcs  | fluentd    | ruby   |
+|--------------------|------------|--------|
+| >= 0.4.0           | >= v0.14.0 | >= 2.4 |
+|  < 0.4.0           | >= v0.12.0 | >= 1.9 |
+
 ## Installation
 
 ``` shell
-gem install fluent-plugin-gcs
+$ gem install fluent-plugin-gcs -v "~> 0.3" --no-document # for fluentd v0.12 or later
+$ gem install fluent-plugin-gcs -v "0.4.0" --no-document # for fluentd v0.14 or later
 ```
 
 ## Examples
 
+### For v0.14 style
+
+```
+<match pattern>
+  @type gcs
+
+  project YOUR_PROJECT
+  keyfile YOUR_KEYFILE_PATH
+  bucket YOUR_GCS_BUCKET_NAME
+  object_key_format %{path}%{time_slice}_%{index}.%{file_extension}
+  path logs/${tag}/%Y/%m/%d/
+
+  # if you want to use ${tag} or %Y/%m/%d/ like syntax in path / object_key_format,
+  # need to specify tag for ${tag} and time for %Y/%m/%d in <buffer> argument.
+  <buffer tag,time>
+    @type file
+    path /var/log/fluent/gcs
+    timekey 1h # 1 hour partition
+    timekey_wait 10m
+    timekey_use_utc true # use utc
+  </buffer>
+
+  <format>
+    @type json
+  </format>
+</match>
+```
+
+### For v0.12 style
+
 ```
 <match pattern>
   @type gcs
@@ -127,8 +165,6 @@ See also [official Formatter article](http://docs.fluentd.org/articles/formatter
 
 Create GCS bucket if it does not exists. Default is true.
 
-TODO: rate limit
-
 **acl**
 
 Permission for the object in GCS. Acceptable values are:
@@ -142,10 +178,25 @@ Permission for the object in GCS. Acceptable values are:
 
 Default is nil (bucket default object ACL). See also [official document](https://cloud.google.com/storage/docs/access-control/lists).
 
-**encryption_key**, **encryption_key_sha256**
+**storage_class**
+
+Storage class of the file. Acceptable values are:
+
+* `dra`            - Durable Reduced Availability
+* `nearline`       - Nearline Storage
+* `coldline`       - Coldline Storage
+* `multi_regional` - Multi-Regional Storage
+* `regional`       - Regional Storage
+* `standard`       - Standard Storage
+
+Default is nil. See also [official document](https://cloud.google.com/storage/docs/storage-classes).
+
+**encryption_key**
 
 You can also choose to provide your own AES-256 key for server-side encryption. See also [Customer-supplied encryption keys](https://cloud.google.com/storage/docs/encryption#customer-supplied).
 
+`encryption_key_sha256` will be calculated using encryption_key.
+
 **overwrite**
 
 Overwrite already existing path. Default is false, which raises an error
@@ -174,9 +225,18 @@ Use Local time instead of UTC.
 
 Use UTC instead of local time.
 
-
 And see [official Time Sliced Output article](http://docs.fluentd.org/articles/output-plugin-overview#time-sliced-output-parameters)
 
+**blind_write**
+
+Doesn't check if an object exists in GCS before writing. Default is false.
+
+Allows to avoid granting of `storage.objects.get` permission.
+
+Warning! If the object exists and `storage.objects.delete` permission is not
+granted, it will result in an unrecoverable error. Usage of `%{hex_random}` is
+recommended.
+
 ### ObjectMetadata
 
 User provided web-safe keys and arbitrary string values that will returned with requests for the file as "x-goog-meta-" response headers.

data/fluent-plugin-gcs.gemspec

@@ -20,11 +20,10 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "fluentd", "~> 0.12.0"
-  spec.add_runtime_dependency "google-cloud-storage", "~> 0.21"
+  spec.add_runtime_dependency "fluentd", [">= 0.14.22", "< 2"]
+  spec.add_runtime_dependency "google-cloud-storage", "~> 1.1"
 
-  spec.add_development_dependency "bundler", "~> 1.13"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", ">= 12.3.3"
   spec.add_development_dependency "rr", "= 1.1.2"
   spec.add_development_dependency "test-unit", ">= 3.0.8"
   spec.add_development_dependency "test-unit-rr", ">= 1.0.3"

data/lib/fluent/plugin/gcs/version.rb

@@ -1,5 +1,5 @@
 module Fluent
   module GCSPlugin
-    VERSION = "0.1.1"
+    VERSION = "0.4.1"
   end
 end

data/lib/fluent/plugin/out_gcs.rb

@@ -4,11 +4,14 @@ require "socket"
 
 require "fluent/plugin/gcs/object_creator"
 require "fluent/plugin/gcs/version"
+require "fluent/plugin/output"
 
-module Fluent
-  class GCSOutput < TimeSlicedOutput
+module Fluent::Plugin
+  class GCSOutput < Output
     Fluent::Plugin.register_output("gcs", self)
 
+    helpers :compat_parameters, :formatter, :inject
+
     def initialize
       super
       require "google/cloud/storage"
@@ -28,7 +31,7 @@ module Fluent
                  desc: "Format of GCS object keys"
     config_param :path, :string, default: "",
                  desc: "Path prefix of the files on GCS"
-    config_param :store_as, :enum, list: [:gzip, :json, :text], default: :gzip,
+    config_param :store_as, :enum, list: %i(gzip json text), default: :gzip,
                  desc: "Archive format on GCS"
     config_param :transcoding, :bool, default: false,
                  desc: "Enable the decompressive form of transcoding"
@@ -40,46 +43,56 @@ module Fluent
                  desc: "Overwrite already existing path"
     config_param :format, :string, default: "out_file",
                  desc: "Change one line format in the GCS object"
-    config_param :acl, :enum, list: [:auth_read, :owner_full, :owner_read, :private, :project_private, :public_read], default: nil,
+    config_param :acl, :enum, list: %i(auth_read owner_full owner_read private project_private public_read), default: nil,
                  desc: "Permission for the object in GCS"
+    config_param :storage_class, :enum, list: %i(dra nearline coldline multi_regional regional standard), default: nil,
+                 desc: "Storage class of the file"
     config_param :encryption_key, :string, default: nil, secret: true,
                  desc: "Customer-supplied, AES-256 encryption key"
-    config_param :encryption_key_sha256, :string, default: nil, secret: true,
-                 desc: "SHA256 hash of the customer-supplied, AES-256 encryption key"
+    config_param :blind_write, :bool, default: false,
+                 desc: "Whether to check if object already exists by given GCS path. Allows avoiding giving storage.object.get permission"
     config_section :object_metadata, required: false do
       config_param :key, :string, default: ""
       config_param :value, :string, default: ""
     end
-    # TODO: gem "google-cloud-storage" does not support object lavel storage_class yet.
-    # config_param :storage_class, :string, default: "regional"
+
+    DEFAULT_FORMAT_TYPE = "out_file"
+
+    config_section :format do
+      config_set_default :@type, DEFAULT_FORMAT_TYPE
+    end
+
+    config_section :buffer do
+      config_set_default :chunk_keys, ['time']
+      config_set_default :timekey, (60 * 60 * 24)
+    end
 
     MAX_HEX_RANDOM_LENGTH = 32
 
     def configure(conf)
+      compat_parameters_convert(conf, :buffer, :formatter, :inject)
       super
 
-      if @encryption_key && @encryption_key_sha256.nil?
-        raise Fluent::ConfigError, "encryption_key_sha256 parameter must be provided if `encryption_key` is provided."
-      end
-
       if @hex_random_length > MAX_HEX_RANDOM_LENGTH
         raise Fluent::ConfigError, "hex_random_length parameter should be set to #{MAX_HEX_RANDOM_LENGTH} characters or less."
       end
 
-      # The customer-supplied, AES-256 encryption key and hash used to encrypt the file.
+      # The customer-supplied, AES-256 encryption key that will be used to encrypt the file.
       @encryption_opts = {
         encryption_key: @encryption_key,
-        encryption_key_sha256: @encryption_key_sha256
       }
 
       if @object_metadata
         @object_metadata_hash = @object_metadata.map {|m| [m.key, m.value] }.to_h
       end
 
-      @formatter = Fluent::Plugin.new_formatter(@format)
-      @formatter.configure(conf)
+      @formatter = formatter_create
 
       @object_creator = Fluent::GCS.discovered_object_creator(@store_as, transcoding: @transcoding)
+      # For backward compatibility
+      # TODO: Remove time_slice_format when end of support compat_parameters
+      @configured_time_slice_format = conf['time_slice_format']
+      @time_slice_with_tz = Fluent::Timezone.formatter(@timekey_zone, @configured_time_slice_format || timekey_to_timeformat(@buffer_config['timekey']))
     end
 
     def start
@@ -96,7 +109,12 @@ module Fluent
     end
 
     def format(tag, time, record)
-      @formatter.format(tag, time, record)
+      r = inject_values_to_record(tag, time, record)
+      @formatter.format(tag, time, r)
+    end
+
+    def multi_workers_ready?
+      true
     end
 
     def write(chunk)
@@ -106,6 +124,7 @@ module Fluent
         opts = {
           metadata: @object_metadata_hash,
           acl: @acl,
+          storage_class: @storage_class,
           content_type: @object_creator.content_type,
           content_encoding: @object_creator.content_encoding,
         }
@@ -132,23 +151,33 @@ module Fluent
       Digest::MD5.hexdigest(chunk.unique_id)[0...@hex_random_length]
     end
 
-    def format_path(chunk)
-      now = Time.strptime(chunk.key, @time_slice_format)
-      (@localtime ? now : now.utc).strftime(@path)
+    def check_object_exists(path)
+      if !@blind_write
+        return @gcs_bucket.find_file(path, @encryption_opts)
+      else
+        return false
+      end
     end
 
     def generate_path(chunk, i = 0, prev = nil)
+      metadata = chunk.metadata
+      time_slice = if metadata.timekey.nil?
+                     ''.freeze
+                   else
+                     @time_slice_with_tz.call(metadata.timekey)
+                   end
       tags = {
         "%{file_extension}" => @object_creator.file_extension,
         "%{hex_random}" => hex_random(chunk),
         "%{hostname}" => Socket.gethostname,
         "%{index}" => i,
-        "%{path}" => format_path(chunk),
-        "%{time_slice}" => chunk.key,
+        "%{path}" => @path,
+        "%{time_slice}" => time_slice,
         "%{uuid_flush}" => SecureRandom.uuid,
       }
       path = @object_key_format.gsub(Regexp.union(tags.keys), tags)
-      return path unless @gcs_bucket.find_file(path, @encryption_opts)
+      path = extract_placeholders(path, chunk)
+      return path unless check_object_exists(path)
 
       if path == prev
         if @overwrite
@@ -159,5 +188,16 @@ module Fluent
       end
       generate_path(chunk, i + 1, path)
     end
+
+    # This is stolen from Fluentd
+    def timekey_to_timeformat(timekey)
+      case timekey
+      when nil          then ''
+      when 0...60       then '%Y%m%d%H%M%S' # 60 exclusive
+      when 60...3600    then '%Y%m%d%H%M'
+      when 3600...86400 then '%Y%m%d%H'
+      else                   '%Y%m%d'
+      end
+    end
   end
 end

metadata

@@ -1,71 +1,63 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-gcs
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.4.1
 platform: ruby
 authors:
 - Daichi HIRATA
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-11-28 00:00:00.000000000 Z
+date: 2020-10-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.22
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 0.12.0
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.22
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 0.12.0
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: google-cloud-storage
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.21'
+        version: '1.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.21'
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.13'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.13'
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
   name: rr
   requirement: !ruby/object:Gem::Requirement
@@ -131,6 +123,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".travis.yml"
+- CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -160,8 +153,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Google Cloud Storage output plugin for Fluentd