fluent-plugin-gcs 0.4.0.beta1 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: d4e0e0d0018992e06145a4982a26182240998f12
-  data.tar.gz: 5c96a75ae786b74be8f8092ed210ebae68be3ad6
+SHA256:
+  metadata.gz: e6bc239a1a47a29d4cc3dc94aaaafc5e3ed6ed20e23e0ea9807c88309ec1f7a0
+  data.tar.gz: c27f4d556b545ffb6050a72dcc9ff6a7fcfc53d16f44bce305f14560a4482725
 SHA512:
-  metadata.gz: a6d11e853d46efff711d2a0e2f2aeab878ad0923cd7230923bdf90da33ae98d11f7789a76c042faa01151473505e56a98d477904d246a876ca593582491e7f4a
-  data.tar.gz: 0fcffefaf44dc28f0d29d97c6f26e2cd6047ebaa8092eea65726c46637bb9073d31f1d0998ad94de502b1bdd8225f89429a89a9c93b6ed35338495474e0ab499
+  metadata.gz: 3eb03cfb810bf23d8447391145cd8ba6cf6379f1730e1c486a35bcafcb66fdf5b953741b5913b0255bc5969914fc1fda4e37110fe76eb7bf21f65d441466bedb
+  data.tar.gz: 85bcc85ab97e567a872893cfb890b7920343ff0010adf5d7c6d5b36f5ff952640bac4f42b90d811f1f1b6717c1691c40f6581d66a6da45135711a030dcf6b297

data/.github/workflows/test.yaml

@@ -0,0 +1,24 @@
+name: Test
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby-version: ['2.7', '3.0', '3.1']
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+        bundler-cache: true
+    - name: Run tests
+      run: bundle exec rake test

data/CHANGELOG.md

@@ -1,6 +1,19 @@
 ## [Unreleased]
 
-## [0.4.0] - xxxx/xx/xx
+New features / Enhancements
+
+## [0.4.2] - 2022/08/16
+
+Bug fixes
+
+- [Fix automatic conversion from a hash to keyword arguments](https://github.com/daichirata/fluent-plugin-gcs/pull/22)
+
+## [0.4.1] - 2020/04/17
+
+New features
+- [Support blind write to GSC](https://github.com/daichirata/fluent-plugin-gcs/pull/14)
+
+## [0.4.0] - 2019/04/01
 
 New features / Enhancements
 
@@ -26,7 +39,8 @@ New features / Enhancements
 
 - Add support for `%{hostname}` of object_key_format
 
-[Unreleased]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.3.0...HEAD
+[Unreleased]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.4.0...HEAD
+[0.4.0]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.3.0...v0.4.0
 [0.3.0]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.2.0...v0.3.0
-[0.2.0]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.1.a...v0.2.0
+[0.2.0]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.1.0...v0.2.0
 [0.1.1]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.1.0...v0.1.1

data/Gemfile

@@ -1,4 +1,13 @@
-source 'https://rubygems.org'
+# frozen_string_literal: true
+
+source "https://rubygems.org"
 
 # Specify your gem's dependencies in fluent-plugin-gcs.gemspec
 gemspec
+
+gem "rake", "~> 13.0"
+gem "rr", "= 1.1.2"
+gem "test-unit", ">= 3.0.8"
+gem "test-unit-rr", ">= 1.0.3"
+gem "timecop"
+gem "solargraph"

data/README.md

@@ -1,5 +1,5 @@
 # fluent-plugin-gcs
-[![Gem Version](https://badge.fury.io/rb/fluent-plugin-gcs.svg)](https://badge.fury.io/rb/fluent-plugin-gcs) [![Build Status](https://travis-ci.org/daichirata/fluent-plugin-gcs.svg?branch=master)](https://travis-ci.org/daichirata/fluent-plugin-gcs) [![Code Climate](https://codeclimate.com/github/daichirata/fluent-plugin-gcs/badges/gpa.svg)](https://codeclimate.com/github/daichirata/fluent-plugin-gcs)
+[![Gem Version](https://badge.fury.io/rb/fluent-plugin-gcs.svg)](https://badge.fury.io/rb/fluent-plugin-gcs) [![Test](https://github.com/daichirata/fluent-plugin-gcs/actions/workflows/test.yaml/badge.svg)](https://github.com/daichirata/fluent-plugin-gcs/actions/workflows/test.yaml) [![Code Climate](https://codeclimate.com/github/daichirata/fluent-plugin-gcs/badges/gpa.svg)](https://codeclimate.com/github/daichirata/fluent-plugin-gcs)
 
 Google Cloud Storage output plugin for [Fluentd](https://github.com/fluent/fluentd).
 
@@ -7,16 +7,14 @@ Google Cloud Storage output plugin for [Fluentd](https://github.com/fluent/fluen
 
 | fluent-plugin-gcs  | fluentd    | ruby   |
 |--------------------|------------|--------|
-| >= 0.4.0           | >= v0.14.0 | >= 2.1 |
+| >= 0.4.0           | >= v0.14.0 | >= 2.4 |
 |  < 0.4.0           | >= v0.12.0 | >= 1.9 |
 
-NOTE: fluent-plugin-gcs v0.4.0 is now beta.
-
 ## Installation
 
 ``` shell
-$ gem install fluent-plugin-gcs -v "~> 0.3"  --no-document # for fluentd v0.12 or later
-$ gem install fluent-plugin-gcs -v "0.4.0.beta1" --no-document # for fluentd v0.14 or later
+$ gem install fluent-plugin-gcs -v "~> 0.3" --no-document # for fluentd v0.12 or later
+$ gem install fluent-plugin-gcs -v "0.4.0" --no-document # for fluentd v0.14 or later
 ```
 
 ## Examples
@@ -227,9 +225,18 @@ Use Local time instead of UTC.
 
 Use UTC instead of local time.
 
-
 And see [official Time Sliced Output article](http://docs.fluentd.org/articles/output-plugin-overview#time-sliced-output-parameters)
 
+**blind_write**
+
+Doesn't check if an object exists in GCS before writing. Default is false.
+
+Allows to avoid granting of `storage.objects.get` permission.
+
+Warning! If the object exists and `storage.objects.delete` permission is not
+granted, it will result in an unrecoverable error. Usage of `%{hex_random}` is
+recommended.
+
 ### ObjectMetadata
 
 User provided web-safe keys and arbitrary string values that will returned with requests for the file as "x-goog-meta-" response headers.

data/fluent-plugin-gcs.gemspec

@@ -20,13 +20,6 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "fluentd", [">= 0.14.0", "< 2"]
-  spec.add_runtime_dependency "google-cloud-storage", "~> 1.1.0"
-
-  spec.add_development_dependency "bundler", "~> 1.13"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "rr", "= 1.1.2"
-  spec.add_development_dependency "test-unit", ">= 3.0.8"
-  spec.add_development_dependency "test-unit-rr", ">= 1.0.3"
-  spec.add_development_dependency "timecop"
+  spec.add_runtime_dependency "fluentd", [">= 0.14.22", "< 2"]
+  spec.add_runtime_dependency "google-cloud-storage", "~> 1.1"
 end

data/lib/fluent/plugin/gcs/version.rb

@@ -1,5 +1,5 @@
 module Fluent
   module GCSPlugin
-    VERSION = "0.4.0.beta1"
+    VERSION = "0.4.2"
   end
 end

data/lib/fluent/plugin/out_gcs.rb

@@ -49,6 +49,8 @@ module Fluent::Plugin
                  desc: "Storage class of the file"
     config_param :encryption_key, :string, default: nil, secret: true,
                  desc: "Customer-supplied, AES-256 encryption key"
+    config_param :blind_write, :bool, default: false,
+                 desc: "Whether to check if object already exists by given GCS path. Allows avoiding giving storage.object.get permission"
     config_section :object_metadata, required: false do
       config_param :key, :string, default: ""
       config_param :value, :string, default: ""
@@ -129,7 +131,7 @@ module Fluent::Plugin
         opts.merge!(@encryption_opts)
 
         log.debug { "out_gcs: upload chunk:#{chunk.key} to gcs://#{@bucket}/#{path} options: #{opts}" }
-        @gcs_bucket.upload_file(obj.path, path, opts)
+        @gcs_bucket.upload_file(obj.path, path, **opts)
       end
     end
 
@@ -149,6 +151,14 @@ module Fluent::Plugin
       Digest::MD5.hexdigest(chunk.unique_id)[0...@hex_random_length]
     end
 
+    def check_object_exists(path)
+      if !@blind_write
+        return @gcs_bucket.find_file(path, **@encryption_opts)
+      else
+        return false
+      end
+    end
+
     def generate_path(chunk, i = 0, prev = nil)
       metadata = chunk.metadata
       time_slice = if metadata.timekey.nil?
@@ -166,8 +176,8 @@ module Fluent::Plugin
         "%{uuid_flush}" => SecureRandom.uuid,
       }
       path = @object_key_format.gsub(Regexp.union(tags.keys), tags)
-      path = extract_placeholders(path, metadata)
-      return path unless @gcs_bucket.find_file(path, @encryption_opts)
+      path = extract_placeholders(path, chunk)
+      return path unless check_object_exists(path)
 
       if path == prev
         if @overwrite

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-gcs
 version: !ruby/object:Gem::Version
-  version: 0.4.0.beta1
+  version: 0.4.2
 platform: ruby
 authors:
 - Daichi HIRATA
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2017-06-24 00:00:00.000000000 Z
+date: 2022-08-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,7 +16,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.0
+        version: 0.14.22
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.0
+        version: 0.14.22
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -36,98 +36,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.1.0
+        version: '1.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.1.0
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.13'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.13'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '10.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '10.0'
-- !ruby/object:Gem::Dependency
-  name: rr
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 1.1.2
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 1.1.2
-- !ruby/object:Gem::Dependency
-  name: test-unit
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 3.0.8
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 3.0.8
-- !ruby/object:Gem::Dependency
-  name: test-unit-rr
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.0.3
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.0.3
-- !ruby/object:Gem::Dependency
-  name: timecop
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.1'
 description: Google Cloud Storage output plugin for Fluentd
 email:
 - hirata.daichi@gmail.com
@@ -135,8 +51,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/test.yaml"
 - ".gitignore"
-- ".travis.yml"
 - CHANGELOG.md
 - Gemfile
 - LICENSE.txt
@@ -152,7 +68,7 @@ homepage: https://github.com/daichirata/fluent-plugin-gcs
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -163,13 +79,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.1
-signing_key: 
+rubygems_version: 3.3.7
+signing_key:
 specification_version: 4
 summary: Google Cloud Storage output plugin for Fluentd
 test_files: []

data/.travis.yml

@@ -1,11 +0,0 @@
-language: ruby
-
-rvm:
-  - 2.1
-  - 2.2
-  - 2.3.0
-
-before_install:
-  - gem update bundler
-
-script: bundle exec rake test