fluent-plugin-gcs 0.4.2 → 0.4.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e6bc239a1a47a29d4cc3dc94aaaafc5e3ed6ed20e23e0ea9807c88309ec1f7a0
-  data.tar.gz: c27f4d556b545ffb6050a72dcc9ff6a7fcfc53d16f44bce305f14560a4482725
+  metadata.gz: 42a11febaed3fc628877f4825ccf37be26bb72aea69fb403d5a74099353d6af6
+  data.tar.gz: 1a4e644a95db9f2debf96593914f64bdcef2eb1eddf80c983ee5800616cedc68
 SHA512:
-  metadata.gz: 3eb03cfb810bf23d8447391145cd8ba6cf6379f1730e1c486a35bcafcb66fdf5b953741b5913b0255bc5969914fc1fda4e37110fe76eb7bf21f65d441466bedb
-  data.tar.gz: 85bcc85ab97e567a872893cfb890b7920343ff0010adf5d7c6d5b36f5ff952640bac4f42b90d811f1f1b6717c1691c40f6581d66a6da45135711a030dcf6b297
+  metadata.gz: e209ea956498fd8773fe6cf38723cc3faa0336fdfa12a0eef58a81b8447c50027582748d71ed92b4039e8cb4ea77ffad722ad05473c98dcf9442bc870d80735b
+  data.tar.gz: 717e1a3e64cb0c8ba2f1f4fe5eb2f947399558ef7ad85094c39f083840dcf46c88260124f2c9b3463e457e7e705d52348d9f1b7d09a1b1b370ac6a4e5942c16b

data/.github/workflows/test.yaml

@@ -12,9 +12,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby-version: ['2.7', '3.0', '3.1']
+        ruby-version: ['3.1', '3.2']
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Ruby
       uses: ruby/setup-ruby@v1
       with:

data/.gitignore

@@ -7,3 +7,4 @@
 /pkg/
 /spec/reports/
 /tmp/
+/vendor/

data/lib/fluent/plugin/gcs/version.rb

@@ -1,5 +1,5 @@
 module Fluent
   module GCSPlugin
-    VERSION = "0.4.2"
+    VERSION = "0.4.4"
   end
 end

data/lib/fluent/plugin/out_gcs.rb

@@ -15,12 +15,15 @@ module Fluent::Plugin
     def initialize
       super
       require "google/cloud/storage"
+      Google::Apis.logger = log
     end
 
     config_param :project, :string,  default: nil,
                  desc: "Project identifier for GCS"
     config_param :keyfile, :string, default: nil,
                  desc: "Path of GCS service account credentials JSON file"
+    config_param :credentials_json, :hash, default: nil, secret: true,
+                 desc: "GCS service account credentials in JSON format"
     config_param :client_retries, :integer, default: nil,
                  desc: "Number of times to retry requests on server error"
     config_param :client_timeout, :integer, default: nil,
@@ -93,12 +96,18 @@ module Fluent::Plugin
       # TODO: Remove time_slice_format when end of support compat_parameters
       @configured_time_slice_format = conf['time_slice_format']
       @time_slice_with_tz = Fluent::Timezone.formatter(@timekey_zone, @configured_time_slice_format || timekey_to_timeformat(@buffer_config['timekey']))
+
+      if @credentials_json
+        @credentials = @credentials_json
+      else
+        @credentials = keyfile
+      end
     end
 
     def start
       @gcs = Google::Cloud::Storage.new(
         project: @project,
-        keyfile: @keyfile,
+        keyfile: @credentials,
         retries: @client_retries,
         timeout: @client_timeout
       )
@@ -159,7 +168,7 @@ module Fluent::Plugin
       end
     end
 
-    def generate_path(chunk, i = 0, prev = nil)
+    def generate_path(chunk)
       metadata = chunk.metadata
       time_slice = if metadata.timekey.nil?
                      ''.freeze
@@ -170,23 +179,34 @@ module Fluent::Plugin
         "%{file_extension}" => @object_creator.file_extension,
         "%{hex_random}" => hex_random(chunk),
         "%{hostname}" => Socket.gethostname,
-        "%{index}" => i,
         "%{path}" => @path,
         "%{time_slice}" => time_slice,
-        "%{uuid_flush}" => SecureRandom.uuid,
       }
-      path = @object_key_format.gsub(Regexp.union(tags.keys), tags)
-      path = extract_placeholders(path, chunk)
-      return path unless check_object_exists(path)
-
-      if path == prev
-        if @overwrite
-          log.warn "object `#{path}` already exists but overwrites it"
-          return path
+
+      prev = nil
+      i = 0
+
+      until i < 0 do # Until overflow
+        tags["%{uuid_flush}"] = SecureRandom.uuid
+        tags["%{index}"] = i
+
+        path = @object_key_format.gsub(Regexp.union(tags.keys), tags)
+        path = extract_placeholders(path, chunk)
+        return path unless check_object_exists(path)
+
+        if path == prev
+          if @overwrite
+            log.warn "object `#{path}` already exists but overwrites it"
+            return path
+          end
+          raise "object `#{path}` already exists"
         end
-        raise "object `#{path}` already exists"
+
+        i += 1
+        prev = path
       end
-      generate_path(chunk, i + 1, path)
+
+      raise "cannot find an unoccupied GCS path"
     end
 
     # This is stolen from Fluentd

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-gcs
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.4.4
 platform: ruby
 authors:
 - Daichi HIRATA
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-08-16 00:00:00.000000000 Z
+date: 2024-02-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd