fluent-plugin-gcs 0.4.2 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: e6bc239a1a47a29d4cc3dc94aaaafc5e3ed6ed20e23e0ea9807c88309ec1f7a0
4
- data.tar.gz: c27f4d556b545ffb6050a72dcc9ff6a7fcfc53d16f44bce305f14560a4482725
3
+ metadata.gz: 42a11febaed3fc628877f4825ccf37be26bb72aea69fb403d5a74099353d6af6
4
+ data.tar.gz: 1a4e644a95db9f2debf96593914f64bdcef2eb1eddf80c983ee5800616cedc68
5
5
  SHA512:
6
- metadata.gz: 3eb03cfb810bf23d8447391145cd8ba6cf6379f1730e1c486a35bcafcb66fdf5b953741b5913b0255bc5969914fc1fda4e37110fe76eb7bf21f65d441466bedb
7
- data.tar.gz: 85bcc85ab97e567a872893cfb890b7920343ff0010adf5d7c6d5b36f5ff952640bac4f42b90d811f1f1b6717c1691c40f6581d66a6da45135711a030dcf6b297
6
+ metadata.gz: e209ea956498fd8773fe6cf38723cc3faa0336fdfa12a0eef58a81b8447c50027582748d71ed92b4039e8cb4ea77ffad722ad05473c98dcf9442bc870d80735b
7
+ data.tar.gz: 717e1a3e64cb0c8ba2f1f4fe5eb2f947399558ef7ad85094c39f083840dcf46c88260124f2c9b3463e457e7e705d52348d9f1b7d09a1b1b370ac6a4e5942c16b
@@ -12,9 +12,9 @@ jobs:
12
12
  strategy:
13
13
  fail-fast: false
14
14
  matrix:
15
- ruby-version: ['2.7', '3.0', '3.1']
15
+ ruby-version: ['3.1', '3.2']
16
16
  steps:
17
- - uses: actions/checkout@v3
17
+ - uses: actions/checkout@v4
18
18
  - name: Set up Ruby
19
19
  uses: ruby/setup-ruby@v1
20
20
  with:
data/.gitignore CHANGED
@@ -7,3 +7,4 @@
7
7
  /pkg/
8
8
  /spec/reports/
9
9
  /tmp/
10
+ /vendor/
@@ -1,5 +1,5 @@
1
1
  module Fluent
2
2
  module GCSPlugin
3
- VERSION = "0.4.2"
3
+ VERSION = "0.4.4"
4
4
  end
5
5
  end
@@ -15,12 +15,15 @@ module Fluent::Plugin
15
15
  def initialize
16
16
  super
17
17
  require "google/cloud/storage"
18
+ Google::Apis.logger = log
18
19
  end
19
20
 
20
21
  config_param :project, :string, default: nil,
21
22
  desc: "Project identifier for GCS"
22
23
  config_param :keyfile, :string, default: nil,
23
24
  desc: "Path of GCS service account credentials JSON file"
25
+ config_param :credentials_json, :hash, default: nil, secret: true,
26
+ desc: "GCS service account credentials in JSON format"
24
27
  config_param :client_retries, :integer, default: nil,
25
28
  desc: "Number of times to retry requests on server error"
26
29
  config_param :client_timeout, :integer, default: nil,
@@ -93,12 +96,18 @@ module Fluent::Plugin
93
96
  # TODO: Remove time_slice_format when end of support compat_parameters
94
97
  @configured_time_slice_format = conf['time_slice_format']
95
98
  @time_slice_with_tz = Fluent::Timezone.formatter(@timekey_zone, @configured_time_slice_format || timekey_to_timeformat(@buffer_config['timekey']))
99
+
100
+ if @credentials_json
101
+ @credentials = @credentials_json
102
+ else
103
+ @credentials = keyfile
104
+ end
96
105
  end
97
106
 
98
107
  def start
99
108
  @gcs = Google::Cloud::Storage.new(
100
109
  project: @project,
101
- keyfile: @keyfile,
110
+ keyfile: @credentials,
102
111
  retries: @client_retries,
103
112
  timeout: @client_timeout
104
113
  )
@@ -159,7 +168,7 @@ module Fluent::Plugin
159
168
  end
160
169
  end
161
170
 
162
- def generate_path(chunk, i = 0, prev = nil)
171
+ def generate_path(chunk)
163
172
  metadata = chunk.metadata
164
173
  time_slice = if metadata.timekey.nil?
165
174
  ''.freeze
@@ -170,23 +179,34 @@ module Fluent::Plugin
170
179
  "%{file_extension}" => @object_creator.file_extension,
171
180
  "%{hex_random}" => hex_random(chunk),
172
181
  "%{hostname}" => Socket.gethostname,
173
- "%{index}" => i,
174
182
  "%{path}" => @path,
175
183
  "%{time_slice}" => time_slice,
176
- "%{uuid_flush}" => SecureRandom.uuid,
177
184
  }
178
- path = @object_key_format.gsub(Regexp.union(tags.keys), tags)
179
- path = extract_placeholders(path, chunk)
180
- return path unless check_object_exists(path)
181
-
182
- if path == prev
183
- if @overwrite
184
- log.warn "object `#{path}` already exists but overwrites it"
185
- return path
185
+
186
+ prev = nil
187
+ i = 0
188
+
189
+ until i < 0 do # Until overflow
190
+ tags["%{uuid_flush}"] = SecureRandom.uuid
191
+ tags["%{index}"] = i
192
+
193
+ path = @object_key_format.gsub(Regexp.union(tags.keys), tags)
194
+ path = extract_placeholders(path, chunk)
195
+ return path unless check_object_exists(path)
196
+
197
+ if path == prev
198
+ if @overwrite
199
+ log.warn "object `#{path}` already exists but overwrites it"
200
+ return path
201
+ end
202
+ raise "object `#{path}` already exists"
186
203
  end
187
- raise "object `#{path}` already exists"
204
+
205
+ i += 1
206
+ prev = path
188
207
  end
189
- generate_path(chunk, i + 1, path)
208
+
209
+ raise "cannot find an unoccupied GCS path"
190
210
  end
191
211
 
192
212
  # This is stolen from Fluentd
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-gcs
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.2
4
+ version: 0.4.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Daichi HIRATA
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2022-08-16 00:00:00.000000000 Z
11
+ date: 2024-02-03 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd