fluent-plugin-gcs 0.4.2 → 0.4.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/test.yaml +2 -2
- data/.gitignore +1 -0
- data/lib/fluent/plugin/gcs/version.rb +1 -1
- data/lib/fluent/plugin/out_gcs.rb +34 -14
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 42a11febaed3fc628877f4825ccf37be26bb72aea69fb403d5a74099353d6af6
|
4
|
+
data.tar.gz: 1a4e644a95db9f2debf96593914f64bdcef2eb1eddf80c983ee5800616cedc68
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e209ea956498fd8773fe6cf38723cc3faa0336fdfa12a0eef58a81b8447c50027582748d71ed92b4039e8cb4ea77ffad722ad05473c98dcf9442bc870d80735b
|
7
|
+
data.tar.gz: 717e1a3e64cb0c8ba2f1f4fe5eb2f947399558ef7ad85094c39f083840dcf46c88260124f2c9b3463e457e7e705d52348d9f1b7d09a1b1b370ac6a4e5942c16b
|
data/.github/workflows/test.yaml
CHANGED
data/.gitignore
CHANGED
@@ -15,12 +15,15 @@ module Fluent::Plugin
|
|
15
15
|
def initialize
|
16
16
|
super
|
17
17
|
require "google/cloud/storage"
|
18
|
+
Google::Apis.logger = log
|
18
19
|
end
|
19
20
|
|
20
21
|
config_param :project, :string, default: nil,
|
21
22
|
desc: "Project identifier for GCS"
|
22
23
|
config_param :keyfile, :string, default: nil,
|
23
24
|
desc: "Path of GCS service account credentials JSON file"
|
25
|
+
config_param :credentials_json, :hash, default: nil, secret: true,
|
26
|
+
desc: "GCS service account credentials in JSON format"
|
24
27
|
config_param :client_retries, :integer, default: nil,
|
25
28
|
desc: "Number of times to retry requests on server error"
|
26
29
|
config_param :client_timeout, :integer, default: nil,
|
@@ -93,12 +96,18 @@ module Fluent::Plugin
|
|
93
96
|
# TODO: Remove time_slice_format when end of support compat_parameters
|
94
97
|
@configured_time_slice_format = conf['time_slice_format']
|
95
98
|
@time_slice_with_tz = Fluent::Timezone.formatter(@timekey_zone, @configured_time_slice_format || timekey_to_timeformat(@buffer_config['timekey']))
|
99
|
+
|
100
|
+
if @credentials_json
|
101
|
+
@credentials = @credentials_json
|
102
|
+
else
|
103
|
+
@credentials = keyfile
|
104
|
+
end
|
96
105
|
end
|
97
106
|
|
98
107
|
def start
|
99
108
|
@gcs = Google::Cloud::Storage.new(
|
100
109
|
project: @project,
|
101
|
-
keyfile: @
|
110
|
+
keyfile: @credentials,
|
102
111
|
retries: @client_retries,
|
103
112
|
timeout: @client_timeout
|
104
113
|
)
|
@@ -159,7 +168,7 @@ module Fluent::Plugin
|
|
159
168
|
end
|
160
169
|
end
|
161
170
|
|
162
|
-
def generate_path(chunk
|
171
|
+
def generate_path(chunk)
|
163
172
|
metadata = chunk.metadata
|
164
173
|
time_slice = if metadata.timekey.nil?
|
165
174
|
''.freeze
|
@@ -170,23 +179,34 @@ module Fluent::Plugin
|
|
170
179
|
"%{file_extension}" => @object_creator.file_extension,
|
171
180
|
"%{hex_random}" => hex_random(chunk),
|
172
181
|
"%{hostname}" => Socket.gethostname,
|
173
|
-
"%{index}" => i,
|
174
182
|
"%{path}" => @path,
|
175
183
|
"%{time_slice}" => time_slice,
|
176
|
-
"%{uuid_flush}" => SecureRandom.uuid,
|
177
184
|
}
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
185
|
+
|
186
|
+
prev = nil
|
187
|
+
i = 0
|
188
|
+
|
189
|
+
until i < 0 do # Until overflow
|
190
|
+
tags["%{uuid_flush}"] = SecureRandom.uuid
|
191
|
+
tags["%{index}"] = i
|
192
|
+
|
193
|
+
path = @object_key_format.gsub(Regexp.union(tags.keys), tags)
|
194
|
+
path = extract_placeholders(path, chunk)
|
195
|
+
return path unless check_object_exists(path)
|
196
|
+
|
197
|
+
if path == prev
|
198
|
+
if @overwrite
|
199
|
+
log.warn "object `#{path}` already exists but overwrites it"
|
200
|
+
return path
|
201
|
+
end
|
202
|
+
raise "object `#{path}` already exists"
|
186
203
|
end
|
187
|
-
|
204
|
+
|
205
|
+
i += 1
|
206
|
+
prev = path
|
188
207
|
end
|
189
|
-
|
208
|
+
|
209
|
+
raise "cannot find an unoccupied GCS path"
|
190
210
|
end
|
191
211
|
|
192
212
|
# This is stolen from Fluentd
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-gcs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daichi HIRATA
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2024-02-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|