fluent-plugin-gcs 0.4.2 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/test.yaml +2 -2
- data/.gitignore +1 -0
- data/lib/fluent/plugin/gcs/version.rb +1 -1
- data/lib/fluent/plugin/out_gcs.rb +34 -14
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 42a11febaed3fc628877f4825ccf37be26bb72aea69fb403d5a74099353d6af6
|
4
|
+
data.tar.gz: 1a4e644a95db9f2debf96593914f64bdcef2eb1eddf80c983ee5800616cedc68
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e209ea956498fd8773fe6cf38723cc3faa0336fdfa12a0eef58a81b8447c50027582748d71ed92b4039e8cb4ea77ffad722ad05473c98dcf9442bc870d80735b
|
7
|
+
data.tar.gz: 717e1a3e64cb0c8ba2f1f4fe5eb2f947399558ef7ad85094c39f083840dcf46c88260124f2c9b3463e457e7e705d52348d9f1b7d09a1b1b370ac6a4e5942c16b
|
data/.github/workflows/test.yaml
CHANGED
data/.gitignore
CHANGED
@@ -15,12 +15,15 @@ module Fluent::Plugin
|
|
15
15
|
def initialize
|
16
16
|
super
|
17
17
|
require "google/cloud/storage"
|
18
|
+
Google::Apis.logger = log
|
18
19
|
end
|
19
20
|
|
20
21
|
config_param :project, :string, default: nil,
|
21
22
|
desc: "Project identifier for GCS"
|
22
23
|
config_param :keyfile, :string, default: nil,
|
23
24
|
desc: "Path of GCS service account credentials JSON file"
|
25
|
+
config_param :credentials_json, :hash, default: nil, secret: true,
|
26
|
+
desc: "GCS service account credentials in JSON format"
|
24
27
|
config_param :client_retries, :integer, default: nil,
|
25
28
|
desc: "Number of times to retry requests on server error"
|
26
29
|
config_param :client_timeout, :integer, default: nil,
|
@@ -93,12 +96,18 @@ module Fluent::Plugin
|
|
93
96
|
# TODO: Remove time_slice_format when end of support compat_parameters
|
94
97
|
@configured_time_slice_format = conf['time_slice_format']
|
95
98
|
@time_slice_with_tz = Fluent::Timezone.formatter(@timekey_zone, @configured_time_slice_format || timekey_to_timeformat(@buffer_config['timekey']))
|
99
|
+
|
100
|
+
if @credentials_json
|
101
|
+
@credentials = @credentials_json
|
102
|
+
else
|
103
|
+
@credentials = keyfile
|
104
|
+
end
|
96
105
|
end
|
97
106
|
|
98
107
|
def start
|
99
108
|
@gcs = Google::Cloud::Storage.new(
|
100
109
|
project: @project,
|
101
|
-
keyfile: @
|
110
|
+
keyfile: @credentials,
|
102
111
|
retries: @client_retries,
|
103
112
|
timeout: @client_timeout
|
104
113
|
)
|
@@ -159,7 +168,7 @@ module Fluent::Plugin
|
|
159
168
|
end
|
160
169
|
end
|
161
170
|
|
162
|
-
def generate_path(chunk
|
171
|
+
def generate_path(chunk)
|
163
172
|
metadata = chunk.metadata
|
164
173
|
time_slice = if metadata.timekey.nil?
|
165
174
|
''.freeze
|
@@ -170,23 +179,34 @@ module Fluent::Plugin
|
|
170
179
|
"%{file_extension}" => @object_creator.file_extension,
|
171
180
|
"%{hex_random}" => hex_random(chunk),
|
172
181
|
"%{hostname}" => Socket.gethostname,
|
173
|
-
"%{index}" => i,
|
174
182
|
"%{path}" => @path,
|
175
183
|
"%{time_slice}" => time_slice,
|
176
|
-
"%{uuid_flush}" => SecureRandom.uuid,
|
177
184
|
}
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
185
|
+
|
186
|
+
prev = nil
|
187
|
+
i = 0
|
188
|
+
|
189
|
+
until i < 0 do # Until overflow
|
190
|
+
tags["%{uuid_flush}"] = SecureRandom.uuid
|
191
|
+
tags["%{index}"] = i
|
192
|
+
|
193
|
+
path = @object_key_format.gsub(Regexp.union(tags.keys), tags)
|
194
|
+
path = extract_placeholders(path, chunk)
|
195
|
+
return path unless check_object_exists(path)
|
196
|
+
|
197
|
+
if path == prev
|
198
|
+
if @overwrite
|
199
|
+
log.warn "object `#{path}` already exists but overwrites it"
|
200
|
+
return path
|
201
|
+
end
|
202
|
+
raise "object `#{path}` already exists"
|
186
203
|
end
|
187
|
-
|
204
|
+
|
205
|
+
i += 1
|
206
|
+
prev = path
|
188
207
|
end
|
189
|
-
|
208
|
+
|
209
|
+
raise "cannot find an unoccupied GCS path"
|
190
210
|
end
|
191
211
|
|
192
212
|
# This is stolen from Fluentd
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-gcs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daichi HIRATA
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2024-02-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|