fluent-plugin-gcs 0.1.1 → 0.4.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.travis.yml +3 -3
- data/CHANGELOG.md +40 -0
- data/README.md +65 -5
- data/fluent-plugin-gcs.gemspec +3 -4
- data/lib/fluent/plugin/gcs/version.rb +1 -1
- data/lib/fluent/plugin/out_gcs.rb +63 -23
- metadata +20 -28
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 85b1b9b0ed8cd2b264dacc7f54fb1bf9e15b2f1e6f28871b17e0bb7e844e1669
|
4
|
+
data.tar.gz: a90fec1f3d5509be2df1a72d2479e9db6ba7f133ad67c60789abbdbb1360e519
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8bcb9225215a56ea5ad0d34ac2447e0b39074ae9269e885c2ac70ba2679213340653b47941ea2c5c395861535fd565ae81a6e153e48adef175cf6966b1849a20
|
7
|
+
data.tar.gz: dfe13e0a8ea31f3d80d1174aa4c35385cf6206820d5574f31d501767e972b144f646a48711e64c63b10261045385aed4af9a2e7516aeeac0f6aa460cb37cf807
|
data/.travis.yml
CHANGED
data/CHANGELOG.md
ADDED
@@ -0,0 +1,40 @@
|
|
1
|
+
## [Unreleased]
|
2
|
+
|
3
|
+
New features / Enhancements
|
4
|
+
|
5
|
+
## [0.4.1] - 2020/04/17
|
6
|
+
|
7
|
+
New features
|
8
|
+
- [Support blind write to GSC](https://github.com/daichirata/fluent-plugin-gcs/pull/14)
|
9
|
+
|
10
|
+
## [0.4.0] - 2019/04/01
|
11
|
+
|
12
|
+
New features / Enhancements
|
13
|
+
|
14
|
+
- [Support v0.14 (by @cosmo0920)](https://github.com/daichirata/fluent-plugin-gcs/pull/6)
|
15
|
+
|
16
|
+
## [0.3.0] - 2017/02/28
|
17
|
+
|
18
|
+
New features / Enhancements
|
19
|
+
|
20
|
+
- [Add support for setting a File's storage_class on file creation](https://github.com/daichirata/fluent-plugin-gcs/pull/4)
|
21
|
+
- see also https://cloud.google.com/storage/docs/storage-classes
|
22
|
+
|
23
|
+
## [0.2.0] - 2017/01/16
|
24
|
+
|
25
|
+
Bug fixes
|
26
|
+
|
27
|
+
- [Remove encryption_key_sha256 parameter.](https://github.com/daichirata/fluent-plugin-gcs/pull/2)
|
28
|
+
- see also. https://github.com/GoogleCloudPlatform/google-cloud-ruby/blob/master/google-cloud-storage/CHANGELOG.md#0230--2016-12-8
|
29
|
+
|
30
|
+
## [0.1.1] - 2016/11/28
|
31
|
+
|
32
|
+
New features / Enhancements
|
33
|
+
|
34
|
+
- Add support for `%{hostname}` of object_key_format
|
35
|
+
|
36
|
+
[Unreleased]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.4.0...HEAD
|
37
|
+
[0.4.0]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.3.0...v0.4.0
|
38
|
+
[0.3.0]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.2.0...v0.3.0
|
39
|
+
[0.2.0]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.1.0...v0.2.0
|
40
|
+
[0.1.1]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.1.0...v0.1.1
|
data/README.md
CHANGED
@@ -3,14 +3,52 @@
|
|
3
3
|
|
4
4
|
Google Cloud Storage output plugin for [Fluentd](https://github.com/fluent/fluentd).
|
5
5
|
|
6
|
+
## Requirements
|
7
|
+
|
8
|
+
| fluent-plugin-gcs | fluentd | ruby |
|
9
|
+
|--------------------|------------|--------|
|
10
|
+
| >= 0.4.0 | >= v0.14.0 | >= 2.4 |
|
11
|
+
| < 0.4.0 | >= v0.12.0 | >= 1.9 |
|
12
|
+
|
6
13
|
## Installation
|
7
14
|
|
8
15
|
``` shell
|
9
|
-
gem install fluent-plugin-gcs
|
16
|
+
$ gem install fluent-plugin-gcs -v "~> 0.3" --no-document # for fluentd v0.12 or later
|
17
|
+
$ gem install fluent-plugin-gcs -v "0.4.0" --no-document # for fluentd v0.14 or later
|
10
18
|
```
|
11
19
|
|
12
20
|
## Examples
|
13
21
|
|
22
|
+
### For v0.14 style
|
23
|
+
|
24
|
+
```
|
25
|
+
<match pattern>
|
26
|
+
@type gcs
|
27
|
+
|
28
|
+
project YOUR_PROJECT
|
29
|
+
keyfile YOUR_KEYFILE_PATH
|
30
|
+
bucket YOUR_GCS_BUCKET_NAME
|
31
|
+
object_key_format %{path}%{time_slice}_%{index}.%{file_extension}
|
32
|
+
path logs/${tag}/%Y/%m/%d/
|
33
|
+
|
34
|
+
# if you want to use ${tag} or %Y/%m/%d/ like syntax in path / object_key_format,
|
35
|
+
# need to specify tag for ${tag} and time for %Y/%m/%d in <buffer> argument.
|
36
|
+
<buffer tag,time>
|
37
|
+
@type file
|
38
|
+
path /var/log/fluent/gcs
|
39
|
+
timekey 1h # 1 hour partition
|
40
|
+
timekey_wait 10m
|
41
|
+
timekey_use_utc true # use utc
|
42
|
+
</buffer>
|
43
|
+
|
44
|
+
<format>
|
45
|
+
@type json
|
46
|
+
</format>
|
47
|
+
</match>
|
48
|
+
```
|
49
|
+
|
50
|
+
### For v0.12 style
|
51
|
+
|
14
52
|
```
|
15
53
|
<match pattern>
|
16
54
|
@type gcs
|
@@ -127,8 +165,6 @@ See also [official Formatter article](http://docs.fluentd.org/articles/formatter
|
|
127
165
|
|
128
166
|
Create GCS bucket if it does not exists. Default is true.
|
129
167
|
|
130
|
-
TODO: rate limit
|
131
|
-
|
132
168
|
**acl**
|
133
169
|
|
134
170
|
Permission for the object in GCS. Acceptable values are:
|
@@ -142,10 +178,25 @@ Permission for the object in GCS. Acceptable values are:
|
|
142
178
|
|
143
179
|
Default is nil (bucket default object ACL). See also [official document](https://cloud.google.com/storage/docs/access-control/lists).
|
144
180
|
|
145
|
-
**
|
181
|
+
**storage_class**
|
182
|
+
|
183
|
+
Storage class of the file. Acceptable values are:
|
184
|
+
|
185
|
+
* `dra` - Durable Reduced Availability
|
186
|
+
* `nearline` - Nearline Storage
|
187
|
+
* `coldline` - Coldline Storage
|
188
|
+
* `multi_regional` - Multi-Regional Storage
|
189
|
+
* `regional` - Regional Storage
|
190
|
+
* `standard` - Standard Storage
|
191
|
+
|
192
|
+
Default is nil. See also [official document](https://cloud.google.com/storage/docs/storage-classes).
|
193
|
+
|
194
|
+
**encryption_key**
|
146
195
|
|
147
196
|
You can also choose to provide your own AES-256 key for server-side encryption. See also [Customer-supplied encryption keys](https://cloud.google.com/storage/docs/encryption#customer-supplied).
|
148
197
|
|
198
|
+
`encryption_key_sha256` will be calculated using encryption_key.
|
199
|
+
|
149
200
|
**overwrite**
|
150
201
|
|
151
202
|
Overwrite already existing path. Default is false, which raises an error
|
@@ -174,9 +225,18 @@ Use Local time instead of UTC.
|
|
174
225
|
|
175
226
|
Use UTC instead of local time.
|
176
227
|
|
177
|
-
|
178
228
|
And see [official Time Sliced Output article](http://docs.fluentd.org/articles/output-plugin-overview#time-sliced-output-parameters)
|
179
229
|
|
230
|
+
**blind_write**
|
231
|
+
|
232
|
+
Doesn't check if an object exists in GCS before writing. Default is false.
|
233
|
+
|
234
|
+
Allows to avoid granting of `storage.objects.get` permission.
|
235
|
+
|
236
|
+
Warning! If the object exists and `storage.objects.delete` permission is not
|
237
|
+
granted, it will result in an unrecoverable error. Usage of `%{hex_random}` is
|
238
|
+
recommended.
|
239
|
+
|
180
240
|
### ObjectMetadata
|
181
241
|
|
182
242
|
User provided web-safe keys and arbitrary string values that will returned with requests for the file as "x-goog-meta-" response headers.
|
data/fluent-plugin-gcs.gemspec
CHANGED
@@ -20,11 +20,10 @@ Gem::Specification.new do |spec|
|
|
20
20
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
21
21
|
spec.require_paths = ["lib"]
|
22
22
|
|
23
|
-
spec.add_runtime_dependency "fluentd", "
|
24
|
-
spec.add_runtime_dependency "google-cloud-storage", "~>
|
23
|
+
spec.add_runtime_dependency "fluentd", [">= 0.14.22", "< 2"]
|
24
|
+
spec.add_runtime_dependency "google-cloud-storage", "~> 1.1"
|
25
25
|
|
26
|
-
spec.add_development_dependency "
|
27
|
-
spec.add_development_dependency "rake", "~> 10.0"
|
26
|
+
spec.add_development_dependency "rake", ">= 12.3.3"
|
28
27
|
spec.add_development_dependency "rr", "= 1.1.2"
|
29
28
|
spec.add_development_dependency "test-unit", ">= 3.0.8"
|
30
29
|
spec.add_development_dependency "test-unit-rr", ">= 1.0.3"
|
@@ -4,11 +4,14 @@ require "socket"
|
|
4
4
|
|
5
5
|
require "fluent/plugin/gcs/object_creator"
|
6
6
|
require "fluent/plugin/gcs/version"
|
7
|
+
require "fluent/plugin/output"
|
7
8
|
|
8
|
-
module Fluent
|
9
|
-
class GCSOutput <
|
9
|
+
module Fluent::Plugin
|
10
|
+
class GCSOutput < Output
|
10
11
|
Fluent::Plugin.register_output("gcs", self)
|
11
12
|
|
13
|
+
helpers :compat_parameters, :formatter, :inject
|
14
|
+
|
12
15
|
def initialize
|
13
16
|
super
|
14
17
|
require "google/cloud/storage"
|
@@ -28,7 +31,7 @@ module Fluent
|
|
28
31
|
desc: "Format of GCS object keys"
|
29
32
|
config_param :path, :string, default: "",
|
30
33
|
desc: "Path prefix of the files on GCS"
|
31
|
-
config_param :store_as, :enum, list:
|
34
|
+
config_param :store_as, :enum, list: %i(gzip json text), default: :gzip,
|
32
35
|
desc: "Archive format on GCS"
|
33
36
|
config_param :transcoding, :bool, default: false,
|
34
37
|
desc: "Enable the decompressive form of transcoding"
|
@@ -40,46 +43,56 @@ module Fluent
|
|
40
43
|
desc: "Overwrite already existing path"
|
41
44
|
config_param :format, :string, default: "out_file",
|
42
45
|
desc: "Change one line format in the GCS object"
|
43
|
-
config_param :acl, :enum, list:
|
46
|
+
config_param :acl, :enum, list: %i(auth_read owner_full owner_read private project_private public_read), default: nil,
|
44
47
|
desc: "Permission for the object in GCS"
|
48
|
+
config_param :storage_class, :enum, list: %i(dra nearline coldline multi_regional regional standard), default: nil,
|
49
|
+
desc: "Storage class of the file"
|
45
50
|
config_param :encryption_key, :string, default: nil, secret: true,
|
46
51
|
desc: "Customer-supplied, AES-256 encryption key"
|
47
|
-
config_param :
|
48
|
-
desc: "
|
52
|
+
config_param :blind_write, :bool, default: false,
|
53
|
+
desc: "Whether to check if object already exists by given GCS path. Allows avoiding giving storage.object.get permission"
|
49
54
|
config_section :object_metadata, required: false do
|
50
55
|
config_param :key, :string, default: ""
|
51
56
|
config_param :value, :string, default: ""
|
52
57
|
end
|
53
|
-
|
54
|
-
|
58
|
+
|
59
|
+
DEFAULT_FORMAT_TYPE = "out_file"
|
60
|
+
|
61
|
+
config_section :format do
|
62
|
+
config_set_default :@type, DEFAULT_FORMAT_TYPE
|
63
|
+
end
|
64
|
+
|
65
|
+
config_section :buffer do
|
66
|
+
config_set_default :chunk_keys, ['time']
|
67
|
+
config_set_default :timekey, (60 * 60 * 24)
|
68
|
+
end
|
55
69
|
|
56
70
|
MAX_HEX_RANDOM_LENGTH = 32
|
57
71
|
|
58
72
|
def configure(conf)
|
73
|
+
compat_parameters_convert(conf, :buffer, :formatter, :inject)
|
59
74
|
super
|
60
75
|
|
61
|
-
if @encryption_key && @encryption_key_sha256.nil?
|
62
|
-
raise Fluent::ConfigError, "encryption_key_sha256 parameter must be provided if `encryption_key` is provided."
|
63
|
-
end
|
64
|
-
|
65
76
|
if @hex_random_length > MAX_HEX_RANDOM_LENGTH
|
66
77
|
raise Fluent::ConfigError, "hex_random_length parameter should be set to #{MAX_HEX_RANDOM_LENGTH} characters or less."
|
67
78
|
end
|
68
79
|
|
69
|
-
# The customer-supplied, AES-256 encryption key
|
80
|
+
# The customer-supplied, AES-256 encryption key that will be used to encrypt the file.
|
70
81
|
@encryption_opts = {
|
71
82
|
encryption_key: @encryption_key,
|
72
|
-
encryption_key_sha256: @encryption_key_sha256
|
73
83
|
}
|
74
84
|
|
75
85
|
if @object_metadata
|
76
86
|
@object_metadata_hash = @object_metadata.map {|m| [m.key, m.value] }.to_h
|
77
87
|
end
|
78
88
|
|
79
|
-
@formatter =
|
80
|
-
@formatter.configure(conf)
|
89
|
+
@formatter = formatter_create
|
81
90
|
|
82
91
|
@object_creator = Fluent::GCS.discovered_object_creator(@store_as, transcoding: @transcoding)
|
92
|
+
# For backward compatibility
|
93
|
+
# TODO: Remove time_slice_format when end of support compat_parameters
|
94
|
+
@configured_time_slice_format = conf['time_slice_format']
|
95
|
+
@time_slice_with_tz = Fluent::Timezone.formatter(@timekey_zone, @configured_time_slice_format || timekey_to_timeformat(@buffer_config['timekey']))
|
83
96
|
end
|
84
97
|
|
85
98
|
def start
|
@@ -96,7 +109,12 @@ module Fluent
|
|
96
109
|
end
|
97
110
|
|
98
111
|
def format(tag, time, record)
|
99
|
-
|
112
|
+
r = inject_values_to_record(tag, time, record)
|
113
|
+
@formatter.format(tag, time, r)
|
114
|
+
end
|
115
|
+
|
116
|
+
def multi_workers_ready?
|
117
|
+
true
|
100
118
|
end
|
101
119
|
|
102
120
|
def write(chunk)
|
@@ -106,6 +124,7 @@ module Fluent
|
|
106
124
|
opts = {
|
107
125
|
metadata: @object_metadata_hash,
|
108
126
|
acl: @acl,
|
127
|
+
storage_class: @storage_class,
|
109
128
|
content_type: @object_creator.content_type,
|
110
129
|
content_encoding: @object_creator.content_encoding,
|
111
130
|
}
|
@@ -132,23 +151,33 @@ module Fluent
|
|
132
151
|
Digest::MD5.hexdigest(chunk.unique_id)[0...@hex_random_length]
|
133
152
|
end
|
134
153
|
|
135
|
-
def
|
136
|
-
|
137
|
-
|
154
|
+
def check_object_exists(path)
|
155
|
+
if !@blind_write
|
156
|
+
return @gcs_bucket.find_file(path, @encryption_opts)
|
157
|
+
else
|
158
|
+
return false
|
159
|
+
end
|
138
160
|
end
|
139
161
|
|
140
162
|
def generate_path(chunk, i = 0, prev = nil)
|
163
|
+
metadata = chunk.metadata
|
164
|
+
time_slice = if metadata.timekey.nil?
|
165
|
+
''.freeze
|
166
|
+
else
|
167
|
+
@time_slice_with_tz.call(metadata.timekey)
|
168
|
+
end
|
141
169
|
tags = {
|
142
170
|
"%{file_extension}" => @object_creator.file_extension,
|
143
171
|
"%{hex_random}" => hex_random(chunk),
|
144
172
|
"%{hostname}" => Socket.gethostname,
|
145
173
|
"%{index}" => i,
|
146
|
-
"%{path}" =>
|
147
|
-
"%{time_slice}" =>
|
174
|
+
"%{path}" => @path,
|
175
|
+
"%{time_slice}" => time_slice,
|
148
176
|
"%{uuid_flush}" => SecureRandom.uuid,
|
149
177
|
}
|
150
178
|
path = @object_key_format.gsub(Regexp.union(tags.keys), tags)
|
151
|
-
|
179
|
+
path = extract_placeholders(path, chunk)
|
180
|
+
return path unless check_object_exists(path)
|
152
181
|
|
153
182
|
if path == prev
|
154
183
|
if @overwrite
|
@@ -159,5 +188,16 @@ module Fluent
|
|
159
188
|
end
|
160
189
|
generate_path(chunk, i + 1, path)
|
161
190
|
end
|
191
|
+
|
192
|
+
# This is stolen from Fluentd
|
193
|
+
def timekey_to_timeformat(timekey)
|
194
|
+
case timekey
|
195
|
+
when nil then ''
|
196
|
+
when 0...60 then '%Y%m%d%H%M%S' # 60 exclusive
|
197
|
+
when 60...3600 then '%Y%m%d%H%M'
|
198
|
+
when 3600...86400 then '%Y%m%d%H'
|
199
|
+
else '%Y%m%d'
|
200
|
+
end
|
201
|
+
end
|
162
202
|
end
|
163
203
|
end
|
metadata
CHANGED
@@ -1,71 +1,63 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-gcs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daichi HIRATA
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-10-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - "
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 0.14.22
|
20
|
+
- - "<"
|
18
21
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
22
|
+
version: '2'
|
20
23
|
type: :runtime
|
21
24
|
prerelease: false
|
22
25
|
version_requirements: !ruby/object:Gem::Requirement
|
23
26
|
requirements:
|
24
|
-
- - "
|
27
|
+
- - ">="
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: 0.14.22
|
30
|
+
- - "<"
|
25
31
|
- !ruby/object:Gem::Version
|
26
|
-
version:
|
32
|
+
version: '2'
|
27
33
|
- !ruby/object:Gem::Dependency
|
28
34
|
name: google-cloud-storage
|
29
35
|
requirement: !ruby/object:Gem::Requirement
|
30
36
|
requirements:
|
31
37
|
- - "~>"
|
32
38
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
39
|
+
version: '1.1'
|
34
40
|
type: :runtime
|
35
41
|
prerelease: false
|
36
42
|
version_requirements: !ruby/object:Gem::Requirement
|
37
43
|
requirements:
|
38
44
|
- - "~>"
|
39
45
|
- !ruby/object:Gem::Version
|
40
|
-
version: '
|
41
|
-
- !ruby/object:Gem::Dependency
|
42
|
-
name: bundler
|
43
|
-
requirement: !ruby/object:Gem::Requirement
|
44
|
-
requirements:
|
45
|
-
- - "~>"
|
46
|
-
- !ruby/object:Gem::Version
|
47
|
-
version: '1.13'
|
48
|
-
type: :development
|
49
|
-
prerelease: false
|
50
|
-
version_requirements: !ruby/object:Gem::Requirement
|
51
|
-
requirements:
|
52
|
-
- - "~>"
|
53
|
-
- !ruby/object:Gem::Version
|
54
|
-
version: '1.13'
|
46
|
+
version: '1.1'
|
55
47
|
- !ruby/object:Gem::Dependency
|
56
48
|
name: rake
|
57
49
|
requirement: !ruby/object:Gem::Requirement
|
58
50
|
requirements:
|
59
|
-
- - "
|
51
|
+
- - ">="
|
60
52
|
- !ruby/object:Gem::Version
|
61
|
-
version:
|
53
|
+
version: 12.3.3
|
62
54
|
type: :development
|
63
55
|
prerelease: false
|
64
56
|
version_requirements: !ruby/object:Gem::Requirement
|
65
57
|
requirements:
|
66
|
-
- - "
|
58
|
+
- - ">="
|
67
59
|
- !ruby/object:Gem::Version
|
68
|
-
version:
|
60
|
+
version: 12.3.3
|
69
61
|
- !ruby/object:Gem::Dependency
|
70
62
|
name: rr
|
71
63
|
requirement: !ruby/object:Gem::Requirement
|
@@ -131,6 +123,7 @@ extra_rdoc_files: []
|
|
131
123
|
files:
|
132
124
|
- ".gitignore"
|
133
125
|
- ".travis.yml"
|
126
|
+
- CHANGELOG.md
|
134
127
|
- Gemfile
|
135
128
|
- LICENSE.txt
|
136
129
|
- README.md
|
@@ -160,8 +153,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
160
153
|
- !ruby/object:Gem::Version
|
161
154
|
version: '0'
|
162
155
|
requirements: []
|
163
|
-
|
164
|
-
rubygems_version: 2.5.1
|
156
|
+
rubygems_version: 3.0.3
|
165
157
|
signing_key:
|
166
158
|
specification_version: 4
|
167
159
|
summary: Google Cloud Storage output plugin for Fluentd
|