fluent-plugin-gcs 0.4.0.beta1 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.github/workflows/test.yaml +24 -0
- data/CHANGELOG.md +17 -3
- data/Gemfile +10 -1
- data/README.md +14 -7
- data/fluent-plugin-gcs.gemspec +2 -9
- data/lib/fluent/plugin/gcs/version.rb +1 -1
- data/lib/fluent/plugin/out_gcs.rb +13 -3
- metadata +13 -98
- data/.travis.yml +0 -11
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
|
-
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: e6bc239a1a47a29d4cc3dc94aaaafc5e3ed6ed20e23e0ea9807c88309ec1f7a0
|
|
4
|
+
data.tar.gz: c27f4d556b545ffb6050a72dcc9ff6a7fcfc53d16f44bce305f14560a4482725
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 3eb03cfb810bf23d8447391145cd8ba6cf6379f1730e1c486a35bcafcb66fdf5b953741b5913b0255bc5969914fc1fda4e37110fe76eb7bf21f65d441466bedb
|
|
7
|
+
data.tar.gz: 85bcc85ab97e567a872893cfb890b7920343ff0010adf5d7c6d5b36f5ff952640bac4f42b90d811f1f1b6717c1691c40f6581d66a6da45135711a030dcf6b297
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
name: Test
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [ master ]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [ master ]
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
test:
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
strategy:
|
|
13
|
+
fail-fast: false
|
|
14
|
+
matrix:
|
|
15
|
+
ruby-version: ['2.7', '3.0', '3.1']
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@v3
|
|
18
|
+
- name: Set up Ruby
|
|
19
|
+
uses: ruby/setup-ruby@v1
|
|
20
|
+
with:
|
|
21
|
+
ruby-version: ${{ matrix.ruby-version }}
|
|
22
|
+
bundler-cache: true
|
|
23
|
+
- name: Run tests
|
|
24
|
+
run: bundle exec rake test
|
data/CHANGELOG.md
CHANGED
|
@@ -1,6 +1,19 @@
|
|
|
1
1
|
## [Unreleased]
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
New features / Enhancements
|
|
4
|
+
|
|
5
|
+
## [0.4.2] - 2022/08/16
|
|
6
|
+
|
|
7
|
+
Bug fixes
|
|
8
|
+
|
|
9
|
+
- [Fix automatic conversion from a hash to keyword arguments](https://github.com/daichirata/fluent-plugin-gcs/pull/22)
|
|
10
|
+
|
|
11
|
+
## [0.4.1] - 2020/04/17
|
|
12
|
+
|
|
13
|
+
New features
|
|
14
|
+
- [Support blind write to GSC](https://github.com/daichirata/fluent-plugin-gcs/pull/14)
|
|
15
|
+
|
|
16
|
+
## [0.4.0] - 2019/04/01
|
|
4
17
|
|
|
5
18
|
New features / Enhancements
|
|
6
19
|
|
|
@@ -26,7 +39,8 @@ New features / Enhancements
|
|
|
26
39
|
|
|
27
40
|
- Add support for `%{hostname}` of object_key_format
|
|
28
41
|
|
|
29
|
-
[Unreleased]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.
|
|
42
|
+
[Unreleased]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.4.0...HEAD
|
|
43
|
+
[0.4.0]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.3.0...v0.4.0
|
|
30
44
|
[0.3.0]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.2.0...v0.3.0
|
|
31
|
-
[0.2.0]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.1.
|
|
45
|
+
[0.2.0]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.1.0...v0.2.0
|
|
32
46
|
[0.1.1]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.1.0...v0.1.1
|
data/Gemfile
CHANGED
|
@@ -1,4 +1,13 @@
|
|
|
1
|
-
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
source "https://rubygems.org"
|
|
2
4
|
|
|
3
5
|
# Specify your gem's dependencies in fluent-plugin-gcs.gemspec
|
|
4
6
|
gemspec
|
|
7
|
+
|
|
8
|
+
gem "rake", "~> 13.0"
|
|
9
|
+
gem "rr", "= 1.1.2"
|
|
10
|
+
gem "test-unit", ">= 3.0.8"
|
|
11
|
+
gem "test-unit-rr", ">= 1.0.3"
|
|
12
|
+
gem "timecop"
|
|
13
|
+
gem "solargraph"
|
data/README.md
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# fluent-plugin-gcs
|
|
2
|
-
[](https://badge.fury.io/rb/fluent-plugin-gcs) [](https://badge.fury.io/rb/fluent-plugin-gcs) [](https://github.com/daichirata/fluent-plugin-gcs/actions/workflows/test.yaml) [](https://codeclimate.com/github/daichirata/fluent-plugin-gcs)
|
|
3
3
|
|
|
4
4
|
Google Cloud Storage output plugin for [Fluentd](https://github.com/fluent/fluentd).
|
|
5
5
|
|
|
@@ -7,16 +7,14 @@ Google Cloud Storage output plugin for [Fluentd](https://github.com/fluent/fluen
|
|
|
7
7
|
|
|
8
8
|
| fluent-plugin-gcs | fluentd | ruby |
|
|
9
9
|
|--------------------|------------|--------|
|
|
10
|
-
| >= 0.4.0 | >= v0.14.0 | >= 2.
|
|
10
|
+
| >= 0.4.0 | >= v0.14.0 | >= 2.4 |
|
|
11
11
|
| < 0.4.0 | >= v0.12.0 | >= 1.9 |
|
|
12
12
|
|
|
13
|
-
NOTE: fluent-plugin-gcs v0.4.0 is now beta.
|
|
14
|
-
|
|
15
13
|
## Installation
|
|
16
14
|
|
|
17
15
|
``` shell
|
|
18
|
-
$ gem install fluent-plugin-gcs -v "~> 0.3"
|
|
19
|
-
$ gem install fluent-plugin-gcs -v "0.4.0
|
|
16
|
+
$ gem install fluent-plugin-gcs -v "~> 0.3" --no-document # for fluentd v0.12 or later
|
|
17
|
+
$ gem install fluent-plugin-gcs -v "0.4.0" --no-document # for fluentd v0.14 or later
|
|
20
18
|
```
|
|
21
19
|
|
|
22
20
|
## Examples
|
|
@@ -227,9 +225,18 @@ Use Local time instead of UTC.
|
|
|
227
225
|
|
|
228
226
|
Use UTC instead of local time.
|
|
229
227
|
|
|
230
|
-
|
|
231
228
|
And see [official Time Sliced Output article](http://docs.fluentd.org/articles/output-plugin-overview#time-sliced-output-parameters)
|
|
232
229
|
|
|
230
|
+
**blind_write**
|
|
231
|
+
|
|
232
|
+
Doesn't check if an object exists in GCS before writing. Default is false.
|
|
233
|
+
|
|
234
|
+
Allows to avoid granting of `storage.objects.get` permission.
|
|
235
|
+
|
|
236
|
+
Warning! If the object exists and `storage.objects.delete` permission is not
|
|
237
|
+
granted, it will result in an unrecoverable error. Usage of `%{hex_random}` is
|
|
238
|
+
recommended.
|
|
239
|
+
|
|
233
240
|
### ObjectMetadata
|
|
234
241
|
|
|
235
242
|
User provided web-safe keys and arbitrary string values that will returned with requests for the file as "x-goog-meta-" response headers.
|
data/fluent-plugin-gcs.gemspec
CHANGED
|
@@ -20,13 +20,6 @@ Gem::Specification.new do |spec|
|
|
|
20
20
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
|
21
21
|
spec.require_paths = ["lib"]
|
|
22
22
|
|
|
23
|
-
spec.add_runtime_dependency "fluentd", [">= 0.14.
|
|
24
|
-
spec.add_runtime_dependency "google-cloud-storage", "~> 1.1
|
|
25
|
-
|
|
26
|
-
spec.add_development_dependency "bundler", "~> 1.13"
|
|
27
|
-
spec.add_development_dependency "rake", "~> 10.0"
|
|
28
|
-
spec.add_development_dependency "rr", "= 1.1.2"
|
|
29
|
-
spec.add_development_dependency "test-unit", ">= 3.0.8"
|
|
30
|
-
spec.add_development_dependency "test-unit-rr", ">= 1.0.3"
|
|
31
|
-
spec.add_development_dependency "timecop"
|
|
23
|
+
spec.add_runtime_dependency "fluentd", [">= 0.14.22", "< 2"]
|
|
24
|
+
spec.add_runtime_dependency "google-cloud-storage", "~> 1.1"
|
|
32
25
|
end
|
|
@@ -49,6 +49,8 @@ module Fluent::Plugin
|
|
|
49
49
|
desc: "Storage class of the file"
|
|
50
50
|
config_param :encryption_key, :string, default: nil, secret: true,
|
|
51
51
|
desc: "Customer-supplied, AES-256 encryption key"
|
|
52
|
+
config_param :blind_write, :bool, default: false,
|
|
53
|
+
desc: "Whether to check if object already exists by given GCS path. Allows avoiding giving storage.object.get permission"
|
|
52
54
|
config_section :object_metadata, required: false do
|
|
53
55
|
config_param :key, :string, default: ""
|
|
54
56
|
config_param :value, :string, default: ""
|
|
@@ -129,7 +131,7 @@ module Fluent::Plugin
|
|
|
129
131
|
opts.merge!(@encryption_opts)
|
|
130
132
|
|
|
131
133
|
log.debug { "out_gcs: upload chunk:#{chunk.key} to gcs://#{@bucket}/#{path} options: #{opts}" }
|
|
132
|
-
@gcs_bucket.upload_file(obj.path, path, opts)
|
|
134
|
+
@gcs_bucket.upload_file(obj.path, path, **opts)
|
|
133
135
|
end
|
|
134
136
|
end
|
|
135
137
|
|
|
@@ -149,6 +151,14 @@ module Fluent::Plugin
|
|
|
149
151
|
Digest::MD5.hexdigest(chunk.unique_id)[0...@hex_random_length]
|
|
150
152
|
end
|
|
151
153
|
|
|
154
|
+
def check_object_exists(path)
|
|
155
|
+
if !@blind_write
|
|
156
|
+
return @gcs_bucket.find_file(path, **@encryption_opts)
|
|
157
|
+
else
|
|
158
|
+
return false
|
|
159
|
+
end
|
|
160
|
+
end
|
|
161
|
+
|
|
152
162
|
def generate_path(chunk, i = 0, prev = nil)
|
|
153
163
|
metadata = chunk.metadata
|
|
154
164
|
time_slice = if metadata.timekey.nil?
|
|
@@ -166,8 +176,8 @@ module Fluent::Plugin
|
|
|
166
176
|
"%{uuid_flush}" => SecureRandom.uuid,
|
|
167
177
|
}
|
|
168
178
|
path = @object_key_format.gsub(Regexp.union(tags.keys), tags)
|
|
169
|
-
path = extract_placeholders(path,
|
|
170
|
-
return path unless
|
|
179
|
+
path = extract_placeholders(path, chunk)
|
|
180
|
+
return path unless check_object_exists(path)
|
|
171
181
|
|
|
172
182
|
if path == prev
|
|
173
183
|
if @overwrite
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-gcs
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.4.
|
|
4
|
+
version: 0.4.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Daichi HIRATA
|
|
8
|
-
autorequire:
|
|
8
|
+
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2022-08-16 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: fluentd
|
|
@@ -16,7 +16,7 @@ dependencies:
|
|
|
16
16
|
requirements:
|
|
17
17
|
- - ">="
|
|
18
18
|
- !ruby/object:Gem::Version
|
|
19
|
-
version: 0.14.
|
|
19
|
+
version: 0.14.22
|
|
20
20
|
- - "<"
|
|
21
21
|
- !ruby/object:Gem::Version
|
|
22
22
|
version: '2'
|
|
@@ -26,7 +26,7 @@ dependencies:
|
|
|
26
26
|
requirements:
|
|
27
27
|
- - ">="
|
|
28
28
|
- !ruby/object:Gem::Version
|
|
29
|
-
version: 0.14.
|
|
29
|
+
version: 0.14.22
|
|
30
30
|
- - "<"
|
|
31
31
|
- !ruby/object:Gem::Version
|
|
32
32
|
version: '2'
|
|
@@ -36,98 +36,14 @@ dependencies:
|
|
|
36
36
|
requirements:
|
|
37
37
|
- - "~>"
|
|
38
38
|
- !ruby/object:Gem::Version
|
|
39
|
-
version: 1.1
|
|
39
|
+
version: '1.1'
|
|
40
40
|
type: :runtime
|
|
41
41
|
prerelease: false
|
|
42
42
|
version_requirements: !ruby/object:Gem::Requirement
|
|
43
43
|
requirements:
|
|
44
44
|
- - "~>"
|
|
45
45
|
- !ruby/object:Gem::Version
|
|
46
|
-
version: 1.1
|
|
47
|
-
- !ruby/object:Gem::Dependency
|
|
48
|
-
name: bundler
|
|
49
|
-
requirement: !ruby/object:Gem::Requirement
|
|
50
|
-
requirements:
|
|
51
|
-
- - "~>"
|
|
52
|
-
- !ruby/object:Gem::Version
|
|
53
|
-
version: '1.13'
|
|
54
|
-
type: :development
|
|
55
|
-
prerelease: false
|
|
56
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
57
|
-
requirements:
|
|
58
|
-
- - "~>"
|
|
59
|
-
- !ruby/object:Gem::Version
|
|
60
|
-
version: '1.13'
|
|
61
|
-
- !ruby/object:Gem::Dependency
|
|
62
|
-
name: rake
|
|
63
|
-
requirement: !ruby/object:Gem::Requirement
|
|
64
|
-
requirements:
|
|
65
|
-
- - "~>"
|
|
66
|
-
- !ruby/object:Gem::Version
|
|
67
|
-
version: '10.0'
|
|
68
|
-
type: :development
|
|
69
|
-
prerelease: false
|
|
70
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
71
|
-
requirements:
|
|
72
|
-
- - "~>"
|
|
73
|
-
- !ruby/object:Gem::Version
|
|
74
|
-
version: '10.0'
|
|
75
|
-
- !ruby/object:Gem::Dependency
|
|
76
|
-
name: rr
|
|
77
|
-
requirement: !ruby/object:Gem::Requirement
|
|
78
|
-
requirements:
|
|
79
|
-
- - '='
|
|
80
|
-
- !ruby/object:Gem::Version
|
|
81
|
-
version: 1.1.2
|
|
82
|
-
type: :development
|
|
83
|
-
prerelease: false
|
|
84
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
85
|
-
requirements:
|
|
86
|
-
- - '='
|
|
87
|
-
- !ruby/object:Gem::Version
|
|
88
|
-
version: 1.1.2
|
|
89
|
-
- !ruby/object:Gem::Dependency
|
|
90
|
-
name: test-unit
|
|
91
|
-
requirement: !ruby/object:Gem::Requirement
|
|
92
|
-
requirements:
|
|
93
|
-
- - ">="
|
|
94
|
-
- !ruby/object:Gem::Version
|
|
95
|
-
version: 3.0.8
|
|
96
|
-
type: :development
|
|
97
|
-
prerelease: false
|
|
98
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
99
|
-
requirements:
|
|
100
|
-
- - ">="
|
|
101
|
-
- !ruby/object:Gem::Version
|
|
102
|
-
version: 3.0.8
|
|
103
|
-
- !ruby/object:Gem::Dependency
|
|
104
|
-
name: test-unit-rr
|
|
105
|
-
requirement: !ruby/object:Gem::Requirement
|
|
106
|
-
requirements:
|
|
107
|
-
- - ">="
|
|
108
|
-
- !ruby/object:Gem::Version
|
|
109
|
-
version: 1.0.3
|
|
110
|
-
type: :development
|
|
111
|
-
prerelease: false
|
|
112
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
113
|
-
requirements:
|
|
114
|
-
- - ">="
|
|
115
|
-
- !ruby/object:Gem::Version
|
|
116
|
-
version: 1.0.3
|
|
117
|
-
- !ruby/object:Gem::Dependency
|
|
118
|
-
name: timecop
|
|
119
|
-
requirement: !ruby/object:Gem::Requirement
|
|
120
|
-
requirements:
|
|
121
|
-
- - ">="
|
|
122
|
-
- !ruby/object:Gem::Version
|
|
123
|
-
version: '0'
|
|
124
|
-
type: :development
|
|
125
|
-
prerelease: false
|
|
126
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
127
|
-
requirements:
|
|
128
|
-
- - ">="
|
|
129
|
-
- !ruby/object:Gem::Version
|
|
130
|
-
version: '0'
|
|
46
|
+
version: '1.1'
|
|
131
47
|
description: Google Cloud Storage output plugin for Fluentd
|
|
132
48
|
email:
|
|
133
49
|
- hirata.daichi@gmail.com
|
|
@@ -135,8 +51,8 @@ executables: []
|
|
|
135
51
|
extensions: []
|
|
136
52
|
extra_rdoc_files: []
|
|
137
53
|
files:
|
|
54
|
+
- ".github/workflows/test.yaml"
|
|
138
55
|
- ".gitignore"
|
|
139
|
-
- ".travis.yml"
|
|
140
56
|
- CHANGELOG.md
|
|
141
57
|
- Gemfile
|
|
142
58
|
- LICENSE.txt
|
|
@@ -152,7 +68,7 @@ homepage: https://github.com/daichirata/fluent-plugin-gcs
|
|
|
152
68
|
licenses:
|
|
153
69
|
- Apache-2.0
|
|
154
70
|
metadata: {}
|
|
155
|
-
post_install_message:
|
|
71
|
+
post_install_message:
|
|
156
72
|
rdoc_options: []
|
|
157
73
|
require_paths:
|
|
158
74
|
- lib
|
|
@@ -163,13 +79,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
163
79
|
version: '0'
|
|
164
80
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
165
81
|
requirements:
|
|
166
|
-
- - "
|
|
82
|
+
- - ">="
|
|
167
83
|
- !ruby/object:Gem::Version
|
|
168
|
-
version:
|
|
84
|
+
version: '0'
|
|
169
85
|
requirements: []
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
signing_key:
|
|
86
|
+
rubygems_version: 3.3.7
|
|
87
|
+
signing_key:
|
|
173
88
|
specification_version: 4
|
|
174
89
|
summary: Google Cloud Storage output plugin for Fluentd
|
|
175
90
|
test_files: []
|