fluent-plugin-gcs 0.4.3 → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/dependabot.yml +20 -0
- data/.github/workflows/test.yml +56 -0
- data/Gemfile +11 -6
- data/README.md +164 -145
- data/lib/fluent/plugin/gcs/object_creator.rb +62 -1
- data/lib/fluent/plugin/gcs/version.rb +1 -1
- data/lib/fluent/plugin/out_gcs.rb +18 -3
- metadata +5 -5
- data/.github/workflows/test.yaml +0 -24
- data/CHANGELOG.md +0 -46
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: e4b7b5382b8e814b5ce2c59b6c798fdd1b15fb39b2a1d3a8c51ca1cbdbf54344
|
|
4
|
+
data.tar.gz: 0ac7404315835df4c1d692cb9a2c03144073a8a8030481f99e59837130bbe049
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: c5a98505ac7f1a0c5115d99afc392855fdf749fec3ecce24c9b85739587b0e116e655b4e15fac271df79fef723fcd00e0f80ce9d13ba745ec88c1d84238f2828
|
|
7
|
+
data.tar.gz: bf14715a1eba9046d1154ba4144508003ddc9b39d4500371af6b4679d92b24eb548623fca9dd952ffa3ecfa2b0688a8c3a00f465e831b9563cb6368881b54d73
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
version: 2
|
|
2
|
+
updates:
|
|
3
|
+
- package-ecosystem: github-actions
|
|
4
|
+
directory: /
|
|
5
|
+
schedule:
|
|
6
|
+
interval: weekly
|
|
7
|
+
groups:
|
|
8
|
+
actions:
|
|
9
|
+
patterns:
|
|
10
|
+
- "*"
|
|
11
|
+
|
|
12
|
+
- package-ecosystem: bundler
|
|
13
|
+
directory: /
|
|
14
|
+
schedule:
|
|
15
|
+
interval: weekly
|
|
16
|
+
groups:
|
|
17
|
+
development:
|
|
18
|
+
dependency-type: development
|
|
19
|
+
production:
|
|
20
|
+
dependency-type: production
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
name: Test
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [main]
|
|
8
|
+
workflow_dispatch:
|
|
9
|
+
|
|
10
|
+
permissions:
|
|
11
|
+
contents: read
|
|
12
|
+
|
|
13
|
+
concurrency:
|
|
14
|
+
group: ${{ github.workflow }}-${{ github.ref }}
|
|
15
|
+
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
|
|
16
|
+
|
|
17
|
+
jobs:
|
|
18
|
+
test:
|
|
19
|
+
runs-on: ubuntu-latest
|
|
20
|
+
strategy:
|
|
21
|
+
fail-fast: false
|
|
22
|
+
matrix:
|
|
23
|
+
ruby-version: ['3.3', '3.4']
|
|
24
|
+
steps:
|
|
25
|
+
- uses: actions/checkout@v6
|
|
26
|
+
- name: Set up Ruby
|
|
27
|
+
uses: ruby/setup-ruby@v1
|
|
28
|
+
with:
|
|
29
|
+
ruby-version: ${{ matrix.ruby-version }}
|
|
30
|
+
bundler-cache: true
|
|
31
|
+
- name: Run tests
|
|
32
|
+
run: bundle exec rake test
|
|
33
|
+
|
|
34
|
+
build:
|
|
35
|
+
runs-on: ubuntu-latest
|
|
36
|
+
steps:
|
|
37
|
+
- uses: actions/checkout@v6
|
|
38
|
+
- name: Set up Ruby
|
|
39
|
+
uses: ruby/setup-ruby@v1
|
|
40
|
+
with:
|
|
41
|
+
ruby-version: '3.4'
|
|
42
|
+
bundler-cache: true
|
|
43
|
+
- name: Build gem
|
|
44
|
+
run: gem build fluent-plugin-gcs.gemspec
|
|
45
|
+
|
|
46
|
+
audit:
|
|
47
|
+
runs-on: ubuntu-latest
|
|
48
|
+
steps:
|
|
49
|
+
- uses: actions/checkout@v6
|
|
50
|
+
- name: Set up Ruby
|
|
51
|
+
uses: ruby/setup-ruby@v1
|
|
52
|
+
with:
|
|
53
|
+
ruby-version: '3.4'
|
|
54
|
+
bundler-cache: true
|
|
55
|
+
- name: Bundler audit
|
|
56
|
+
run: bundle exec bundler-audit check --update
|
data/Gemfile
CHANGED
|
@@ -5,9 +5,14 @@ source "https://rubygems.org"
|
|
|
5
5
|
# Specify your gem's dependencies in fluent-plugin-gcs.gemspec
|
|
6
6
|
gemspec
|
|
7
7
|
|
|
8
|
-
|
|
9
|
-
gem "
|
|
10
|
-
gem "test-unit", ">= 3.0.8"
|
|
11
|
-
gem "
|
|
12
|
-
gem "timecop"
|
|
13
|
-
gem "
|
|
8
|
+
group :test do
|
|
9
|
+
gem "rake", "~> 13.0"
|
|
10
|
+
gem "test-unit", ">= 3.0.8"
|
|
11
|
+
gem "mocha", "~> 3.1"
|
|
12
|
+
gem "timecop"
|
|
13
|
+
gem "bundler-audit", "~> 0.9"
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
group :development, optional: true do
|
|
17
|
+
gem "solargraph"
|
|
18
|
+
end
|
data/README.md
CHANGED
|
@@ -1,27 +1,48 @@
|
|
|
1
1
|
# fluent-plugin-gcs
|
|
2
|
-
|
|
2
|
+
|
|
3
|
+
[](https://github.com/daichirata/fluent-plugin-gcs/actions/workflows/test.yml)
|
|
4
|
+
[](https://badge.fury.io/rb/fluent-plugin-gcs)
|
|
3
5
|
|
|
4
6
|
Google Cloud Storage output plugin for [Fluentd](https://github.com/fluent/fluentd).
|
|
5
7
|
|
|
8
|
+
## Table of contents
|
|
9
|
+
|
|
10
|
+
- [Requirements](#requirements)
|
|
11
|
+
- [Installation](#installation)
|
|
12
|
+
- [Example](#example)
|
|
13
|
+
- [Configuration](#configuration)
|
|
14
|
+
- [Authentication](#authentication)
|
|
15
|
+
- [Bucket and object placement](#bucket-and-object-placement)
|
|
16
|
+
- [Storage format](#storage-format)
|
|
17
|
+
- [Server-side options](#server-side-options)
|
|
18
|
+
- [Object metadata](#object-metadata)
|
|
19
|
+
- [Development](#development)
|
|
20
|
+
- [Author](#author)
|
|
21
|
+
- [License](#license)
|
|
22
|
+
|
|
6
23
|
## Requirements
|
|
7
24
|
|
|
8
|
-
| fluent-plugin-gcs
|
|
9
|
-
|
|
10
|
-
| >= 0.4.
|
|
11
|
-
|
|
|
25
|
+
| fluent-plugin-gcs | fluentd | ruby |
|
|
26
|
+
|-------------------|-------------|--------|
|
|
27
|
+
| >= 0.4.5 | >= 0.14.22 | >= 3.3 |
|
|
28
|
+
| >= 0.4.0 | >= 0.14.22 | >= 2.4 |
|
|
29
|
+
| < 0.4.0 | >= 0.12.0 | >= 1.9 |
|
|
12
30
|
|
|
13
31
|
## Installation
|
|
14
32
|
|
|
15
|
-
```
|
|
16
|
-
|
|
17
|
-
$ gem install fluent-plugin-gcs -v "0.4.0" --no-document # for fluentd v0.14 or later
|
|
33
|
+
```shell
|
|
34
|
+
gem install fluent-plugin-gcs
|
|
18
35
|
```
|
|
19
36
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
### For v0.14 style
|
|
37
|
+
If you use td-agent or fluent-package:
|
|
23
38
|
|
|
39
|
+
```shell
|
|
40
|
+
fluent-gem install fluent-plugin-gcs
|
|
24
41
|
```
|
|
42
|
+
|
|
43
|
+
## Example
|
|
44
|
+
|
|
45
|
+
```aconf
|
|
25
46
|
<match pattern>
|
|
26
47
|
@type gcs
|
|
27
48
|
|
|
@@ -31,14 +52,14 @@ $ gem install fluent-plugin-gcs -v "0.4.0" --no-document # for fluentd v0.14 or
|
|
|
31
52
|
object_key_format %{path}%{time_slice}_%{index}.%{file_extension}
|
|
32
53
|
path logs/${tag}/%Y/%m/%d/
|
|
33
54
|
|
|
34
|
-
#
|
|
35
|
-
#
|
|
55
|
+
# If you want to use ${tag} or %Y/%m/%d/ in path / object_key_format,
|
|
56
|
+
# specify the corresponding chunk keys in the <buffer> argument.
|
|
36
57
|
<buffer tag,time>
|
|
37
58
|
@type file
|
|
38
59
|
path /var/log/fluent/gcs
|
|
39
|
-
timekey 1h
|
|
60
|
+
timekey 1h # 1 hour partition
|
|
40
61
|
timekey_wait 10m
|
|
41
|
-
timekey_use_utc true
|
|
62
|
+
timekey_use_utc true
|
|
42
63
|
</buffer>
|
|
43
64
|
|
|
44
65
|
<format>
|
|
@@ -47,201 +68,182 @@ $ gem install fluent-plugin-gcs -v "0.4.0" --no-document # for fluentd v0.14 or
|
|
|
47
68
|
</match>
|
|
48
69
|
```
|
|
49
70
|
|
|
50
|
-
### For v0.12 style
|
|
51
|
-
|
|
52
|
-
```
|
|
53
|
-
<match pattern>
|
|
54
|
-
@type gcs
|
|
55
|
-
|
|
56
|
-
project YOUR_PROJECT
|
|
57
|
-
keyfile YOUR_KEYFILE_PATH
|
|
58
|
-
bucket YOUR_GCS_BUCKET_NAME
|
|
59
|
-
object_key_format %{path}%{time_slice}_%{index}.%{file_extension}
|
|
60
|
-
path logs/
|
|
61
|
-
buffer_path /var/log/fluent/gcs
|
|
62
|
-
|
|
63
|
-
time_slice_format %Y%m%d-%H
|
|
64
|
-
time_slice_wait 10m
|
|
65
|
-
utc
|
|
66
|
-
</match>
|
|
67
|
-
```
|
|
68
|
-
|
|
69
71
|
## Configuration
|
|
70
72
|
|
|
71
73
|
### Authentication
|
|
72
74
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
+
Provide the project and credentials explicitly, or rely on the
|
|
76
|
+
Application Default Credentials when running on Google Compute Engine,
|
|
77
|
+
GKE, or Cloud Run.
|
|
75
78
|
|
|
76
|
-
|
|
79
|
+
#### project
|
|
77
80
|
|
|
78
|
-
Project identifier for GCS. Project
|
|
79
|
-
* Specify project in `project`
|
|
80
|
-
* Discover project in environment variables `STORAGE_PROJECT`, `GOOGLE_CLOUD_PROJECT`, `GCLOUD_PROJECT`
|
|
81
|
-
* Discover GCE credentials
|
|
81
|
+
Project identifier for GCS. Project is discovered in the following order:
|
|
82
82
|
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
* Specify credentials path in `keyfile`
|
|
87
|
-
* Discover credentials path in environment variables `GOOGLE_CLOUD_KEYFILE`, `GCLOUD_KEYFILE`
|
|
88
|
-
* Discover credentials JSON in environment variables `GOOGLE_CLOUD_KEYFILE_JSON`, `GCLOUD_KEYFILE_JSON`
|
|
89
|
-
* Discover credentials file in the Cloud SDK's path
|
|
90
|
-
* Discover GCE credentials
|
|
83
|
+
* `project` setting
|
|
84
|
+
* Environment variables `STORAGE_PROJECT`, `GOOGLE_CLOUD_PROJECT`, `GCLOUD_PROJECT`
|
|
85
|
+
* GCE metadata
|
|
91
86
|
|
|
92
|
-
|
|
87
|
+
#### keyfile
|
|
93
88
|
|
|
94
|
-
|
|
89
|
+
Path of GCS service account credentials JSON file. Credentials are discovered in the following order:
|
|
95
90
|
|
|
96
|
-
|
|
91
|
+
* `keyfile` setting
|
|
92
|
+
* Environment variables `GOOGLE_CLOUD_KEYFILE`, `GCLOUD_KEYFILE`
|
|
93
|
+
* Environment variables `GOOGLE_CLOUD_KEYFILE_JSON`, `GCLOUD_KEYFILE_JSON`
|
|
94
|
+
* The Cloud SDK's well-known credentials path
|
|
95
|
+
* GCE metadata
|
|
97
96
|
|
|
98
|
-
|
|
97
|
+
#### credentials_json
|
|
99
98
|
|
|
100
|
-
|
|
99
|
+
GCS service account credentials in JSON form. Takes precedence over `keyfile` when both are set. Marked as a secret so it is not logged.
|
|
101
100
|
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
101
|
+
```aconf
|
|
102
|
+
credentials_json {"type": "service_account", "project_id": "...", ...}
|
|
103
|
+
```
|
|
105
104
|
|
|
106
|
-
|
|
105
|
+
#### client_retries
|
|
107
106
|
|
|
108
|
-
|
|
109
|
-
* json
|
|
110
|
-
* text
|
|
107
|
+
Number of times to retry requests on server error.
|
|
111
108
|
|
|
112
|
-
|
|
109
|
+
#### client_timeout
|
|
113
110
|
|
|
114
|
-
|
|
111
|
+
Default timeout (seconds) used for requests.
|
|
115
112
|
|
|
116
|
-
|
|
113
|
+
### Bucket and object placement
|
|
117
114
|
|
|
118
|
-
|
|
115
|
+
#### bucket (required)
|
|
119
116
|
|
|
120
|
-
|
|
121
|
-
* %{time_slice}
|
|
122
|
-
* %{index}
|
|
123
|
-
* %{file_extension}
|
|
124
|
-
* %{uuid_flush}
|
|
125
|
-
* %{hex_random}
|
|
126
|
-
* %{hostname}
|
|
117
|
+
GCS bucket name.
|
|
127
118
|
|
|
128
|
-
|
|
119
|
+
#### path
|
|
129
120
|
|
|
130
|
-
|
|
131
|
-
* `%{time_slice}` is the time-slice in text that are formatted with `time_slice_format`.
|
|
132
|
-
* `%{index}` is the sequential number starts from 0, increments when multiple files are uploaded to GCS in the same time slice.
|
|
133
|
-
* `%{file_extention}` is changed by the value of `store_as`.
|
|
134
|
-
* gzip - gz
|
|
135
|
-
* json - json
|
|
136
|
-
* text - txt
|
|
137
|
-
* `%{uuid_flush}` a uuid that is replaced everytime the buffer will be flushed
|
|
138
|
-
* `%{hex_random}` a random hex string that is replaced for each buffer chunk, not assured to be unique. You can configure the length of string with a `hex_random_length` parameter (Default: 4).
|
|
139
|
-
* `%{hostname}` is set to the standard host name of the system of the running server.
|
|
121
|
+
Path prefix of the files on GCS. Default is `""` (no prefix).
|
|
140
122
|
|
|
141
|
-
|
|
123
|
+
#### object_key_format
|
|
142
124
|
|
|
143
|
-
|
|
125
|
+
The format of GCS object keys. The default is `%{path}%{time_slice}_%{index}.%{file_extension}`.
|
|
144
126
|
|
|
145
|
-
|
|
127
|
+
Available placeholders:
|
|
146
128
|
|
|
147
|
-
|
|
129
|
+
| Placeholder | Description |
|
|
130
|
+
|--------------------|-------------|
|
|
131
|
+
| `%{path}` | The value of `path` |
|
|
132
|
+
| `%{time_slice}` | The time slice text formatted based on the `<buffer>` `timekey` |
|
|
133
|
+
| `%{index}` | Sequential number starting from 0, increments when multiple files are uploaded in the same time slice |
|
|
134
|
+
| `%{file_extension}` | Inferred from `store_as` (`gz` for gzip / gzip_command, `json` for json, `txt` for text) |
|
|
135
|
+
| `%{uuid_flush}` | A UUID generated each time the buffer is flushed |
|
|
136
|
+
| `%{hex_random}` | A random hex string generated for each buffer chunk. Configurable via `hex_random_length` (default: 4) |
|
|
137
|
+
| `%{hostname}` | The hostname of the running server |
|
|
148
138
|
|
|
149
|
-
|
|
139
|
+
#### hex_random_length
|
|
150
140
|
|
|
151
|
-
|
|
141
|
+
Length of the `%{hex_random}` placeholder. Max 32.
|
|
152
142
|
|
|
153
|
-
|
|
143
|
+
#### overwrite
|
|
154
144
|
|
|
155
|
-
|
|
145
|
+
Overwrite the existing object at the resolved path. Default is `false`, which raises an error if an object of the same path already exists, or increments `%{index}` until finding an unused path.
|
|
156
146
|
|
|
157
|
-
|
|
158
|
-
* json
|
|
159
|
-
* ltsv
|
|
160
|
-
* single_value
|
|
147
|
+
#### blind_write
|
|
161
148
|
|
|
162
|
-
|
|
149
|
+
Skip checking whether the object exists in GCS before writing. Default is `false`.
|
|
163
150
|
|
|
164
|
-
|
|
151
|
+
Useful when you do not want to grant `storage.objects.get` permission.
|
|
165
152
|
|
|
166
|
-
|
|
153
|
+
> **Warning**
|
|
154
|
+
> If the object already exists and `storage.objects.delete` is not granted either, you get an unrecoverable error. Use `%{hex_random}` or `%{uuid_flush}` to keep object keys unique.
|
|
167
155
|
|
|
168
|
-
|
|
156
|
+
### Storage format
|
|
169
157
|
|
|
170
|
-
|
|
158
|
+
#### store_as
|
|
171
159
|
|
|
172
|
-
|
|
173
|
-
* `owner_full` - File owner gets OWNER access, and project team owners get OWNER access.
|
|
174
|
-
* `owner_read` - File owner gets OWNER access, and project team owners get READER access.
|
|
175
|
-
* `private` - File owner gets OWNER access.
|
|
176
|
-
* `project_private` - File owner gets OWNER access, and project team members get access according to their roles.
|
|
177
|
-
* `public_read` - File owner gets OWNER access, and allUsers get READER access.
|
|
160
|
+
Archive format on GCS. Default is `gzip`.
|
|
178
161
|
|
|
179
|
-
|
|
162
|
+
| Value | Description |
|
|
163
|
+
|-----------------|-------------|
|
|
164
|
+
| `gzip` | Compress with the Ruby built-in `Zlib::GzipWriter` |
|
|
165
|
+
| `gzip_command` | Compress with an external `gzip` command. Faster for large chunks. Falls back to `Zlib::GzipWriter` if the command fails |
|
|
166
|
+
| `json` | Upload as `application/json` |
|
|
167
|
+
| `text` | Upload as `text/plain` |
|
|
180
168
|
|
|
181
|
-
|
|
169
|
+
#### gzip_command_parameter
|
|
182
170
|
|
|
183
|
-
|
|
171
|
+
Additional parameters passed to the external `gzip` command when `store_as` is `gzip_command`. Parsed with `shellsplit`, so the value is **not** evaluated by a shell. Default is `""`.
|
|
184
172
|
|
|
185
|
-
|
|
186
|
-
* `nearline` - Nearline Storage
|
|
187
|
-
* `coldline` - Coldline Storage
|
|
188
|
-
* `multi_regional` - Multi-Regional Storage
|
|
189
|
-
* `regional` - Regional Storage
|
|
190
|
-
* `standard` - Standard Storage
|
|
173
|
+
Examples:
|
|
191
174
|
|
|
192
|
-
|
|
175
|
+
* `-1` for fast compression
|
|
176
|
+
* `-9` for best compression
|
|
193
177
|
|
|
194
|
-
|
|
178
|
+
```aconf
|
|
179
|
+
<match pattern>
|
|
180
|
+
@type gcs
|
|
181
|
+
store_as gzip_command
|
|
182
|
+
gzip_command_parameter -1
|
|
183
|
+
...
|
|
184
|
+
</match>
|
|
185
|
+
```
|
|
195
186
|
|
|
196
|
-
|
|
187
|
+
#### transcoding
|
|
197
188
|
|
|
198
|
-
|
|
189
|
+
Enable the decompressive form of transcoding. See [Transcoding of gzip-compressed files](https://cloud.google.com/storage/docs/transcoding).
|
|
199
190
|
|
|
200
|
-
|
|
191
|
+
#### format
|
|
201
192
|
|
|
202
|
-
|
|
203
|
-
if a GCS object of the same path already exists, or increment the
|
|
204
|
-
`%{index}` placeholder until finding an absent path.
|
|
193
|
+
Per-line format inside the uploaded object. Default is `out_file`. Common values:
|
|
205
194
|
|
|
206
|
-
|
|
195
|
+
* `out_file` (default)
|
|
196
|
+
* `json`
|
|
197
|
+
* `ltsv`
|
|
198
|
+
* `single_value`
|
|
207
199
|
|
|
208
|
-
|
|
200
|
+
See the [official Formatter documentation](https://docs.fluentd.org/formatter).
|
|
209
201
|
|
|
210
|
-
|
|
202
|
+
### Server-side options
|
|
211
203
|
|
|
212
|
-
|
|
213
|
-
'%Y%m%d%H' to split files hourly.
|
|
204
|
+
#### auto_create_bucket
|
|
214
205
|
|
|
215
|
-
|
|
206
|
+
Create the GCS bucket if it does not exist. Default is `true`.
|
|
216
207
|
|
|
217
|
-
|
|
218
|
-
old logs may reache.
|
|
208
|
+
#### acl
|
|
219
209
|
|
|
220
|
-
|
|
210
|
+
Permission for the uploaded object. Acceptable values:
|
|
221
211
|
|
|
222
|
-
|
|
212
|
+
| Value | Description |
|
|
213
|
+
|-------------------|-------------|
|
|
214
|
+
| `auth_read` | File owner gets OWNER access, and allAuthenticatedUsers get READER access |
|
|
215
|
+
| `owner_full` | File owner gets OWNER access, and project team owners get OWNER access |
|
|
216
|
+
| `owner_read` | File owner gets OWNER access, and project team owners get READER access |
|
|
217
|
+
| `private` | File owner gets OWNER access |
|
|
218
|
+
| `project_private` | File owner gets OWNER access, and project team members get access according to their roles |
|
|
219
|
+
| `public_read` | File owner gets OWNER access, and allUsers get READER access |
|
|
223
220
|
|
|
224
|
-
|
|
221
|
+
Default is `nil` (bucket default object ACL). See the [GCS access control documentation](https://cloud.google.com/storage/docs/access-control/lists).
|
|
225
222
|
|
|
226
|
-
|
|
223
|
+
#### storage_class
|
|
227
224
|
|
|
228
|
-
|
|
225
|
+
Storage class of the uploaded object. Acceptable values:
|
|
229
226
|
|
|
230
|
-
|
|
227
|
+
| Value | Description |
|
|
228
|
+
|------------------|-------------|
|
|
229
|
+
| `dra` | Durable Reduced Availability |
|
|
230
|
+
| `nearline` | Nearline Storage |
|
|
231
|
+
| `coldline` | Coldline Storage |
|
|
232
|
+
| `multi_regional` | Multi-Regional Storage |
|
|
233
|
+
| `regional` | Regional Storage |
|
|
234
|
+
| `standard` | Standard Storage |
|
|
231
235
|
|
|
232
|
-
|
|
236
|
+
Default is `nil`. See the [GCS storage classes documentation](https://cloud.google.com/storage/docs/storage-classes).
|
|
233
237
|
|
|
234
|
-
|
|
238
|
+
#### encryption_key
|
|
235
239
|
|
|
236
|
-
|
|
237
|
-
granted, it will result in an unrecoverable error. Usage of `%{hex_random}` is
|
|
238
|
-
recommended.
|
|
240
|
+
Customer-supplied AES-256 key for server-side encryption. `encryption_key_sha256` is computed automatically. See [Customer-supplied encryption keys](https://cloud.google.com/storage/docs/encryption#customer-supplied).
|
|
239
241
|
|
|
240
|
-
###
|
|
242
|
+
### Object metadata
|
|
241
243
|
|
|
242
|
-
User
|
|
244
|
+
User-supplied web-safe keys and values that are returned with object requests as `x-goog-meta-*` response headers.
|
|
243
245
|
|
|
244
|
-
```
|
|
246
|
+
```aconf
|
|
245
247
|
<match *>
|
|
246
248
|
@type gcs
|
|
247
249
|
|
|
@@ -256,3 +258,20 @@ User provided web-safe keys and arbitrary string values that will returned with
|
|
|
256
258
|
</object_metadata>
|
|
257
259
|
</match>
|
|
258
260
|
```
|
|
261
|
+
|
|
262
|
+
## Development
|
|
263
|
+
|
|
264
|
+
```shell
|
|
265
|
+
bundle install
|
|
266
|
+
bundle exec rake test
|
|
267
|
+
bundle exec bundler-audit check --update
|
|
268
|
+
gem build fluent-plugin-gcs.gemspec
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
## Author
|
|
272
|
+
|
|
273
|
+
Daichi HIRATA
|
|
274
|
+
|
|
275
|
+
## License
|
|
276
|
+
|
|
277
|
+
Apache License 2.0. See [LICENSE.txt](LICENSE.txt).
|
|
@@ -1,12 +1,20 @@
|
|
|
1
1
|
require "tempfile"
|
|
2
2
|
require "zlib"
|
|
3
|
+
require "open3"
|
|
4
|
+
require "shellwords"
|
|
3
5
|
|
|
4
6
|
module Fluent
|
|
5
7
|
module GCS
|
|
6
|
-
def self.discovered_object_creator(store_as, transcoding: nil)
|
|
8
|
+
def self.discovered_object_creator(store_as, transcoding: nil, command_parameter: nil, log: nil)
|
|
7
9
|
case store_as
|
|
8
10
|
when :gzip
|
|
9
11
|
Fluent::GCS::GZipObjectCreator.new(transcoding)
|
|
12
|
+
when :gzip_command
|
|
13
|
+
Fluent::GCS::GZipCommandObjectCreator.new(
|
|
14
|
+
transcoding: transcoding,
|
|
15
|
+
command_parameter: command_parameter,
|
|
16
|
+
log: log
|
|
17
|
+
)
|
|
10
18
|
when :json
|
|
11
19
|
Fluent::GCS::JSONObjectCreator.new
|
|
12
20
|
when :text
|
|
@@ -65,6 +73,59 @@ module Fluent
|
|
|
65
73
|
end
|
|
66
74
|
end
|
|
67
75
|
|
|
76
|
+
class GZipCommandObjectCreator < ObjectCreator
|
|
77
|
+
def initialize(transcoding:, command_parameter:, log:)
|
|
78
|
+
@transcoding = transcoding
|
|
79
|
+
@command_parameter = command_parameter || ""
|
|
80
|
+
@log = log
|
|
81
|
+
check_gzip_command
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def content_type
|
|
85
|
+
@transcoding ? "text/plain" : "application/gzip"
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def content_encoding
|
|
89
|
+
@transcoding ? "gzip" : nil
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def file_extension
|
|
93
|
+
"gz"
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def write(chunk, io)
|
|
97
|
+
cmd = ["gzip", *@command_parameter.shellsplit, "-c"]
|
|
98
|
+
status = Open3.pipeline_w(cmd, out: io.path) do |stdin, wait_thrs|
|
|
99
|
+
chunk.write_to(stdin)
|
|
100
|
+
stdin.close
|
|
101
|
+
wait_thrs.last.value
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
unless status.success?
|
|
105
|
+
@log&.warn("failed to execute gzip command. Fallback to GzipWriter. status = #{status}")
|
|
106
|
+
io.truncate(0)
|
|
107
|
+
io.rewind
|
|
108
|
+
fallback_to_gzip_writer(chunk, io)
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
private
|
|
113
|
+
|
|
114
|
+
def check_gzip_command
|
|
115
|
+
begin
|
|
116
|
+
Open3.capture3("gzip -V")
|
|
117
|
+
rescue Errno::ENOENT
|
|
118
|
+
raise Fluent::ConfigError, "'gzip' utility must be in PATH for gzip_command compression"
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def fallback_to_gzip_writer(chunk, io)
|
|
123
|
+
writer = Zlib::GzipWriter.new(io)
|
|
124
|
+
chunk.write_to(writer)
|
|
125
|
+
writer.finish
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
|
|
68
129
|
class TextObjectCreator < ObjectCreator
|
|
69
130
|
def content_type
|
|
70
131
|
"text/plain"
|
|
@@ -22,6 +22,8 @@ module Fluent::Plugin
|
|
|
22
22
|
desc: "Project identifier for GCS"
|
|
23
23
|
config_param :keyfile, :string, default: nil,
|
|
24
24
|
desc: "Path of GCS service account credentials JSON file"
|
|
25
|
+
config_param :credentials_json, :hash, default: nil, secret: true,
|
|
26
|
+
desc: "GCS service account credentials in JSON format"
|
|
25
27
|
config_param :client_retries, :integer, default: nil,
|
|
26
28
|
desc: "Number of times to retry requests on server error"
|
|
27
29
|
config_param :client_timeout, :integer, default: nil,
|
|
@@ -32,10 +34,12 @@ module Fluent::Plugin
|
|
|
32
34
|
desc: "Format of GCS object keys"
|
|
33
35
|
config_param :path, :string, default: "",
|
|
34
36
|
desc: "Path prefix of the files on GCS"
|
|
35
|
-
config_param :store_as, :enum, list: %i(gzip json text), default: :gzip,
|
|
37
|
+
config_param :store_as, :enum, list: %i(gzip gzip_command json text), default: :gzip,
|
|
36
38
|
desc: "Archive format on GCS"
|
|
37
39
|
config_param :transcoding, :bool, default: false,
|
|
38
40
|
desc: "Enable the decompressive form of transcoding"
|
|
41
|
+
config_param :gzip_command_parameter, :string, default: "",
|
|
42
|
+
desc: "Additional parameters for gzip command (e.g. '-1' for fast compression)"
|
|
39
43
|
config_param :auto_create_bucket, :bool, default: true,
|
|
40
44
|
desc: "Create GCS bucket if it does not exists"
|
|
41
45
|
config_param :hex_random_length, :integer, default: 4,
|
|
@@ -89,17 +93,28 @@ module Fluent::Plugin
|
|
|
89
93
|
|
|
90
94
|
@formatter = formatter_create
|
|
91
95
|
|
|
92
|
-
@object_creator = Fluent::GCS.discovered_object_creator(
|
|
96
|
+
@object_creator = Fluent::GCS.discovered_object_creator(
|
|
97
|
+
@store_as,
|
|
98
|
+
transcoding: @transcoding,
|
|
99
|
+
command_parameter: @gzip_command_parameter,
|
|
100
|
+
log: log
|
|
101
|
+
)
|
|
93
102
|
# For backward compatibility
|
|
94
103
|
# TODO: Remove time_slice_format when end of support compat_parameters
|
|
95
104
|
@configured_time_slice_format = conf['time_slice_format']
|
|
96
105
|
@time_slice_with_tz = Fluent::Timezone.formatter(@timekey_zone, @configured_time_slice_format || timekey_to_timeformat(@buffer_config['timekey']))
|
|
106
|
+
|
|
107
|
+
if @credentials_json
|
|
108
|
+
@credentials = @credentials_json
|
|
109
|
+
else
|
|
110
|
+
@credentials = keyfile
|
|
111
|
+
end
|
|
97
112
|
end
|
|
98
113
|
|
|
99
114
|
def start
|
|
100
115
|
@gcs = Google::Cloud::Storage.new(
|
|
101
116
|
project: @project,
|
|
102
|
-
keyfile: @
|
|
117
|
+
keyfile: @credentials,
|
|
103
118
|
retries: @client_retries,
|
|
104
119
|
timeout: @client_timeout
|
|
105
120
|
)
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-gcs
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.4.
|
|
4
|
+
version: 0.4.5
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Daichi HIRATA
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2026-05-26 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: fluentd
|
|
@@ -51,9 +51,9 @@ executables: []
|
|
|
51
51
|
extensions: []
|
|
52
52
|
extra_rdoc_files: []
|
|
53
53
|
files:
|
|
54
|
-
- ".github/
|
|
54
|
+
- ".github/dependabot.yml"
|
|
55
|
+
- ".github/workflows/test.yml"
|
|
55
56
|
- ".gitignore"
|
|
56
|
-
- CHANGELOG.md
|
|
57
57
|
- Gemfile
|
|
58
58
|
- LICENSE.txt
|
|
59
59
|
- README.md
|
|
@@ -83,7 +83,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
83
83
|
- !ruby/object:Gem::Version
|
|
84
84
|
version: '0'
|
|
85
85
|
requirements: []
|
|
86
|
-
rubygems_version: 3.
|
|
86
|
+
rubygems_version: 3.5.22
|
|
87
87
|
signing_key:
|
|
88
88
|
specification_version: 4
|
|
89
89
|
summary: Google Cloud Storage output plugin for Fluentd
|
data/.github/workflows/test.yaml
DELETED
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
name: Test
|
|
2
|
-
|
|
3
|
-
on:
|
|
4
|
-
push:
|
|
5
|
-
branches: [ master ]
|
|
6
|
-
pull_request:
|
|
7
|
-
branches: [ master ]
|
|
8
|
-
|
|
9
|
-
jobs:
|
|
10
|
-
test:
|
|
11
|
-
runs-on: ubuntu-latest
|
|
12
|
-
strategy:
|
|
13
|
-
fail-fast: false
|
|
14
|
-
matrix:
|
|
15
|
-
ruby-version: ['3.1', '3.2']
|
|
16
|
-
steps:
|
|
17
|
-
- uses: actions/checkout@v4
|
|
18
|
-
- name: Set up Ruby
|
|
19
|
-
uses: ruby/setup-ruby@v1
|
|
20
|
-
with:
|
|
21
|
-
ruby-version: ${{ matrix.ruby-version }}
|
|
22
|
-
bundler-cache: true
|
|
23
|
-
- name: Run tests
|
|
24
|
-
run: bundle exec rake test
|
data/CHANGELOG.md
DELETED
|
@@ -1,46 +0,0 @@
|
|
|
1
|
-
## [Unreleased]
|
|
2
|
-
|
|
3
|
-
New features / Enhancements
|
|
4
|
-
|
|
5
|
-
## [0.4.2] - 2022/08/16
|
|
6
|
-
|
|
7
|
-
Bug fixes
|
|
8
|
-
|
|
9
|
-
- [Fix automatic conversion from a hash to keyword arguments](https://github.com/daichirata/fluent-plugin-gcs/pull/22)
|
|
10
|
-
|
|
11
|
-
## [0.4.1] - 2020/04/17
|
|
12
|
-
|
|
13
|
-
New features
|
|
14
|
-
- [Support blind write to GSC](https://github.com/daichirata/fluent-plugin-gcs/pull/14)
|
|
15
|
-
|
|
16
|
-
## [0.4.0] - 2019/04/01
|
|
17
|
-
|
|
18
|
-
New features / Enhancements
|
|
19
|
-
|
|
20
|
-
- [Support v0.14 (by @cosmo0920)](https://github.com/daichirata/fluent-plugin-gcs/pull/6)
|
|
21
|
-
|
|
22
|
-
## [0.3.0] - 2017/02/28
|
|
23
|
-
|
|
24
|
-
New features / Enhancements
|
|
25
|
-
|
|
26
|
-
- [Add support for setting a File's storage_class on file creation](https://github.com/daichirata/fluent-plugin-gcs/pull/4)
|
|
27
|
-
- see also https://cloud.google.com/storage/docs/storage-classes
|
|
28
|
-
|
|
29
|
-
## [0.2.0] - 2017/01/16
|
|
30
|
-
|
|
31
|
-
Bug fixes
|
|
32
|
-
|
|
33
|
-
- [Remove encryption_key_sha256 parameter.](https://github.com/daichirata/fluent-plugin-gcs/pull/2)
|
|
34
|
-
- see also. https://github.com/GoogleCloudPlatform/google-cloud-ruby/blob/master/google-cloud-storage/CHANGELOG.md#0230--2016-12-8
|
|
35
|
-
|
|
36
|
-
## [0.1.1] - 2016/11/28
|
|
37
|
-
|
|
38
|
-
New features / Enhancements
|
|
39
|
-
|
|
40
|
-
- Add support for `%{hostname}` of object_key_format
|
|
41
|
-
|
|
42
|
-
[Unreleased]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.4.0...HEAD
|
|
43
|
-
[0.4.0]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.3.0...v0.4.0
|
|
44
|
-
[0.3.0]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.2.0...v0.3.0
|
|
45
|
-
[0.2.0]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.1.0...v0.2.0
|
|
46
|
-
[0.1.1]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.1.0...v0.1.1
|