fluent-plugin-gcloud-pubsub-universe-domain 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.github/workflows/release.yml +35 -0
- data/CHANGELOG.md +168 -0
- data/Gemfile +8 -0
- data/LICENSE +22 -0
- data/README.md +225 -0
- data/Rakefile +13 -0
- data/fluent-plugin-gcloud-pubsub-universe-domain.gemspec +25 -0
- data/fluent.conf +42 -0
- data/lib/fluent/plugin/gcloud_pubsub/client.rb +91 -0
- data/lib/fluent/plugin/in_gcloud_pubsub.rb +273 -0
- data/lib/fluent/plugin/out_gcloud_pubsub.rb +161 -0
- data/test/helper.rb +23 -0
- data/test/plugin/test_in_gcloud_pubsub.rb +424 -0
- data/test/plugin/test_out_gcloud_pubsub.rb +375 -0
- metadata +148 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 2fef25d22bbb3dd746e26ff57f1415db29900b1c1779a553b76aabd33aef2b25
|
|
4
|
+
data.tar.gz: 75de1bd3768067cffa2cd1a857c85b912f93f7c0b55cec10cee5d9090ed74a1e
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: 1656a98b6e2825370f23bc94f1207132d4e7e1859da12db0d844a4c14ec39b400d7f6994c7c37de949b98d18fc90466972dd571e59fa5e22ae2c67087ae872e5
|
|
7
|
+
data.tar.gz: 70ef22d6cdf2a7dc44f4fbbeee67600f6a289f0a56801d1fdfc29e3ba41bcb73ffae7d850c8541830718e5bd10965e3c718e295b4abcf123beb7579167ebc324
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
name: Build and publish gem to RubyGems.org
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags:
|
|
6
|
+
- 'v*'
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
build-and-publish:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
permissions:
|
|
12
|
+
contents: read
|
|
13
|
+
steps:
|
|
14
|
+
- name: Checkout repository
|
|
15
|
+
uses: actions/checkout@v4
|
|
16
|
+
|
|
17
|
+
- name: Set up Ruby
|
|
18
|
+
uses: ruby/setup-ruby@v1
|
|
19
|
+
with:
|
|
20
|
+
ruby-version: '3.2'
|
|
21
|
+
bundler-cache: true
|
|
22
|
+
|
|
23
|
+
- name: Install dependencies
|
|
24
|
+
run: bundle install
|
|
25
|
+
|
|
26
|
+
- name: Run tests
|
|
27
|
+
run: bundle exec rake test
|
|
28
|
+
|
|
29
|
+
- name: Build gem
|
|
30
|
+
run: gem build fluent-plugin-gcloud-pubsub-universe-domain.gemspec
|
|
31
|
+
|
|
32
|
+
- name: Publish gem to RubyGems.org
|
|
33
|
+
env:
|
|
34
|
+
GEM_HOST_API_KEY: ${{ secrets.RUBYGEMS_API_KEY }}
|
|
35
|
+
run: gem push fluent-plugin-gcloud-pubsub-universe-domain-*.gem
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
## ChangeLog
|
|
2
|
+
|
|
3
|
+
### Release 2.0.0 - 2026/10/15
|
|
4
|
+
|
|
5
|
+
- Bump up google-cloud-pubsub to v3.4.0
|
|
6
|
+
- `Topic` and `Subscription` are now `Publisher` and `Subscriber` (dedicated objects for sending and receiving messages)
|
|
7
|
+
- Legacy resource management API removed; topic creation uses the topic admin client
|
|
8
|
+
- Output plugin
|
|
9
|
+
- Add `universe_domain` parameter
|
|
10
|
+
- Input plugin
|
|
11
|
+
- Add `universe_domain` parameter
|
|
12
|
+
|
|
13
|
+
### Release 1.6.0 - 2021/06/17
|
|
14
|
+
|
|
15
|
+
- Output plugin
|
|
16
|
+
- Add `timeout` feature
|
|
17
|
+
- Add `attribute_key_values` feature
|
|
18
|
+
|
|
19
|
+
### Release 1.5.0 - 2021/01/26
|
|
20
|
+
|
|
21
|
+
- Bump up google-cloud-pubsub to v2.3.x
|
|
22
|
+
- Add `endpoint` parameter
|
|
23
|
+
- Add gzip compression/decompression feature
|
|
24
|
+
|
|
25
|
+
### Release 1.4.0 - 2020/08/11
|
|
26
|
+
|
|
27
|
+
- Bump up google-cloud-pubsub to v0.39.x
|
|
28
|
+
- In preparation for updating to v1.x
|
|
29
|
+
- Output plugin
|
|
30
|
+
- Add `dest_project` parameter
|
|
31
|
+
|
|
32
|
+
### Release 1.3.2 - 2019/08/16
|
|
33
|
+
|
|
34
|
+
- Input plugin
|
|
35
|
+
- Make `topic` parameter optional
|
|
36
|
+
|
|
37
|
+
### Release 1.3.1 - 2018/06/08
|
|
38
|
+
|
|
39
|
+
- Input plugin
|
|
40
|
+
- Dumps raw string to logs for parsing errors
|
|
41
|
+
|
|
42
|
+
### Release 1.3.0 - 2018/05/18
|
|
43
|
+
|
|
44
|
+
- Make attributes available
|
|
45
|
+
|
|
46
|
+
### Release 1.2.0 - 2018/04/24
|
|
47
|
+
|
|
48
|
+
- Output plugin
|
|
49
|
+
- Support inject section
|
|
50
|
+
|
|
51
|
+
### Release 1.1.0 - 2018/03/30
|
|
52
|
+
|
|
53
|
+
- Output plugin
|
|
54
|
+
- The placeholder is now available in topic param
|
|
55
|
+
|
|
56
|
+
### Release 1.0.3 - 2018/03/29
|
|
57
|
+
|
|
58
|
+
- Bump up google-cloud-pubsub to v0.30.x
|
|
59
|
+
- Make config file Fluentd v1 compatible
|
|
60
|
+
|
|
61
|
+
### Release 1.0.2 - 2017/09/11
|
|
62
|
+
|
|
63
|
+
- Bump up google-cloud-pubsub to v0.27.x
|
|
64
|
+
|
|
65
|
+
### Release 1.0.1 - 2017/09/05
|
|
66
|
+
|
|
67
|
+
- Bump up google-cloud-pubsub to v0.26.x
|
|
68
|
+
|
|
69
|
+
### Release 1.0.0 - 2017/06/23
|
|
70
|
+
|
|
71
|
+
- Fluentd v0.14 ready
|
|
72
|
+
- Fluentd v0.12 is not supported in the later version
|
|
73
|
+
|
|
74
|
+
### Release 0.4.6 - 2017/05/14
|
|
75
|
+
|
|
76
|
+
- Output plugin
|
|
77
|
+
- Make messages exceeding configured size not be published because Pub/Sub clients cannot receive it
|
|
78
|
+
|
|
79
|
+
### Release 0.4.5 - 2017/04/02
|
|
80
|
+
|
|
81
|
+
- Bump up google-cloud-pubsub to v0.24.x
|
|
82
|
+
|
|
83
|
+
### Release 0.4.4 - 2017/03/07
|
|
84
|
+
|
|
85
|
+
- Bump up google-cloud-pubsub to v0.23.x
|
|
86
|
+
|
|
87
|
+
### Release 0.4.3 - 2017/02/16
|
|
88
|
+
|
|
89
|
+
- Input plugin
|
|
90
|
+
- Add "status" method to the http rpc api
|
|
91
|
+
|
|
92
|
+
### Release 0.4.2 - 2017/02/03
|
|
93
|
+
|
|
94
|
+
- Make retry to get topic/subscription when Pub/Sub API returns 50x code
|
|
95
|
+
|
|
96
|
+
### Release 0.4.1 - 2017/02/02
|
|
97
|
+
|
|
98
|
+
- Bump up google-cloud-pubsub to v0.22.x
|
|
99
|
+
- Input plugin
|
|
100
|
+
- Rescue 50x errors on acknowledge api
|
|
101
|
+
|
|
102
|
+
### Release 0.4.0 - 2017/01/21
|
|
103
|
+
|
|
104
|
+
- Input plugin
|
|
105
|
+
- Add feature to use record key as tag
|
|
106
|
+
|
|
107
|
+
### Release 0.3.4 - 2017/01/03
|
|
108
|
+
|
|
109
|
+
- Output plugin
|
|
110
|
+
- Rescue 50x errors
|
|
111
|
+
- Input plugin
|
|
112
|
+
- Guard emit to be called with multi-threading
|
|
113
|
+
- Rescue 50x errors
|
|
114
|
+
- Enabled to select whether to raise an exception if message processing failed
|
|
115
|
+
|
|
116
|
+
### Release 0.3.3 - 2016/12/03
|
|
117
|
+
|
|
118
|
+
- Input plugin
|
|
119
|
+
- Fix undefined variable error
|
|
120
|
+
|
|
121
|
+
### Release 0.3.2 - 2016/11/13
|
|
122
|
+
|
|
123
|
+
- Add plugin param desc
|
|
124
|
+
- Input plugin
|
|
125
|
+
- Improve handling to acknowledge messages
|
|
126
|
+
|
|
127
|
+
### Release 0.3.1 - 2016/11/03
|
|
128
|
+
|
|
129
|
+
- Output plugin
|
|
130
|
+
- Improve error handling
|
|
131
|
+
|
|
132
|
+
### Release 0.3.0 - 2016/10/30
|
|
133
|
+
|
|
134
|
+
- Bump up google-cloud-pubsub to v0.21
|
|
135
|
+
- Input plugin
|
|
136
|
+
- Add multithreaded pulling feature
|
|
137
|
+
|
|
138
|
+
### Release 0.2.0 - 2016/10/15
|
|
139
|
+
|
|
140
|
+
- Input plugin
|
|
141
|
+
- Add HTTP RPC feature
|
|
142
|
+
|
|
143
|
+
### Release 0.1.4 - 2016/09/19
|
|
144
|
+
|
|
145
|
+
- Input plugin
|
|
146
|
+
- `pull_interval` can be specified float value
|
|
147
|
+
- `topic` must be specified
|
|
148
|
+
|
|
149
|
+
### Release 0.1.3 - 2016/09/17
|
|
150
|
+
|
|
151
|
+
- Input plugin
|
|
152
|
+
- Fix error handling and add debug logging
|
|
153
|
+
|
|
154
|
+
### Release 0.1.2 - 2016/09/11
|
|
155
|
+
|
|
156
|
+
- Output plugin
|
|
157
|
+
- Change default max message size and add debug message
|
|
158
|
+
|
|
159
|
+
### Release 0.1.1 - 2016/08/27
|
|
160
|
+
|
|
161
|
+
- Bump up google-cloud-pubsub (gcloud-ruby) to 0.20
|
|
162
|
+
|
|
163
|
+
### Release 0.1.0 - 2016/08/22
|
|
164
|
+
|
|
165
|
+
- Use formatter / parser plugin and add format configuration
|
|
166
|
+
- Bump up gcloud-ruby to 0.12
|
|
167
|
+
- Remove dependency on lightening buffer
|
|
168
|
+
- Fix error caused by Pub/Sub quotas
|
data/Gemfile
ADDED
data/LICENSE
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2015 mdoi
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
22
|
+
|
data/README.md
ADDED
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
# fluent-plugin-gcloud-pubsub-universe-domain
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
[](http://badge.fury.io/rb/fluent-plugin-gcloud-pubsub-universe-domain)
|
|
5
|
+
|
|
6
|
+
This plugin is forked from https://github.com/mdoi/fluent-plugin-gcloud-pubsub
|
|
7
|
+
|
|
8
|
+
## Overview
|
|
9
|
+
|
|
10
|
+
[Google Cloud Pub/Sub](https://cloud.google.com/pubsub/) Input/Output(BufferedOutput) plugin for [Fluentd](http://www.fluentd.org/) with [google-cloud](https://googlecloudplatform.github.io/google-cloud-ruby/) gem
|
|
11
|
+
|
|
12
|
+
- Publish messages to Google Cloud Pub/Sub
|
|
13
|
+
- Pull messages from Google Cloud Pub/Sub
|
|
14
|
+
|
|
15
|
+
## Preparation
|
|
16
|
+
|
|
17
|
+
- Create a project on Google Developer Console
|
|
18
|
+
- Add a topic of Cloud Pub/Sub to the project
|
|
19
|
+
- Add a pull style subscription to the topic
|
|
20
|
+
- Download your credential (json) or [set scope on GCE instance](https://cloud.google.com/compute/docs/api/how-tos/authorization)
|
|
21
|
+
|
|
22
|
+
When using output plugin, you need to grant Pub/Sub Publisher and Pub/Sub Viewer role to IAM.
|
|
23
|
+
|
|
24
|
+
Also, when using input plugin, you need to grant Pub/Sub Subscriber and Pub/Sub Viewer role to IAM.
|
|
25
|
+
|
|
26
|
+
## Requirements
|
|
27
|
+
|
|
28
|
+
| fluent-plugin-gcloud-pubsub-universe-domain | fluentd | ruby |
|
|
29
|
+
|------------------------|---------|------|
|
|
30
|
+
| >= 2.0.0 | >= v0.14.0 | >= 3.2 |
|
|
31
|
+
| >= 1.0.0 | >= v0.14.0 | >= 2.1 |
|
|
32
|
+
| < 1.0.0 | >= v0.12.0 | >= 1.9 |
|
|
33
|
+
|
|
34
|
+
## Installation
|
|
35
|
+
|
|
36
|
+
Install by gem:
|
|
37
|
+
|
|
38
|
+
```shell
|
|
39
|
+
$ gem install fluent-plugin-gcloud-pubsub-universe-domain
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
**Caution**
|
|
43
|
+
|
|
44
|
+
This plugin doesn't work in [td-agent](http://docs.fluentd.org/articles/install-by-rpm).
|
|
45
|
+
|
|
46
|
+
Please use in [Fluentd installed by gem](http://docs.fluentd.org/articles/install-by-gem).
|
|
47
|
+
|
|
48
|
+
## Configuration
|
|
49
|
+
|
|
50
|
+
### Publish messages
|
|
51
|
+
|
|
52
|
+
Use `gcloud_pubsub` output plugin.
|
|
53
|
+
|
|
54
|
+
```
|
|
55
|
+
<match example.publish>
|
|
56
|
+
@type gcloud_pubsub
|
|
57
|
+
project <YOUR PROJECT>
|
|
58
|
+
key <YOUR KEY>
|
|
59
|
+
topic <YOUR TOPIC>
|
|
60
|
+
autocreate_topic false
|
|
61
|
+
max_messages 1000
|
|
62
|
+
max_total_size 9800000
|
|
63
|
+
max_message_size 4000000
|
|
64
|
+
endpoint <Endpoint URL>
|
|
65
|
+
emulator_host <PUBSUB_EMULATOR_HOST>
|
|
66
|
+
timeout 60
|
|
67
|
+
<buffer>
|
|
68
|
+
@type memory
|
|
69
|
+
flush_interval 1s
|
|
70
|
+
</buffer>
|
|
71
|
+
<format>
|
|
72
|
+
@type json
|
|
73
|
+
</format>
|
|
74
|
+
</match>
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
- `project` (optional)
|
|
78
|
+
- Set your GCP project.
|
|
79
|
+
- Running fluentd on GCP, you don't have to specify.
|
|
80
|
+
- You can also use environment variable such as `GCLOUD_PROJECT`.
|
|
81
|
+
- `key` (optional)
|
|
82
|
+
- Set your credential file path.
|
|
83
|
+
- Running fluentd on GCP, you can use scope instead of specifying this.
|
|
84
|
+
- You can also use environment variable such as `GCLOUD_KEYFILE`.
|
|
85
|
+
- `topic` (required)
|
|
86
|
+
- Set topic name to publish.
|
|
87
|
+
- You can use placeholder in this param. See: https://docs.fluentd.org/v1.0/articles/buffer-section
|
|
88
|
+
- `dest_project` (optional, default: `nil`)
|
|
89
|
+
- Set your destination GCP project if you publish messages cross project.
|
|
90
|
+
- `autocreate_topic` (optional, default: `false`)
|
|
91
|
+
- If set to `true`, specified topic will be created when it doesn't exist.
|
|
92
|
+
- `max_messages` (optional, default: `1000`)
|
|
93
|
+
- Publishing messages count per request to Cloud Pub/Sub.
|
|
94
|
+
- See https://cloud.google.com/pubsub/quotas#other_limits
|
|
95
|
+
- `max_total_size` (optional, default: `9800000` = `9.8MB`)
|
|
96
|
+
- Publishing messages bytesize per request to Cloud Pub/Sub. This parameter affects only message size. You should specify a little smaller value than quota.
|
|
97
|
+
- See https://cloud.google.com/pubsub/quotas#other_limits
|
|
98
|
+
- `max_message_size` (optional, default: `4000000` = `4MB`)
|
|
99
|
+
- Messages exceeding `max_message_size` are not published because Pub/Sub clients cannot receive it.
|
|
100
|
+
- `attribute_keys` (optional, default: `[]`)
|
|
101
|
+
- Publishing the set fields as attributes generated from input message.
|
|
102
|
+
- `attribute_key_values` (optional, default: `{}`)
|
|
103
|
+
- Publishing the set fields as attributes generated from input params
|
|
104
|
+
- `endpoint`(optional)
|
|
105
|
+
- Set Pub/Sub service endpoint. For more information, see [Service Endpoints](https://cloud.google.com/pubsub/docs/reference/service_apis_overview#service_endpoints)
|
|
106
|
+
- `compression` (optional, default: `nil`)
|
|
107
|
+
- If set to `gzip`, messages will be compressed with gzip.
|
|
108
|
+
- `emulator_host`(optional)
|
|
109
|
+
- Host name of the emulator. For more information, see [Testing apps locally with the emulator](https://cloud.google.com/pubsub/docs/emulator#accessing_environment_variables)
|
|
110
|
+
- `timeout` (optional)
|
|
111
|
+
- Set default timeout to use in publish requests.
|
|
112
|
+
- `universe_domain` (optional)
|
|
113
|
+
- Set universe domain for the Cloud Pub/Sub client. See google-cloud-pubsub v3.x.
|
|
114
|
+
|
|
115
|
+
### Pull messages
|
|
116
|
+
|
|
117
|
+
Use `gcloud_pubsub` input plugin.
|
|
118
|
+
|
|
119
|
+
```
|
|
120
|
+
<source>
|
|
121
|
+
@type gcloud_pubsub
|
|
122
|
+
tag example.pull
|
|
123
|
+
project <YOUR PROJECT>
|
|
124
|
+
key <YOUR KEY>
|
|
125
|
+
topic <YOUR TOPIC>
|
|
126
|
+
subscription <YOUR SUBSCRIPTION>
|
|
127
|
+
max_messages 1000
|
|
128
|
+
return_immediately true
|
|
129
|
+
pull_interval 0.5
|
|
130
|
+
pull_threads 2
|
|
131
|
+
parse_error_action exception
|
|
132
|
+
enable_rpc true
|
|
133
|
+
rpc_bind 0.0.0.0
|
|
134
|
+
rpc_port 24680
|
|
135
|
+
<parse>
|
|
136
|
+
@type json
|
|
137
|
+
</parse>
|
|
138
|
+
</source>
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
- `tag` (required)
|
|
142
|
+
- Set tag of messages.
|
|
143
|
+
- If `tag_key` is specified, `tag` is used as tag when record don't have specified key.
|
|
144
|
+
- `tag_key` (optional)
|
|
145
|
+
- Set key to be used as tag.
|
|
146
|
+
- `project` (optional)
|
|
147
|
+
- Set your GCP project
|
|
148
|
+
- Running fluentd on GCP, you don't have to specify.
|
|
149
|
+
- You can also use environment variable such as `GCLOUD_PROJECT`.
|
|
150
|
+
- `key` (optional)
|
|
151
|
+
- Set your credential file path.
|
|
152
|
+
- Running fluentd on GCP, you can use scope instead of specifying this.
|
|
153
|
+
- You can also use environment variable such as `GCLOUD_KEYFILE`.
|
|
154
|
+
- `topic` (optional)
|
|
155
|
+
- Set topic name that the subscription belongs to.
|
|
156
|
+
- `subscription` (required)
|
|
157
|
+
- Set subscription name to pull.
|
|
158
|
+
- `max_messages` (optional, default: `100`)
|
|
159
|
+
- See maxMessages on https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions/pull
|
|
160
|
+
- `return_immediately` (optional, default: `true`)
|
|
161
|
+
- See returnImmediately on https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions/pull
|
|
162
|
+
- If `return_immediately` is `true` or pulling message is stopped by HTTP RPC, this plugin wait `pull_interval` each pull.
|
|
163
|
+
- `pull_interval` (optional, default: `5.0`)
|
|
164
|
+
- Pulling messages by intervals of specified seconds.
|
|
165
|
+
- `pull_threads` (optional, default: `1`)
|
|
166
|
+
- Set number of threads to pull messages.
|
|
167
|
+
- `attribute_keys` (optional, default: `[]`)
|
|
168
|
+
- Specify the key of the attribute to be emitted as the field of record.
|
|
169
|
+
- `parse_error_action` (optional, default: `exception`)
|
|
170
|
+
- Set error type when parsing messages fails.
|
|
171
|
+
- `exception`: Raise exception. Messages are not acknowledged.
|
|
172
|
+
- `warning`: Only logging as warning.
|
|
173
|
+
- `enable_rpc` (optional, default: `false`)
|
|
174
|
+
- If `true` is specified, HTTP RPC to stop or start pulling message is enabled.
|
|
175
|
+
- `rpc_bind` (optional, default: `0.0.0.0`)
|
|
176
|
+
- Bind IP address for HTTP RPC.
|
|
177
|
+
- `rpc_port` (optional, default: `24680`)
|
|
178
|
+
- Port for HTTP RPC.
|
|
179
|
+
- `decompression` (optional, default: `nil`)
|
|
180
|
+
- If set to `gzip`, messages will be decompressed with gzip.
|
|
181
|
+
- `universe_domain` (optional)
|
|
182
|
+
- Set universe domain for the Cloud Pub/Sub client. See google-cloud-pubsub v3.x.
|
|
183
|
+
|
|
184
|
+
## Contributing
|
|
185
|
+
|
|
186
|
+
1. Fork it
|
|
187
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
|
188
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
|
189
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
|
190
|
+
5. Create a new Pull Request
|
|
191
|
+
|
|
192
|
+
## TODO
|
|
193
|
+
|
|
194
|
+
- Add `tag` attribute in output plugin and use `tag` attribute as tag in input plugin.
|
|
195
|
+
- Send ack after other output plugin committed (if possible).
|
|
196
|
+
|
|
197
|
+
## Authors
|
|
198
|
+
|
|
199
|
+
- [@mdoi](https://github.com/mdoi) : First author
|
|
200
|
+
- [@mia-0032](https://github.com/mia-0032) : Original fork author
|
|
201
|
+
- [Théo Berthier](https://github.com/theo-berthier) : Current maintainer
|
|
202
|
+
|
|
203
|
+
## Major Changes
|
|
204
|
+
|
|
205
|
+
### v2.0.0 (current)
|
|
206
|
+
|
|
207
|
+
- **Fork & maintenance** by Théo Berthier (originally from `mdo i/fluent-plugin-gcloud-pubsub`)
|
|
208
|
+
- **Ruby 3.2+ support** (tested with Ruby 4.0.7)
|
|
209
|
+
- **Fluentd 1.x support** (gem dependency `< 2`)
|
|
210
|
+
- **Output plugin fixes**:
|
|
211
|
+
- Fixed `ASCII-8BIT` encoding issue when publishing multibyte strings (e.g. Japanese/Chinese characters) via JSON formatter
|
|
212
|
+
- Fixed `autocreate_topic` path: `topic_path` is now called without `project_name:` when `dest_project` is nil
|
|
213
|
+
- Fixed mock-based unit tests compatibility with `rr` under Ruby 4.x
|
|
214
|
+
- **Docker support**: Added `Dockerfile`, `docker-compose.yml`, `fluent.conf`, `fluent.conf.sub` and build script for containerized Fluentd deployment
|
|
215
|
+
- **CI/CD**: GitHub Actions workflow to build and push the gem to RubyGems.org on version tag
|
|
216
|
+
|
|
217
|
+
### v1.0.0
|
|
218
|
+
|
|
219
|
+
- Initial fork from `mdo i/fluent-plugin-gcloud-pubsub`
|
|
220
|
+
- Basic Pub/Sub publish and subscribe support
|
|
221
|
+
|
|
222
|
+
## TODO
|
|
223
|
+
|
|
224
|
+
- Add `tag` attribute in output plugin and use `tag` attribute as tag in input plugin.
|
|
225
|
+
- Send ack after other output plugin committed (if possible).
|
data/Rakefile
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
require 'bundler'
|
|
2
|
+
Bundler::GemHelper.install_tasks
|
|
3
|
+
|
|
4
|
+
require 'rake/testtask'
|
|
5
|
+
|
|
6
|
+
Rake::TestTask.new(:test) do |test|
|
|
7
|
+
test.libs << 'lib' << 'test'
|
|
8
|
+
test.test_files = FileList['test/plugin/test_*.rb']
|
|
9
|
+
test.verbose = true
|
|
10
|
+
test.warning = false
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
task :default => [:build]
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# encoding: utf-8
|
|
2
|
+
$:.push File.expand_path('../lib', __FILE__)
|
|
3
|
+
|
|
4
|
+
Gem::Specification.new do |gem|
|
|
5
|
+
gem.name = "fluent-plugin-gcloud-pubsub-universe-domain"
|
|
6
|
+
gem.description = "Google Cloud Pub/Sub input/output plugin for Fluentd event collector with universe_domain parameter"
|
|
7
|
+
gem.license = "MIT"
|
|
8
|
+
gem.homepage = "https://github.com/theoberthier/fluent-plugin-gcloud-pubsub-custom"
|
|
9
|
+
gem.summary = gem.description
|
|
10
|
+
gem.version = "2.0.0"
|
|
11
|
+
gem.authors = ["Théo Berthier"]
|
|
12
|
+
gem.email = "berthiertheo@hotmail.fr"
|
|
13
|
+
gem.files = `git ls-files`.split("\n")
|
|
14
|
+
gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
|
15
|
+
gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
|
16
|
+
gem.require_paths = ['lib']
|
|
17
|
+
|
|
18
|
+
gem.add_runtime_dependency "fluentd", [">= 0.14.15", "< 2"]
|
|
19
|
+
gem.add_runtime_dependency "google-cloud-pubsub", "~> 3.4.0"
|
|
20
|
+
|
|
21
|
+
gem.add_development_dependency "bundler"
|
|
22
|
+
gem.add_development_dependency "rake"
|
|
23
|
+
gem.add_development_dependency "test-unit"
|
|
24
|
+
gem.add_development_dependency "test-unit-rr"
|
|
25
|
+
end
|
data/fluent.conf
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
<match **>
|
|
2
|
+
@type gcloud_pubsub
|
|
3
|
+
|
|
4
|
+
# GCP Project
|
|
5
|
+
project exemple
|
|
6
|
+
|
|
7
|
+
# Your Pub/Sub topic
|
|
8
|
+
topic projects/exemple/topics/test
|
|
9
|
+
|
|
10
|
+
# GCP key path; if not provided, Application Default Credentials (ADC) are used
|
|
11
|
+
key /etc/fluent/gcloud-key.json
|
|
12
|
+
|
|
13
|
+
# Autocreate topic if the required roles are provided
|
|
14
|
+
autocreate_topic true
|
|
15
|
+
|
|
16
|
+
# Maximum Pub/Sub request
|
|
17
|
+
max_messages 1000
|
|
18
|
+
|
|
19
|
+
# Maximum request size (bytes)
|
|
20
|
+
max_total_size 9800000
|
|
21
|
+
|
|
22
|
+
# Maximum message size (bytes)
|
|
23
|
+
max_message_size 4000000
|
|
24
|
+
|
|
25
|
+
# Message compression (nil or gzip)
|
|
26
|
+
compression gzip
|
|
27
|
+
|
|
28
|
+
# Key to extract from Pub/Sub
|
|
29
|
+
# attribute_keys env,service
|
|
30
|
+
|
|
31
|
+
# Static values to target (uncomment if necessary)
|
|
32
|
+
# attribute_key_values {"environment": "dev", "service": "fluentd"}
|
|
33
|
+
|
|
34
|
+
# <buffer>
|
|
35
|
+
# @type memory
|
|
36
|
+
# flush_interval 5s
|
|
37
|
+
# chunk_limit_size 1m
|
|
38
|
+
# queue_limit_length 1000
|
|
39
|
+
# retry_type exponential_backoff
|
|
40
|
+
# retry_max_interval 30s
|
|
41
|
+
# retry_forever true
|
|
42
|
+
# </buffer>
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
require 'google/cloud/pubsub'
|
|
2
|
+
|
|
3
|
+
module Fluent
|
|
4
|
+
module GcloudPubSub
|
|
5
|
+
class Error < StandardError
|
|
6
|
+
end
|
|
7
|
+
class RetryableError < Error
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
class Message
|
|
11
|
+
attr_reader :message, :attributes
|
|
12
|
+
def initialize(message, attributes={})
|
|
13
|
+
@message = message
|
|
14
|
+
@attributes = attributes
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def bytesize()
|
|
18
|
+
attr_size = 0
|
|
19
|
+
@attributes.each do |key, val|
|
|
20
|
+
attr_size += key.bytesize + val.bytesize
|
|
21
|
+
end
|
|
22
|
+
@message.bytesize + attr_size
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
class Publisher
|
|
27
|
+
def initialize(project, key, autocreate_topic, dest_project, endpoint, timeout, emulator_host, universe_domain)
|
|
28
|
+
@pubsub = Google::Cloud::Pubsub.new project_id: project, credentials: key, endpoint: endpoint, timeout: timeout, emulator_host: emulator_host, universe_domain: universe_domain
|
|
29
|
+
@autocreate_topic = autocreate_topic
|
|
30
|
+
@dest_project = dest_project
|
|
31
|
+
@topics = {}
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def topic(topic_name)
|
|
35
|
+
return @topics[topic_name] if @topics.key? topic_name
|
|
36
|
+
|
|
37
|
+
begin
|
|
38
|
+
if @dest_project.nil?
|
|
39
|
+
client = @pubsub.publisher topic_name
|
|
40
|
+
else
|
|
41
|
+
client = @pubsub.publisher topic_name, project: @dest_project
|
|
42
|
+
end
|
|
43
|
+
rescue Google::Cloud::NotFoundError
|
|
44
|
+
if @autocreate_topic
|
|
45
|
+
topic_path = if @dest_project.nil?
|
|
46
|
+
@pubsub.topic_path(topic_name)
|
|
47
|
+
else
|
|
48
|
+
@pubsub.topic_path(topic_name, project_name: @dest_project)
|
|
49
|
+
end
|
|
50
|
+
grpc = @pubsub.topic_admin.create_topic topic: topic_path
|
|
51
|
+
client = Publisher.from_grpc grpc, @pubsub.service
|
|
52
|
+
else
|
|
53
|
+
raise Error.new "topic:#{topic_name} does not exist."
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
@topics[topic_name] = client
|
|
58
|
+
client
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def publish(topic_name, messages)
|
|
62
|
+
topic(topic_name).publish do |batch|
|
|
63
|
+
messages.each do |m|
|
|
64
|
+
batch.publish m.message, m.attributes
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
rescue Google::Cloud::UnavailableError, Google::Cloud::DeadlineExceededError, Google::Cloud::InternalError => ex
|
|
68
|
+
raise RetryableError.new "Google api returns error:#{ex.class.to_s} message:#{ex.to_s}"
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
class Subscriber
|
|
73
|
+
def initialize(project, key, topic_name, subscription_name, universe_domain)
|
|
74
|
+
pubsub = Google::Cloud::Pubsub.new project_id: project, credentials: key, universe_domain: universe_domain
|
|
75
|
+
@client = pubsub.subscriber subscription_name
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def pull(immediate, max)
|
|
79
|
+
@client.pull immediate: immediate, max: max
|
|
80
|
+
rescue Google::Cloud::UnavailableError, Google::Cloud::DeadlineExceededError, Google::Cloud::InternalError => ex
|
|
81
|
+
raise RetryableError.new "Google pull api returns error:#{ex.class.to_s} message:#{ex.to_s}"
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def acknowledge(messages)
|
|
85
|
+
@client.acknowledge messages
|
|
86
|
+
rescue Google::Cloud::UnavailableError, Google::Cloud::DeadlineExceededError, Google::Cloud::InternalError => ex
|
|
87
|
+
raise RetryableError.new "Google acknowledge api returns error:#{ex.class.to_s} message:#{ex.to_s}"
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
end
|