fluent-plugin-gcloud-pubsub-custom 1.3.0 → 1.6.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +25 -0
- data/CHANGELOG.md +29 -0
- data/README.md +18 -4
- data/Rakefile +1 -0
- data/fluent-plugin-gcloud-pubsub-custom.gemspec +2 -3
- data/lib/fluent/plugin/gcloud_pubsub/client.rb +16 -7
- data/lib/fluent/plugin/in_gcloud_pubsub.rb +22 -4
- data/lib/fluent/plugin/out_gcloud_pubsub.rb +29 -3
- data/test/{test_helper.rb → helper.rb} +6 -14
- data/test/plugin/test_in_gcloud_pubsub.rb +34 -1
- data/test/plugin/test_out_gcloud_pubsub.rb +123 -1
- metadata +11 -12
- data/.travis.yml +0 -25
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3bd7e53135bef7ce1f7ce3796659b6deb1ebaf6ff303755a84fdd886c4f46bf3
|
4
|
+
data.tar.gz: a04ddb36d5c0bceb7ac1fb8947b537ee0c38b9b26f6b5d5c53bb2ed034deb4e0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f0d6537d00021bb6d849ff7c9c9e8b4b233214826b098db649820a0a310d7d60170a8642db3b7b7acee2f83b70a392ad6a9ef8adbb47f22bc2d1be2022129d03
|
7
|
+
data.tar.gz: 2b742512024102940dd447f6450377684a1834c89582e38d6498957dc6219c5625afa49320f2c6c0139ff73bd7e98ad61085029b0c4794e3418c0324b0346bc0
|
@@ -0,0 +1,25 @@
|
|
1
|
+
name: Test
|
2
|
+
|
3
|
+
on:
|
4
|
+
push:
|
5
|
+
branches: [ master ]
|
6
|
+
pull_request:
|
7
|
+
branches: [ master ]
|
8
|
+
|
9
|
+
jobs:
|
10
|
+
test:
|
11
|
+
runs-on: ubuntu-latest
|
12
|
+
strategy:
|
13
|
+
fail-fast: false
|
14
|
+
matrix:
|
15
|
+
ruby-version: ['2.4', '2.5', '2.6', '2.7']
|
16
|
+
|
17
|
+
steps:
|
18
|
+
- uses: actions/checkout@v2
|
19
|
+
- name: Set up Ruby
|
20
|
+
uses: ruby/setup-ruby@v1
|
21
|
+
with:
|
22
|
+
ruby-version: ${{ matrix.ruby-version }}
|
23
|
+
bundler-cache: true
|
24
|
+
- name: Run tests
|
25
|
+
run: bundle exec rake test
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,34 @@
|
|
1
1
|
## ChangeLog
|
2
2
|
|
3
|
+
### Release 1.6.0 - 2021/06/17
|
4
|
+
|
5
|
+
- Output plugin
|
6
|
+
- Add `timeout` feature
|
7
|
+
- Add `attribute_key_values` feature
|
8
|
+
|
9
|
+
### Release 1.5.0 - 2021/01/26
|
10
|
+
|
11
|
+
- Bump up google-cloud-pubsub to v2.3.x
|
12
|
+
- Add `endpoint` parameter
|
13
|
+
- Add gzip compression/decompression feature
|
14
|
+
|
15
|
+
### Release 1.4.0 - 2020/08/11
|
16
|
+
|
17
|
+
- Bump up google-cloud-pubsub to v0.39.x
|
18
|
+
- In preparation for updating to v1.x
|
19
|
+
- Output plugin
|
20
|
+
- Add `dest_project` parameter
|
21
|
+
|
22
|
+
### Release 1.3.2 - 2019/08/16
|
23
|
+
|
24
|
+
- Input plugin
|
25
|
+
- Make `topic` parameter optional
|
26
|
+
|
27
|
+
### Release 1.3.1 - 2018/06/08
|
28
|
+
|
29
|
+
- Input plugin
|
30
|
+
- Dumps raw string to logs for parsing errors
|
31
|
+
|
3
32
|
### Release 1.3.0 - 2018/05/18
|
4
33
|
|
5
34
|
- Make attributes available
|
data/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# fluent-plugin-gcloud-pubsub-custom
|
2
2
|
|
3
|
-
|
3
|
+
![Test](https://github.com/mia-0032/fluent-plugin-gcloud-pubsub-custom/workflows/Test/badge.svg)
|
4
4
|
[![Gem Version](https://badge.fury.io/rb/fluent-plugin-gcloud-pubsub-custom.svg)](http://badge.fury.io/rb/fluent-plugin-gcloud-pubsub-custom)
|
5
5
|
|
6
6
|
This plugin is forked from https://github.com/mdoi/fluent-plugin-gcloud-pubsub
|
@@ -60,6 +60,8 @@ Use `gcloud_pubsub` output plugin.
|
|
60
60
|
max_messages 1000
|
61
61
|
max_total_size 9800000
|
62
62
|
max_message_size 4000000
|
63
|
+
endpoint <Endpoint URL>
|
64
|
+
timeout 60
|
63
65
|
<buffer>
|
64
66
|
@type memory
|
65
67
|
flush_interval 1s
|
@@ -81,6 +83,8 @@ Use `gcloud_pubsub` output plugin.
|
|
81
83
|
- `topic` (required)
|
82
84
|
- Set topic name to publish.
|
83
85
|
- You can use placeholder in this param. See: https://docs.fluentd.org/v1.0/articles/buffer-section
|
86
|
+
- `dest_project` (optional, default: `nil`)
|
87
|
+
- Set your destination GCP project if you publish messages cross project.
|
84
88
|
- `autocreate_topic` (optional, default: `false`)
|
85
89
|
- If set to `true`, specified topic will be created when it doesn't exist.
|
86
90
|
- `max_messages` (optional, default: `1000`)
|
@@ -92,7 +96,15 @@ Use `gcloud_pubsub` output plugin.
|
|
92
96
|
- `max_message_size` (optional, default: `4000000` = `4MB`)
|
93
97
|
- Messages exceeding `max_message_size` are not published because Pub/Sub clients cannot receive it.
|
94
98
|
- `attribute_keys` (optional, default: `[]`)
|
95
|
-
- Publishing the set fields as attributes.
|
99
|
+
- Publishing the set fields as attributes generated from input message.
|
100
|
+
- `attribute_key_values` (optional, default: `{}`)
|
101
|
+
- Publishing the set fields as attributes generated from input params
|
102
|
+
- `endpoint`(optional)
|
103
|
+
- Set Pub/Sub service endpoint. For more information, see [Service Endpoints](https://cloud.google.com/pubsub/docs/reference/service_apis_overview#service_endpoints)
|
104
|
+
- `compression` (optional, default: `nil`)
|
105
|
+
- If set to `gzip`, messages will be compressed with gzip.
|
106
|
+
- `timeout` (optional)
|
107
|
+
- Set default timeout to use in publish requests.
|
96
108
|
|
97
109
|
### Pull messages
|
98
110
|
|
@@ -133,8 +145,8 @@ Use `gcloud_pubsub` input plugin.
|
|
133
145
|
- Set your credential file path.
|
134
146
|
- Running fluentd on GCP, you can use scope instead of specifying this.
|
135
147
|
- You can also use environment variable such as `GCLOUD_KEYFILE`.
|
136
|
-
- `topic` (
|
137
|
-
- Set topic name to
|
148
|
+
- `topic` (optional)
|
149
|
+
- Set topic name that the subscription belongs to.
|
138
150
|
- `subscription` (required)
|
139
151
|
- Set subscription name to pull.
|
140
152
|
- `max_messages` (optional, default: `100`)
|
@@ -158,6 +170,8 @@ Use `gcloud_pubsub` input plugin.
|
|
158
170
|
- Bind IP address for HTTP RPC.
|
159
171
|
- `rpc_port` (optional, default: `24680`)
|
160
172
|
- Port for HTTP RPC.
|
173
|
+
- `decompression` (optional, default: `nil`)
|
174
|
+
- If set to `gzip`, messages will be decompressed with gzip.
|
161
175
|
|
162
176
|
## Contributing
|
163
177
|
|
data/Rakefile
CHANGED
@@ -7,17 +7,16 @@ Gem::Specification.new do |gem|
|
|
7
7
|
gem.license = "MIT"
|
8
8
|
gem.homepage = "https://github.com/mia-0032/fluent-plugin-gcloud-pubsub-custom"
|
9
9
|
gem.summary = gem.description
|
10
|
-
gem.version = "1.
|
10
|
+
gem.version = "1.6.0"
|
11
11
|
gem.authors = ["Yoshihiro MIYAI"]
|
12
12
|
gem.email = "msparrow17@gmail.com"
|
13
|
-
gem.has_rdoc = false
|
14
13
|
gem.files = `git ls-files`.split("\n")
|
15
14
|
gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
16
15
|
gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
17
16
|
gem.require_paths = ['lib']
|
18
17
|
|
19
18
|
gem.add_runtime_dependency "fluentd", [">= 0.14.15", "< 2"]
|
20
|
-
gem.add_runtime_dependency "google-cloud-pubsub", "~>
|
19
|
+
gem.add_runtime_dependency "google-cloud-pubsub", "~> 2.3.1"
|
21
20
|
|
22
21
|
gem.add_development_dependency "bundler"
|
23
22
|
gem.add_development_dependency "rake"
|
@@ -24,18 +24,23 @@ module Fluent
|
|
24
24
|
end
|
25
25
|
|
26
26
|
class Publisher
|
27
|
-
def initialize(project, key, autocreate_topic)
|
28
|
-
@pubsub = Google::Cloud::Pubsub.new project_id: project, credentials: key
|
27
|
+
def initialize(project, key, autocreate_topic, dest_project, endpoint, timeout)
|
28
|
+
@pubsub = Google::Cloud::Pubsub.new project_id: project, credentials: key, endpoint: endpoint, timeout: timeout
|
29
29
|
@autocreate_topic = autocreate_topic
|
30
|
+
@dest_project = dest_project
|
30
31
|
@topics = {}
|
31
32
|
end
|
32
33
|
|
33
34
|
def topic(topic_name)
|
34
35
|
return @topics[topic_name] if @topics.has_key? topic_name
|
35
36
|
|
36
|
-
|
37
|
-
|
38
|
-
client
|
37
|
+
if @dest_project.nil?
|
38
|
+
client = @pubsub.topic topic_name
|
39
|
+
if client.nil? && @autocreate_topic
|
40
|
+
client = @pubsub.create_topic topic_name
|
41
|
+
end
|
42
|
+
else
|
43
|
+
client = @pubsub.topic topic_name, project: @dest_project
|
39
44
|
end
|
40
45
|
if client.nil?
|
41
46
|
raise Error.new "topic:#{topic_name} does not exist."
|
@@ -59,8 +64,12 @@ module Fluent
|
|
59
64
|
class Subscriber
|
60
65
|
def initialize(project, key, topic_name, subscription_name)
|
61
66
|
pubsub = Google::Cloud::Pubsub.new project_id: project, credentials: key
|
62
|
-
|
63
|
-
|
67
|
+
if topic_name.nil?
|
68
|
+
@client = pubsub.subscription subscription_name
|
69
|
+
else
|
70
|
+
topic = pubsub.topic topic_name
|
71
|
+
@client = topic.subscription subscription_name
|
72
|
+
end
|
64
73
|
raise Error.new "subscription:#{subscription_name} does not exist." if @client.nil?
|
65
74
|
end
|
66
75
|
|
@@ -1,6 +1,7 @@
|
|
1
1
|
require 'json'
|
2
2
|
require 'webrick'
|
3
3
|
|
4
|
+
require 'fluent/plugin/compressable'
|
4
5
|
require 'fluent/plugin/input'
|
5
6
|
require 'fluent/plugin/parser'
|
6
7
|
|
@@ -8,6 +9,8 @@ require 'fluent/plugin/gcloud_pubsub/client'
|
|
8
9
|
|
9
10
|
module Fluent::Plugin
|
10
11
|
class GcloudPubSubInput < Input
|
12
|
+
include Fluent::Plugin::Compressable
|
13
|
+
|
11
14
|
Fluent::Plugin.register_input('gcloud_pubsub', self)
|
12
15
|
|
13
16
|
helpers :compat_parameters, :parser, :thread
|
@@ -26,7 +29,7 @@ module Fluent::Plugin
|
|
26
29
|
desc 'Set your credential file path.'
|
27
30
|
config_param :key, :string, default: nil
|
28
31
|
desc 'Set topic name to pull.'
|
29
|
-
config_param :topic, :string
|
32
|
+
config_param :topic, :string, default: nil
|
30
33
|
desc 'Set subscription name to pull.'
|
31
34
|
config_param :subscription, :string
|
32
35
|
desc 'Pulling messages by intervals of specified seconds.'
|
@@ -48,6 +51,8 @@ module Fluent::Plugin
|
|
48
51
|
config_param :rpc_bind, :string, default: '0.0.0.0'
|
49
52
|
desc 'Port for HTTP RPC.'
|
50
53
|
config_param :rpc_port, :integer, default: 24680
|
54
|
+
desc 'Decompress messages'
|
55
|
+
config_param :decompression, :string, default: nil
|
51
56
|
|
52
57
|
config_section :parse do
|
53
58
|
config_set_default :@type, DEFAULT_PARSER_TYPE
|
@@ -114,6 +119,11 @@ module Fluent::Plugin
|
|
114
119
|
end
|
115
120
|
|
116
121
|
@parser = parser_create
|
122
|
+
@decompress = if @decompression == 'gzip'
|
123
|
+
method(:gzip_decompress)
|
124
|
+
else
|
125
|
+
method(:no_decompress)
|
126
|
+
end
|
117
127
|
end
|
118
128
|
|
119
129
|
def start
|
@@ -168,6 +178,14 @@ module Fluent::Plugin
|
|
168
178
|
record.delete(@tag_key) || @tag
|
169
179
|
end
|
170
180
|
|
181
|
+
def gzip_decompress(message)
|
182
|
+
decompress message
|
183
|
+
end
|
184
|
+
|
185
|
+
def no_decompress(message)
|
186
|
+
message
|
187
|
+
end
|
188
|
+
|
171
189
|
def start_rpc
|
172
190
|
log.info "listening http rpc server on http://#{@rpc_bind}:#{@rpc_port}/"
|
173
191
|
@rpc_srv = WEBrick::HTTPServer.new(
|
@@ -221,7 +239,7 @@ module Fluent::Plugin
|
|
221
239
|
end
|
222
240
|
|
223
241
|
messages.each do |m|
|
224
|
-
line = m.message.data.chomp
|
242
|
+
line = @decompress.call(m.message.data).chomp
|
225
243
|
attributes = m.attributes
|
226
244
|
@parser.parse(line) do |time, record|
|
227
245
|
if time && record
|
@@ -233,9 +251,9 @@ module Fluent::Plugin
|
|
233
251
|
else
|
234
252
|
case @parse_error_action
|
235
253
|
when :exception
|
236
|
-
raise FailedParseError.new "pattern not match: #{line
|
254
|
+
raise FailedParseError.new "pattern not match: #{line}"
|
237
255
|
else
|
238
|
-
log.warn 'pattern not match', record: line
|
256
|
+
log.warn 'pattern not match', record: line
|
239
257
|
end
|
240
258
|
end
|
241
259
|
end
|
@@ -1,9 +1,11 @@
|
|
1
|
+
require 'fluent/plugin/compressable'
|
1
2
|
require 'fluent/plugin/output'
|
2
3
|
require 'fluent/plugin/gcloud_pubsub/client'
|
3
4
|
require 'fluent/plugin_helper/inject'
|
4
5
|
|
5
6
|
module Fluent::Plugin
|
6
7
|
class GcloudPubSubOutput < Output
|
8
|
+
include Fluent::Plugin::Compressable
|
7
9
|
include Fluent::PluginHelper::Inject
|
8
10
|
|
9
11
|
Fluent::Plugin.register_output('gcloud_pubsub', self)
|
@@ -19,6 +21,8 @@ module Fluent::Plugin
|
|
19
21
|
config_param :key, :string, :default => nil
|
20
22
|
desc 'Set topic name to publish.'
|
21
23
|
config_param :topic, :string
|
24
|
+
desc "Set your dest GCP project if publishing cross project"
|
25
|
+
config_param :dest_project, :string, :default => nil
|
22
26
|
desc "If set to `true`, specified topic will be created when it doesn't exist."
|
23
27
|
config_param :autocreate_topic, :bool, :default => false
|
24
28
|
desc 'Publishing messages count per request to Cloud Pub/Sub.'
|
@@ -27,8 +31,16 @@ module Fluent::Plugin
|
|
27
31
|
config_param :max_total_size, :integer, :default => 9800000 # 9.8MB
|
28
32
|
desc 'Limit bytesize per message.'
|
29
33
|
config_param :max_message_size, :integer, :default => 4000000 # 4MB
|
30
|
-
desc 'Publishing the set field as an attribute'
|
34
|
+
desc 'Publishing the set field as an attribute created from input message'
|
31
35
|
config_param :attribute_keys, :array, :default => []
|
36
|
+
desc 'Publishing the set field as an attribute created from input config params'
|
37
|
+
config_param :attribute_key_values, :hash, :default => {}
|
38
|
+
desc 'Set service endpoint'
|
39
|
+
config_param :endpoint, :string, :default => nil
|
40
|
+
desc 'Compress messages'
|
41
|
+
config_param :compression, :string, :default => nil
|
42
|
+
desc 'Set default timeout to use in publish requests'
|
43
|
+
config_param :timeout, :integer, :default => nil
|
32
44
|
|
33
45
|
config_section :buffer do
|
34
46
|
config_set_default :@type, DEFAULT_BUFFER_TYPE
|
@@ -43,11 +55,16 @@ module Fluent::Plugin
|
|
43
55
|
super
|
44
56
|
placeholder_validate!(:topic, @topic)
|
45
57
|
@formatter = formatter_create
|
58
|
+
@compress = if @compression == 'gzip'
|
59
|
+
method(:gzip_compress)
|
60
|
+
else
|
61
|
+
method(:no_compress)
|
62
|
+
end
|
46
63
|
end
|
47
64
|
|
48
65
|
def start
|
49
66
|
super
|
50
|
-
@publisher = Fluent::GcloudPubSub::Publisher.new @project, @key, @autocreate_topic
|
67
|
+
@publisher = Fluent::GcloudPubSub::Publisher.new @project, @key, @autocreate_topic, @dest_project, @endpoint, @timeout
|
51
68
|
end
|
52
69
|
|
53
70
|
def format(tag, time, record)
|
@@ -56,7 +73,8 @@ module Fluent::Plugin
|
|
56
73
|
@attribute_keys.each do |key|
|
57
74
|
attributes[key] = record.delete(key)
|
58
75
|
end
|
59
|
-
|
76
|
+
attributes.merge! @attribute_key_values
|
77
|
+
[@compress.call(@formatter.format(tag, time, record)), attributes].to_msgpack
|
60
78
|
end
|
61
79
|
|
62
80
|
def formatted_to_msgpack_binary?
|
@@ -106,5 +124,13 @@ module Fluent::Plugin
|
|
106
124
|
log.debug "send message topic:#{topic} length:#{messages.length} size:#{messages.map(&:bytesize).inject(:+)}"
|
107
125
|
@publisher.publish(topic, messages)
|
108
126
|
end
|
127
|
+
|
128
|
+
def gzip_compress(message)
|
129
|
+
compress message
|
130
|
+
end
|
131
|
+
|
132
|
+
def no_compress(message)
|
133
|
+
message
|
134
|
+
end
|
109
135
|
end
|
110
136
|
end
|
@@ -10,22 +10,14 @@ end
|
|
10
10
|
require 'test/unit'
|
11
11
|
require "test/unit/rr"
|
12
12
|
|
13
|
-
$LOAD_PATH.unshift(File.
|
14
|
-
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
13
|
+
$LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
|
15
14
|
|
16
|
-
require
|
17
|
-
|
18
|
-
|
19
|
-
nulllogger.instance_eval {|obj|
|
20
|
-
def method_missing(method, *args)
|
21
|
-
# pass
|
22
|
-
end
|
23
|
-
}
|
24
|
-
$log = nulllogger
|
25
|
-
end
|
15
|
+
require "fluent/test"
|
16
|
+
require "fluent/test/driver/output"
|
17
|
+
require "fluent/test/helpers"
|
26
18
|
|
27
19
|
require 'fluent/plugin/in_gcloud_pubsub'
|
28
20
|
require 'fluent/plugin/out_gcloud_pubsub'
|
29
21
|
|
30
|
-
|
31
|
-
|
22
|
+
Test::Unit::TestCase.include(Fluent::Test::Helpers)
|
23
|
+
Test::Unit::TestCase.extend(Fluent::Test::Helpers)
|
@@ -1,7 +1,8 @@
|
|
1
1
|
require 'net/http'
|
2
2
|
require 'json'
|
3
3
|
|
4
|
-
require_relative "../
|
4
|
+
require_relative "../helper"
|
5
|
+
require 'fluent/plugin/compressable'
|
5
6
|
require "fluent/test/driver/input"
|
6
7
|
|
7
8
|
class GcloudPubSubInputTest < Test::Unit::TestCase
|
@@ -164,6 +165,22 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
|
|
164
165
|
end
|
165
166
|
end
|
166
167
|
|
168
|
+
class DummyMsgDataGzipped
|
169
|
+
include Fluent::Plugin::Compressable
|
170
|
+
|
171
|
+
def data
|
172
|
+
compress '{"foo": "bar"}'
|
173
|
+
end
|
174
|
+
end
|
175
|
+
class DummyMessageGzipped
|
176
|
+
def message
|
177
|
+
DummyMsgDataGzipped.new
|
178
|
+
end
|
179
|
+
def attributes
|
180
|
+
{"attr_1" => "a", "attr_2" => "b"}
|
181
|
+
end
|
182
|
+
end
|
183
|
+
|
167
184
|
setup do
|
168
185
|
@subscriber = mock!
|
169
186
|
@topic_mock = mock!.subscription('subscription-test') { @subscriber }
|
@@ -263,6 +280,22 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
|
|
263
280
|
end
|
264
281
|
end
|
265
282
|
|
283
|
+
test 'with gzip decompression' do
|
284
|
+
messages = Array.new(1, DummyMessageGzipped.new)
|
285
|
+
@subscriber.pull(immediate: true, max: 100).at_least(1) { messages }
|
286
|
+
@subscriber.acknowledge(messages).at_least(1)
|
287
|
+
|
288
|
+
d = create_driver("#{CONFIG}\ndecompression gzip")
|
289
|
+
d.run(expect_emits: 1, timeout: 3)
|
290
|
+
emits = d.events
|
291
|
+
|
292
|
+
assert_equal(1, emits.length)
|
293
|
+
emits.each do |tag, time, record|
|
294
|
+
assert_equal("test", tag)
|
295
|
+
assert_equal({"foo" => "bar"}, record)
|
296
|
+
end
|
297
|
+
end
|
298
|
+
|
266
299
|
test 'invalid messages with parse_error_action warning' do
|
267
300
|
messages = Array.new(1, DummyInvalidMessage.new)
|
268
301
|
@subscriber.pull(immediate: true, max: 100).at_least(1) { messages }
|
@@ -1,10 +1,14 @@
|
|
1
1
|
# coding: utf-8
|
2
|
-
require_relative "../
|
2
|
+
require_relative "../helper"
|
3
|
+
require 'fluent/plugin/compressable'
|
3
4
|
require "fluent/test/driver/output"
|
4
5
|
require "fluent/test/helpers"
|
6
|
+
require 'json'
|
7
|
+
require 'msgpack'
|
5
8
|
|
6
9
|
class GcloudPubSubOutputTest < Test::Unit::TestCase
|
7
10
|
include Fluent::Test::Helpers
|
11
|
+
include Fluent::Plugin::Compressable
|
8
12
|
|
9
13
|
CONFIG = %[
|
10
14
|
project project-test
|
@@ -62,6 +66,17 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
|
|
62
66
|
|
63
67
|
assert_equal(true, d.instance.autocreate_topic)
|
64
68
|
end
|
69
|
+
|
70
|
+
test '"dest_project" can be specified' do
|
71
|
+
d = create_driver(%[
|
72
|
+
project project-test
|
73
|
+
topic topic-test
|
74
|
+
key key-test
|
75
|
+
dest_project dest-project-test
|
76
|
+
])
|
77
|
+
|
78
|
+
assert_equal("dest-project-test", d.instance.dest_project)
|
79
|
+
end
|
65
80
|
end
|
66
81
|
|
67
82
|
sub_test_case 'topic' do
|
@@ -69,6 +84,13 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
|
|
69
84
|
@publisher = mock!
|
70
85
|
@pubsub_mock = mock!
|
71
86
|
stub(Google::Cloud::Pubsub).new { @pubsub_mock }
|
87
|
+
|
88
|
+
@old_report_on_exception = Thread.report_on_exception
|
89
|
+
Thread.report_on_exception = false
|
90
|
+
end
|
91
|
+
|
92
|
+
teardown do
|
93
|
+
Thread.report_on_exception = @old_report_on_exception
|
72
94
|
end
|
73
95
|
|
74
96
|
test '"autocreate_topic" is enabled' do
|
@@ -87,6 +109,21 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
|
|
87
109
|
end
|
88
110
|
end
|
89
111
|
|
112
|
+
test '"dest_project" is set' do
|
113
|
+
d = create_driver(%[
|
114
|
+
project project-test
|
115
|
+
topic topic-test
|
116
|
+
key key-test
|
117
|
+
dest_project dest-project-test
|
118
|
+
])
|
119
|
+
|
120
|
+
@publisher.publish.once
|
121
|
+
@pubsub_mock.topic("topic-test", {:project=>"dest-project-test"}).once { @publisher }
|
122
|
+
d.run(default_tag: "test") do
|
123
|
+
d.feed(@time, {"a" => "b"})
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
90
127
|
test '40x error occurred on connecting to Pub/Sub' do
|
91
128
|
d = create_driver
|
92
129
|
|
@@ -147,6 +184,13 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
|
|
147
184
|
@publisher = mock!
|
148
185
|
@pubsub_mock = mock!.topic(anything) { @publisher }
|
149
186
|
stub(Google::Cloud::Pubsub).new { @pubsub_mock }
|
187
|
+
|
188
|
+
@old_report_on_exception = Thread.report_on_exception
|
189
|
+
Thread.report_on_exception = false
|
190
|
+
end
|
191
|
+
|
192
|
+
teardown do
|
193
|
+
Thread.report_on_exception = @old_report_on_exception
|
150
194
|
end
|
151
195
|
|
152
196
|
test 'messages are divided into "max_messages"' do
|
@@ -222,5 +266,83 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
|
|
222
266
|
end
|
223
267
|
assert_equal({"tag" => 'test', "foo" => "bar"}, JSON.parse(MessagePack.unpack(d.formatted.first)[0]))
|
224
268
|
end
|
269
|
+
|
270
|
+
test 'with gzip compression' do
|
271
|
+
d = create_driver(%[
|
272
|
+
project project-test
|
273
|
+
topic topic-test
|
274
|
+
key key-test
|
275
|
+
compression gzip
|
276
|
+
])
|
277
|
+
test_msg = {"a" => "abc"}
|
278
|
+
|
279
|
+
@publisher.publish.once
|
280
|
+
d.run(default_tag: "test") do
|
281
|
+
d.feed(test_msg)
|
282
|
+
end
|
283
|
+
|
284
|
+
assert_equal(test_msg, JSON.parse(decompress(MessagePack.unpack(d.formatted.first)[0])))
|
285
|
+
end
|
286
|
+
|
287
|
+
test 'with attribute_keys' do
|
288
|
+
d = create_driver(%[
|
289
|
+
project project-test
|
290
|
+
topic topic-test
|
291
|
+
key key-test
|
292
|
+
attribute_keys a,b
|
293
|
+
])
|
294
|
+
|
295
|
+
@publisher.publish.once
|
296
|
+
d.run(default_tag: "test") do
|
297
|
+
d.feed(@time, {"a" => "foo", "b" => "bar", "c" => "baz"})
|
298
|
+
end
|
299
|
+
|
300
|
+
formatted = MessagePack.unpack(d.formatted[0])
|
301
|
+
message = JSON.parse(formatted[0])
|
302
|
+
assert_equal(message, {"c" => "baz"})
|
303
|
+
attributes = formatted[1]
|
304
|
+
assert_equal(attributes, {"a" => "foo", "b" => "bar"})
|
305
|
+
end
|
306
|
+
|
307
|
+
test 'with attribute_key_values' do
|
308
|
+
d = create_driver(%[
|
309
|
+
project project-test
|
310
|
+
topic topic-test
|
311
|
+
key key-test
|
312
|
+
attribute_key_values {"key1": "value1", "key2": "value2"}
|
313
|
+
])
|
314
|
+
|
315
|
+
@publisher.publish.once
|
316
|
+
d.run(default_tag: "test") do
|
317
|
+
d.feed(@time, {"a" => "foo", "b" => "bar"})
|
318
|
+
end
|
319
|
+
|
320
|
+
formatted = MessagePack.unpack(d.formatted[0])
|
321
|
+
message = JSON.parse(formatted[0])
|
322
|
+
assert_equal(message, {"a" => "foo", "b" => "bar"})
|
323
|
+
attributes = formatted[1]
|
324
|
+
assert_equal(attributes, {"key1" => "value1", "key2" => "value2"})
|
325
|
+
end
|
326
|
+
|
327
|
+
test 'with attribute_keys and attribute_key_values' do
|
328
|
+
d = create_driver(%[
|
329
|
+
project project-test
|
330
|
+
topic topic-test
|
331
|
+
key key-test
|
332
|
+
attribute_keys a
|
333
|
+
attribute_key_values {"key": "value"}
|
334
|
+
])
|
335
|
+
|
336
|
+
@publisher.publish.once
|
337
|
+
d.run(default_tag: "test") do
|
338
|
+
d.feed(@time, {"a" => "foo", "b" => "bar"})
|
339
|
+
end
|
340
|
+
|
341
|
+
formatted = MessagePack.unpack(d.formatted[0])
|
342
|
+
message = JSON.parse(formatted[0])
|
343
|
+
assert_equal(message, {"b" => "bar"})
|
344
|
+
attributes = formatted[1]
|
345
|
+
assert_equal(attributes, {"a" => "foo", "key" => "value"})
|
346
|
+
end
|
225
347
|
end
|
226
348
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-gcloud-pubsub-custom
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yoshihiro MIYAI
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-06-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -36,14 +36,14 @@ dependencies:
|
|
36
36
|
requirements:
|
37
37
|
- - "~>"
|
38
38
|
- !ruby/object:Gem::Version
|
39
|
-
version:
|
39
|
+
version: 2.3.1
|
40
40
|
type: :runtime
|
41
41
|
prerelease: false
|
42
42
|
version_requirements: !ruby/object:Gem::Requirement
|
43
43
|
requirements:
|
44
44
|
- - "~>"
|
45
45
|
- !ruby/object:Gem::Version
|
46
|
-
version:
|
46
|
+
version: 2.3.1
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
48
|
name: bundler
|
49
49
|
requirement: !ruby/object:Gem::Requirement
|
@@ -106,8 +106,8 @@ executables: []
|
|
106
106
|
extensions: []
|
107
107
|
extra_rdoc_files: []
|
108
108
|
files:
|
109
|
+
- ".github/workflows/test.yml"
|
109
110
|
- ".gitignore"
|
110
|
-
- ".travis.yml"
|
111
111
|
- CHANGELOG.md
|
112
112
|
- Gemfile
|
113
113
|
- LICENSE
|
@@ -117,14 +117,14 @@ files:
|
|
117
117
|
- lib/fluent/plugin/gcloud_pubsub/client.rb
|
118
118
|
- lib/fluent/plugin/in_gcloud_pubsub.rb
|
119
119
|
- lib/fluent/plugin/out_gcloud_pubsub.rb
|
120
|
+
- test/helper.rb
|
120
121
|
- test/plugin/test_in_gcloud_pubsub.rb
|
121
122
|
- test/plugin/test_out_gcloud_pubsub.rb
|
122
|
-
- test/test_helper.rb
|
123
123
|
homepage: https://github.com/mia-0032/fluent-plugin-gcloud-pubsub-custom
|
124
124
|
licenses:
|
125
125
|
- MIT
|
126
126
|
metadata: {}
|
127
|
-
post_install_message:
|
127
|
+
post_install_message:
|
128
128
|
rdoc_options: []
|
129
129
|
require_paths:
|
130
130
|
- lib
|
@@ -139,12 +139,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
139
139
|
- !ruby/object:Gem::Version
|
140
140
|
version: '0'
|
141
141
|
requirements: []
|
142
|
-
|
143
|
-
|
144
|
-
signing_key:
|
142
|
+
rubygems_version: 3.1.2
|
143
|
+
signing_key:
|
145
144
|
specification_version: 4
|
146
145
|
summary: Google Cloud Pub/Sub input/output plugin for Fluentd event collector
|
147
146
|
test_files:
|
147
|
+
- test/helper.rb
|
148
148
|
- test/plugin/test_in_gcloud_pubsub.rb
|
149
149
|
- test/plugin/test_out_gcloud_pubsub.rb
|
150
|
-
- test/test_helper.rb
|
data/.travis.yml
DELETED
@@ -1,25 +0,0 @@
|
|
1
|
-
language: ruby
|
2
|
-
|
3
|
-
rvm:
|
4
|
-
- 2.1
|
5
|
-
- 2.2
|
6
|
-
- 2.3.6
|
7
|
-
- 2.4.3
|
8
|
-
- 2.5.0
|
9
|
-
- ruby-head
|
10
|
-
|
11
|
-
gemfile:
|
12
|
-
- Gemfile
|
13
|
-
|
14
|
-
branches:
|
15
|
-
only:
|
16
|
-
- master
|
17
|
-
|
18
|
-
before_install: gem update bundler
|
19
|
-
script: bundle exec rake test
|
20
|
-
|
21
|
-
sudo: false
|
22
|
-
|
23
|
-
matrix:
|
24
|
-
allow_failures:
|
25
|
-
- rvm: ruby-head
|