fluent-plugin-gcloud-pubsub-custom 1.3.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +25 -0
- data/CHANGELOG.md +29 -0
- data/README.md +18 -4
- data/Rakefile +1 -0
- data/fluent-plugin-gcloud-pubsub-custom.gemspec +2 -3
- data/lib/fluent/plugin/gcloud_pubsub/client.rb +16 -7
- data/lib/fluent/plugin/in_gcloud_pubsub.rb +22 -4
- data/lib/fluent/plugin/out_gcloud_pubsub.rb +29 -3
- data/test/{test_helper.rb → helper.rb} +6 -14
- data/test/plugin/test_in_gcloud_pubsub.rb +34 -1
- data/test/plugin/test_out_gcloud_pubsub.rb +123 -1
- metadata +11 -12
- data/.travis.yml +0 -25
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3bd7e53135bef7ce1f7ce3796659b6deb1ebaf6ff303755a84fdd886c4f46bf3
|
4
|
+
data.tar.gz: a04ddb36d5c0bceb7ac1fb8947b537ee0c38b9b26f6b5d5c53bb2ed034deb4e0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f0d6537d00021bb6d849ff7c9c9e8b4b233214826b098db649820a0a310d7d60170a8642db3b7b7acee2f83b70a392ad6a9ef8adbb47f22bc2d1be2022129d03
|
7
|
+
data.tar.gz: 2b742512024102940dd447f6450377684a1834c89582e38d6498957dc6219c5625afa49320f2c6c0139ff73bd7e98ad61085029b0c4794e3418c0324b0346bc0
|
@@ -0,0 +1,25 @@
|
|
1
|
+
name: Test
|
2
|
+
|
3
|
+
on:
|
4
|
+
push:
|
5
|
+
branches: [ master ]
|
6
|
+
pull_request:
|
7
|
+
branches: [ master ]
|
8
|
+
|
9
|
+
jobs:
|
10
|
+
test:
|
11
|
+
runs-on: ubuntu-latest
|
12
|
+
strategy:
|
13
|
+
fail-fast: false
|
14
|
+
matrix:
|
15
|
+
ruby-version: ['2.4', '2.5', '2.6', '2.7']
|
16
|
+
|
17
|
+
steps:
|
18
|
+
- uses: actions/checkout@v2
|
19
|
+
- name: Set up Ruby
|
20
|
+
uses: ruby/setup-ruby@v1
|
21
|
+
with:
|
22
|
+
ruby-version: ${{ matrix.ruby-version }}
|
23
|
+
bundler-cache: true
|
24
|
+
- name: Run tests
|
25
|
+
run: bundle exec rake test
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,34 @@
|
|
1
1
|
## ChangeLog
|
2
2
|
|
3
|
+
### Release 1.6.0 - 2021/06/17
|
4
|
+
|
5
|
+
- Output plugin
|
6
|
+
- Add `timeout` feature
|
7
|
+
- Add `attribute_key_values` feature
|
8
|
+
|
9
|
+
### Release 1.5.0 - 2021/01/26
|
10
|
+
|
11
|
+
- Bump up google-cloud-pubsub to v2.3.x
|
12
|
+
- Add `endpoint` parameter
|
13
|
+
- Add gzip compression/decompression feature
|
14
|
+
|
15
|
+
### Release 1.4.0 - 2020/08/11
|
16
|
+
|
17
|
+
- Bump up google-cloud-pubsub to v0.39.x
|
18
|
+
- In preparation for updating to v1.x
|
19
|
+
- Output plugin
|
20
|
+
- Add `dest_project` parameter
|
21
|
+
|
22
|
+
### Release 1.3.2 - 2019/08/16
|
23
|
+
|
24
|
+
- Input plugin
|
25
|
+
- Make `topic` parameter optional
|
26
|
+
|
27
|
+
### Release 1.3.1 - 2018/06/08
|
28
|
+
|
29
|
+
- Input plugin
|
30
|
+
- Dumps raw string to logs for parsing errors
|
31
|
+
|
3
32
|
### Release 1.3.0 - 2018/05/18
|
4
33
|
|
5
34
|
- Make attributes available
|
data/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# fluent-plugin-gcloud-pubsub-custom
|
2
2
|
|
3
|
-
|
3
|
+

|
4
4
|
[](http://badge.fury.io/rb/fluent-plugin-gcloud-pubsub-custom)
|
5
5
|
|
6
6
|
This plugin is forked from https://github.com/mdoi/fluent-plugin-gcloud-pubsub
|
@@ -60,6 +60,8 @@ Use `gcloud_pubsub` output plugin.
|
|
60
60
|
max_messages 1000
|
61
61
|
max_total_size 9800000
|
62
62
|
max_message_size 4000000
|
63
|
+
endpoint <Endpoint URL>
|
64
|
+
timeout 60
|
63
65
|
<buffer>
|
64
66
|
@type memory
|
65
67
|
flush_interval 1s
|
@@ -81,6 +83,8 @@ Use `gcloud_pubsub` output plugin.
|
|
81
83
|
- `topic` (required)
|
82
84
|
- Set topic name to publish.
|
83
85
|
- You can use placeholder in this param. See: https://docs.fluentd.org/v1.0/articles/buffer-section
|
86
|
+
- `dest_project` (optional, default: `nil`)
|
87
|
+
- Set your destination GCP project if you publish messages cross project.
|
84
88
|
- `autocreate_topic` (optional, default: `false`)
|
85
89
|
- If set to `true`, specified topic will be created when it doesn't exist.
|
86
90
|
- `max_messages` (optional, default: `1000`)
|
@@ -92,7 +96,15 @@ Use `gcloud_pubsub` output plugin.
|
|
92
96
|
- `max_message_size` (optional, default: `4000000` = `4MB`)
|
93
97
|
- Messages exceeding `max_message_size` are not published because Pub/Sub clients cannot receive it.
|
94
98
|
- `attribute_keys` (optional, default: `[]`)
|
95
|
-
- Publishing the set fields as attributes.
|
99
|
+
- Publishing the set fields as attributes generated from input message.
|
100
|
+
- `attribute_key_values` (optional, default: `{}`)
|
101
|
+
- Publishing the set fields as attributes generated from input params
|
102
|
+
- `endpoint`(optional)
|
103
|
+
- Set Pub/Sub service endpoint. For more information, see [Service Endpoints](https://cloud.google.com/pubsub/docs/reference/service_apis_overview#service_endpoints)
|
104
|
+
- `compression` (optional, default: `nil`)
|
105
|
+
- If set to `gzip`, messages will be compressed with gzip.
|
106
|
+
- `timeout` (optional)
|
107
|
+
- Set default timeout to use in publish requests.
|
96
108
|
|
97
109
|
### Pull messages
|
98
110
|
|
@@ -133,8 +145,8 @@ Use `gcloud_pubsub` input plugin.
|
|
133
145
|
- Set your credential file path.
|
134
146
|
- Running fluentd on GCP, you can use scope instead of specifying this.
|
135
147
|
- You can also use environment variable such as `GCLOUD_KEYFILE`.
|
136
|
-
- `topic` (
|
137
|
-
- Set topic name to
|
148
|
+
- `topic` (optional)
|
149
|
+
- Set topic name that the subscription belongs to.
|
138
150
|
- `subscription` (required)
|
139
151
|
- Set subscription name to pull.
|
140
152
|
- `max_messages` (optional, default: `100`)
|
@@ -158,6 +170,8 @@ Use `gcloud_pubsub` input plugin.
|
|
158
170
|
- Bind IP address for HTTP RPC.
|
159
171
|
- `rpc_port` (optional, default: `24680`)
|
160
172
|
- Port for HTTP RPC.
|
173
|
+
- `decompression` (optional, default: `nil`)
|
174
|
+
- If set to `gzip`, messages will be decompressed with gzip.
|
161
175
|
|
162
176
|
## Contributing
|
163
177
|
|
data/Rakefile
CHANGED
@@ -7,17 +7,16 @@ Gem::Specification.new do |gem|
|
|
7
7
|
gem.license = "MIT"
|
8
8
|
gem.homepage = "https://github.com/mia-0032/fluent-plugin-gcloud-pubsub-custom"
|
9
9
|
gem.summary = gem.description
|
10
|
-
gem.version = "1.
|
10
|
+
gem.version = "1.6.0"
|
11
11
|
gem.authors = ["Yoshihiro MIYAI"]
|
12
12
|
gem.email = "msparrow17@gmail.com"
|
13
|
-
gem.has_rdoc = false
|
14
13
|
gem.files = `git ls-files`.split("\n")
|
15
14
|
gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
16
15
|
gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
17
16
|
gem.require_paths = ['lib']
|
18
17
|
|
19
18
|
gem.add_runtime_dependency "fluentd", [">= 0.14.15", "< 2"]
|
20
|
-
gem.add_runtime_dependency "google-cloud-pubsub", "~>
|
19
|
+
gem.add_runtime_dependency "google-cloud-pubsub", "~> 2.3.1"
|
21
20
|
|
22
21
|
gem.add_development_dependency "bundler"
|
23
22
|
gem.add_development_dependency "rake"
|
@@ -24,18 +24,23 @@ module Fluent
|
|
24
24
|
end
|
25
25
|
|
26
26
|
class Publisher
|
27
|
-
def initialize(project, key, autocreate_topic)
|
28
|
-
@pubsub = Google::Cloud::Pubsub.new project_id: project, credentials: key
|
27
|
+
def initialize(project, key, autocreate_topic, dest_project, endpoint, timeout)
|
28
|
+
@pubsub = Google::Cloud::Pubsub.new project_id: project, credentials: key, endpoint: endpoint, timeout: timeout
|
29
29
|
@autocreate_topic = autocreate_topic
|
30
|
+
@dest_project = dest_project
|
30
31
|
@topics = {}
|
31
32
|
end
|
32
33
|
|
33
34
|
def topic(topic_name)
|
34
35
|
return @topics[topic_name] if @topics.has_key? topic_name
|
35
36
|
|
36
|
-
|
37
|
-
|
38
|
-
client
|
37
|
+
if @dest_project.nil?
|
38
|
+
client = @pubsub.topic topic_name
|
39
|
+
if client.nil? && @autocreate_topic
|
40
|
+
client = @pubsub.create_topic topic_name
|
41
|
+
end
|
42
|
+
else
|
43
|
+
client = @pubsub.topic topic_name, project: @dest_project
|
39
44
|
end
|
40
45
|
if client.nil?
|
41
46
|
raise Error.new "topic:#{topic_name} does not exist."
|
@@ -59,8 +64,12 @@ module Fluent
|
|
59
64
|
class Subscriber
|
60
65
|
def initialize(project, key, topic_name, subscription_name)
|
61
66
|
pubsub = Google::Cloud::Pubsub.new project_id: project, credentials: key
|
62
|
-
|
63
|
-
|
67
|
+
if topic_name.nil?
|
68
|
+
@client = pubsub.subscription subscription_name
|
69
|
+
else
|
70
|
+
topic = pubsub.topic topic_name
|
71
|
+
@client = topic.subscription subscription_name
|
72
|
+
end
|
64
73
|
raise Error.new "subscription:#{subscription_name} does not exist." if @client.nil?
|
65
74
|
end
|
66
75
|
|
@@ -1,6 +1,7 @@
|
|
1
1
|
require 'json'
|
2
2
|
require 'webrick'
|
3
3
|
|
4
|
+
require 'fluent/plugin/compressable'
|
4
5
|
require 'fluent/plugin/input'
|
5
6
|
require 'fluent/plugin/parser'
|
6
7
|
|
@@ -8,6 +9,8 @@ require 'fluent/plugin/gcloud_pubsub/client'
|
|
8
9
|
|
9
10
|
module Fluent::Plugin
|
10
11
|
class GcloudPubSubInput < Input
|
12
|
+
include Fluent::Plugin::Compressable
|
13
|
+
|
11
14
|
Fluent::Plugin.register_input('gcloud_pubsub', self)
|
12
15
|
|
13
16
|
helpers :compat_parameters, :parser, :thread
|
@@ -26,7 +29,7 @@ module Fluent::Plugin
|
|
26
29
|
desc 'Set your credential file path.'
|
27
30
|
config_param :key, :string, default: nil
|
28
31
|
desc 'Set topic name to pull.'
|
29
|
-
config_param :topic, :string
|
32
|
+
config_param :topic, :string, default: nil
|
30
33
|
desc 'Set subscription name to pull.'
|
31
34
|
config_param :subscription, :string
|
32
35
|
desc 'Pulling messages by intervals of specified seconds.'
|
@@ -48,6 +51,8 @@ module Fluent::Plugin
|
|
48
51
|
config_param :rpc_bind, :string, default: '0.0.0.0'
|
49
52
|
desc 'Port for HTTP RPC.'
|
50
53
|
config_param :rpc_port, :integer, default: 24680
|
54
|
+
desc 'Decompress messages'
|
55
|
+
config_param :decompression, :string, default: nil
|
51
56
|
|
52
57
|
config_section :parse do
|
53
58
|
config_set_default :@type, DEFAULT_PARSER_TYPE
|
@@ -114,6 +119,11 @@ module Fluent::Plugin
|
|
114
119
|
end
|
115
120
|
|
116
121
|
@parser = parser_create
|
122
|
+
@decompress = if @decompression == 'gzip'
|
123
|
+
method(:gzip_decompress)
|
124
|
+
else
|
125
|
+
method(:no_decompress)
|
126
|
+
end
|
117
127
|
end
|
118
128
|
|
119
129
|
def start
|
@@ -168,6 +178,14 @@ module Fluent::Plugin
|
|
168
178
|
record.delete(@tag_key) || @tag
|
169
179
|
end
|
170
180
|
|
181
|
+
def gzip_decompress(message)
|
182
|
+
decompress message
|
183
|
+
end
|
184
|
+
|
185
|
+
def no_decompress(message)
|
186
|
+
message
|
187
|
+
end
|
188
|
+
|
171
189
|
def start_rpc
|
172
190
|
log.info "listening http rpc server on http://#{@rpc_bind}:#{@rpc_port}/"
|
173
191
|
@rpc_srv = WEBrick::HTTPServer.new(
|
@@ -221,7 +239,7 @@ module Fluent::Plugin
|
|
221
239
|
end
|
222
240
|
|
223
241
|
messages.each do |m|
|
224
|
-
line = m.message.data.chomp
|
242
|
+
line = @decompress.call(m.message.data).chomp
|
225
243
|
attributes = m.attributes
|
226
244
|
@parser.parse(line) do |time, record|
|
227
245
|
if time && record
|
@@ -233,9 +251,9 @@ module Fluent::Plugin
|
|
233
251
|
else
|
234
252
|
case @parse_error_action
|
235
253
|
when :exception
|
236
|
-
raise FailedParseError.new "pattern not match: #{line
|
254
|
+
raise FailedParseError.new "pattern not match: #{line}"
|
237
255
|
else
|
238
|
-
log.warn 'pattern not match', record: line
|
256
|
+
log.warn 'pattern not match', record: line
|
239
257
|
end
|
240
258
|
end
|
241
259
|
end
|
@@ -1,9 +1,11 @@
|
|
1
|
+
require 'fluent/plugin/compressable'
|
1
2
|
require 'fluent/plugin/output'
|
2
3
|
require 'fluent/plugin/gcloud_pubsub/client'
|
3
4
|
require 'fluent/plugin_helper/inject'
|
4
5
|
|
5
6
|
module Fluent::Plugin
|
6
7
|
class GcloudPubSubOutput < Output
|
8
|
+
include Fluent::Plugin::Compressable
|
7
9
|
include Fluent::PluginHelper::Inject
|
8
10
|
|
9
11
|
Fluent::Plugin.register_output('gcloud_pubsub', self)
|
@@ -19,6 +21,8 @@ module Fluent::Plugin
|
|
19
21
|
config_param :key, :string, :default => nil
|
20
22
|
desc 'Set topic name to publish.'
|
21
23
|
config_param :topic, :string
|
24
|
+
desc "Set your dest GCP project if publishing cross project"
|
25
|
+
config_param :dest_project, :string, :default => nil
|
22
26
|
desc "If set to `true`, specified topic will be created when it doesn't exist."
|
23
27
|
config_param :autocreate_topic, :bool, :default => false
|
24
28
|
desc 'Publishing messages count per request to Cloud Pub/Sub.'
|
@@ -27,8 +31,16 @@ module Fluent::Plugin
|
|
27
31
|
config_param :max_total_size, :integer, :default => 9800000 # 9.8MB
|
28
32
|
desc 'Limit bytesize per message.'
|
29
33
|
config_param :max_message_size, :integer, :default => 4000000 # 4MB
|
30
|
-
desc 'Publishing the set field as an attribute'
|
34
|
+
desc 'Publishing the set field as an attribute created from input message'
|
31
35
|
config_param :attribute_keys, :array, :default => []
|
36
|
+
desc 'Publishing the set field as an attribute created from input config params'
|
37
|
+
config_param :attribute_key_values, :hash, :default => {}
|
38
|
+
desc 'Set service endpoint'
|
39
|
+
config_param :endpoint, :string, :default => nil
|
40
|
+
desc 'Compress messages'
|
41
|
+
config_param :compression, :string, :default => nil
|
42
|
+
desc 'Set default timeout to use in publish requests'
|
43
|
+
config_param :timeout, :integer, :default => nil
|
32
44
|
|
33
45
|
config_section :buffer do
|
34
46
|
config_set_default :@type, DEFAULT_BUFFER_TYPE
|
@@ -43,11 +55,16 @@ module Fluent::Plugin
|
|
43
55
|
super
|
44
56
|
placeholder_validate!(:topic, @topic)
|
45
57
|
@formatter = formatter_create
|
58
|
+
@compress = if @compression == 'gzip'
|
59
|
+
method(:gzip_compress)
|
60
|
+
else
|
61
|
+
method(:no_compress)
|
62
|
+
end
|
46
63
|
end
|
47
64
|
|
48
65
|
def start
|
49
66
|
super
|
50
|
-
@publisher = Fluent::GcloudPubSub::Publisher.new @project, @key, @autocreate_topic
|
67
|
+
@publisher = Fluent::GcloudPubSub::Publisher.new @project, @key, @autocreate_topic, @dest_project, @endpoint, @timeout
|
51
68
|
end
|
52
69
|
|
53
70
|
def format(tag, time, record)
|
@@ -56,7 +73,8 @@ module Fluent::Plugin
|
|
56
73
|
@attribute_keys.each do |key|
|
57
74
|
attributes[key] = record.delete(key)
|
58
75
|
end
|
59
|
-
|
76
|
+
attributes.merge! @attribute_key_values
|
77
|
+
[@compress.call(@formatter.format(tag, time, record)), attributes].to_msgpack
|
60
78
|
end
|
61
79
|
|
62
80
|
def formatted_to_msgpack_binary?
|
@@ -106,5 +124,13 @@ module Fluent::Plugin
|
|
106
124
|
log.debug "send message topic:#{topic} length:#{messages.length} size:#{messages.map(&:bytesize).inject(:+)}"
|
107
125
|
@publisher.publish(topic, messages)
|
108
126
|
end
|
127
|
+
|
128
|
+
def gzip_compress(message)
|
129
|
+
compress message
|
130
|
+
end
|
131
|
+
|
132
|
+
def no_compress(message)
|
133
|
+
message
|
134
|
+
end
|
109
135
|
end
|
110
136
|
end
|
@@ -10,22 +10,14 @@ end
|
|
10
10
|
require 'test/unit'
|
11
11
|
require "test/unit/rr"
|
12
12
|
|
13
|
-
$LOAD_PATH.unshift(File.
|
14
|
-
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
13
|
+
$LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
|
15
14
|
|
16
|
-
require
|
17
|
-
|
18
|
-
|
19
|
-
nulllogger.instance_eval {|obj|
|
20
|
-
def method_missing(method, *args)
|
21
|
-
# pass
|
22
|
-
end
|
23
|
-
}
|
24
|
-
$log = nulllogger
|
25
|
-
end
|
15
|
+
require "fluent/test"
|
16
|
+
require "fluent/test/driver/output"
|
17
|
+
require "fluent/test/helpers"
|
26
18
|
|
27
19
|
require 'fluent/plugin/in_gcloud_pubsub'
|
28
20
|
require 'fluent/plugin/out_gcloud_pubsub'
|
29
21
|
|
30
|
-
|
31
|
-
|
22
|
+
Test::Unit::TestCase.include(Fluent::Test::Helpers)
|
23
|
+
Test::Unit::TestCase.extend(Fluent::Test::Helpers)
|
@@ -1,7 +1,8 @@
|
|
1
1
|
require 'net/http'
|
2
2
|
require 'json'
|
3
3
|
|
4
|
-
require_relative "../
|
4
|
+
require_relative "../helper"
|
5
|
+
require 'fluent/plugin/compressable'
|
5
6
|
require "fluent/test/driver/input"
|
6
7
|
|
7
8
|
class GcloudPubSubInputTest < Test::Unit::TestCase
|
@@ -164,6 +165,22 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
|
|
164
165
|
end
|
165
166
|
end
|
166
167
|
|
168
|
+
class DummyMsgDataGzipped
|
169
|
+
include Fluent::Plugin::Compressable
|
170
|
+
|
171
|
+
def data
|
172
|
+
compress '{"foo": "bar"}'
|
173
|
+
end
|
174
|
+
end
|
175
|
+
class DummyMessageGzipped
|
176
|
+
def message
|
177
|
+
DummyMsgDataGzipped.new
|
178
|
+
end
|
179
|
+
def attributes
|
180
|
+
{"attr_1" => "a", "attr_2" => "b"}
|
181
|
+
end
|
182
|
+
end
|
183
|
+
|
167
184
|
setup do
|
168
185
|
@subscriber = mock!
|
169
186
|
@topic_mock = mock!.subscription('subscription-test') { @subscriber }
|
@@ -263,6 +280,22 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
|
|
263
280
|
end
|
264
281
|
end
|
265
282
|
|
283
|
+
test 'with gzip decompression' do
|
284
|
+
messages = Array.new(1, DummyMessageGzipped.new)
|
285
|
+
@subscriber.pull(immediate: true, max: 100).at_least(1) { messages }
|
286
|
+
@subscriber.acknowledge(messages).at_least(1)
|
287
|
+
|
288
|
+
d = create_driver("#{CONFIG}\ndecompression gzip")
|
289
|
+
d.run(expect_emits: 1, timeout: 3)
|
290
|
+
emits = d.events
|
291
|
+
|
292
|
+
assert_equal(1, emits.length)
|
293
|
+
emits.each do |tag, time, record|
|
294
|
+
assert_equal("test", tag)
|
295
|
+
assert_equal({"foo" => "bar"}, record)
|
296
|
+
end
|
297
|
+
end
|
298
|
+
|
266
299
|
test 'invalid messages with parse_error_action warning' do
|
267
300
|
messages = Array.new(1, DummyInvalidMessage.new)
|
268
301
|
@subscriber.pull(immediate: true, max: 100).at_least(1) { messages }
|
@@ -1,10 +1,14 @@
|
|
1
1
|
# coding: utf-8
|
2
|
-
require_relative "../
|
2
|
+
require_relative "../helper"
|
3
|
+
require 'fluent/plugin/compressable'
|
3
4
|
require "fluent/test/driver/output"
|
4
5
|
require "fluent/test/helpers"
|
6
|
+
require 'json'
|
7
|
+
require 'msgpack'
|
5
8
|
|
6
9
|
class GcloudPubSubOutputTest < Test::Unit::TestCase
|
7
10
|
include Fluent::Test::Helpers
|
11
|
+
include Fluent::Plugin::Compressable
|
8
12
|
|
9
13
|
CONFIG = %[
|
10
14
|
project project-test
|
@@ -62,6 +66,17 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
|
|
62
66
|
|
63
67
|
assert_equal(true, d.instance.autocreate_topic)
|
64
68
|
end
|
69
|
+
|
70
|
+
test '"dest_project" can be specified' do
|
71
|
+
d = create_driver(%[
|
72
|
+
project project-test
|
73
|
+
topic topic-test
|
74
|
+
key key-test
|
75
|
+
dest_project dest-project-test
|
76
|
+
])
|
77
|
+
|
78
|
+
assert_equal("dest-project-test", d.instance.dest_project)
|
79
|
+
end
|
65
80
|
end
|
66
81
|
|
67
82
|
sub_test_case 'topic' do
|
@@ -69,6 +84,13 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
|
|
69
84
|
@publisher = mock!
|
70
85
|
@pubsub_mock = mock!
|
71
86
|
stub(Google::Cloud::Pubsub).new { @pubsub_mock }
|
87
|
+
|
88
|
+
@old_report_on_exception = Thread.report_on_exception
|
89
|
+
Thread.report_on_exception = false
|
90
|
+
end
|
91
|
+
|
92
|
+
teardown do
|
93
|
+
Thread.report_on_exception = @old_report_on_exception
|
72
94
|
end
|
73
95
|
|
74
96
|
test '"autocreate_topic" is enabled' do
|
@@ -87,6 +109,21 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
|
|
87
109
|
end
|
88
110
|
end
|
89
111
|
|
112
|
+
test '"dest_project" is set' do
|
113
|
+
d = create_driver(%[
|
114
|
+
project project-test
|
115
|
+
topic topic-test
|
116
|
+
key key-test
|
117
|
+
dest_project dest-project-test
|
118
|
+
])
|
119
|
+
|
120
|
+
@publisher.publish.once
|
121
|
+
@pubsub_mock.topic("topic-test", {:project=>"dest-project-test"}).once { @publisher }
|
122
|
+
d.run(default_tag: "test") do
|
123
|
+
d.feed(@time, {"a" => "b"})
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
90
127
|
test '40x error occurred on connecting to Pub/Sub' do
|
91
128
|
d = create_driver
|
92
129
|
|
@@ -147,6 +184,13 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
|
|
147
184
|
@publisher = mock!
|
148
185
|
@pubsub_mock = mock!.topic(anything) { @publisher }
|
149
186
|
stub(Google::Cloud::Pubsub).new { @pubsub_mock }
|
187
|
+
|
188
|
+
@old_report_on_exception = Thread.report_on_exception
|
189
|
+
Thread.report_on_exception = false
|
190
|
+
end
|
191
|
+
|
192
|
+
teardown do
|
193
|
+
Thread.report_on_exception = @old_report_on_exception
|
150
194
|
end
|
151
195
|
|
152
196
|
test 'messages are divided into "max_messages"' do
|
@@ -222,5 +266,83 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
|
|
222
266
|
end
|
223
267
|
assert_equal({"tag" => 'test', "foo" => "bar"}, JSON.parse(MessagePack.unpack(d.formatted.first)[0]))
|
224
268
|
end
|
269
|
+
|
270
|
+
test 'with gzip compression' do
|
271
|
+
d = create_driver(%[
|
272
|
+
project project-test
|
273
|
+
topic topic-test
|
274
|
+
key key-test
|
275
|
+
compression gzip
|
276
|
+
])
|
277
|
+
test_msg = {"a" => "abc"}
|
278
|
+
|
279
|
+
@publisher.publish.once
|
280
|
+
d.run(default_tag: "test") do
|
281
|
+
d.feed(test_msg)
|
282
|
+
end
|
283
|
+
|
284
|
+
assert_equal(test_msg, JSON.parse(decompress(MessagePack.unpack(d.formatted.first)[0])))
|
285
|
+
end
|
286
|
+
|
287
|
+
test 'with attribute_keys' do
|
288
|
+
d = create_driver(%[
|
289
|
+
project project-test
|
290
|
+
topic topic-test
|
291
|
+
key key-test
|
292
|
+
attribute_keys a,b
|
293
|
+
])
|
294
|
+
|
295
|
+
@publisher.publish.once
|
296
|
+
d.run(default_tag: "test") do
|
297
|
+
d.feed(@time, {"a" => "foo", "b" => "bar", "c" => "baz"})
|
298
|
+
end
|
299
|
+
|
300
|
+
formatted = MessagePack.unpack(d.formatted[0])
|
301
|
+
message = JSON.parse(formatted[0])
|
302
|
+
assert_equal(message, {"c" => "baz"})
|
303
|
+
attributes = formatted[1]
|
304
|
+
assert_equal(attributes, {"a" => "foo", "b" => "bar"})
|
305
|
+
end
|
306
|
+
|
307
|
+
test 'with attribute_key_values' do
|
308
|
+
d = create_driver(%[
|
309
|
+
project project-test
|
310
|
+
topic topic-test
|
311
|
+
key key-test
|
312
|
+
attribute_key_values {"key1": "value1", "key2": "value2"}
|
313
|
+
])
|
314
|
+
|
315
|
+
@publisher.publish.once
|
316
|
+
d.run(default_tag: "test") do
|
317
|
+
d.feed(@time, {"a" => "foo", "b" => "bar"})
|
318
|
+
end
|
319
|
+
|
320
|
+
formatted = MessagePack.unpack(d.formatted[0])
|
321
|
+
message = JSON.parse(formatted[0])
|
322
|
+
assert_equal(message, {"a" => "foo", "b" => "bar"})
|
323
|
+
attributes = formatted[1]
|
324
|
+
assert_equal(attributes, {"key1" => "value1", "key2" => "value2"})
|
325
|
+
end
|
326
|
+
|
327
|
+
test 'with attribute_keys and attribute_key_values' do
|
328
|
+
d = create_driver(%[
|
329
|
+
project project-test
|
330
|
+
topic topic-test
|
331
|
+
key key-test
|
332
|
+
attribute_keys a
|
333
|
+
attribute_key_values {"key": "value"}
|
334
|
+
])
|
335
|
+
|
336
|
+
@publisher.publish.once
|
337
|
+
d.run(default_tag: "test") do
|
338
|
+
d.feed(@time, {"a" => "foo", "b" => "bar"})
|
339
|
+
end
|
340
|
+
|
341
|
+
formatted = MessagePack.unpack(d.formatted[0])
|
342
|
+
message = JSON.parse(formatted[0])
|
343
|
+
assert_equal(message, {"b" => "bar"})
|
344
|
+
attributes = formatted[1]
|
345
|
+
assert_equal(attributes, {"a" => "foo", "key" => "value"})
|
346
|
+
end
|
225
347
|
end
|
226
348
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-gcloud-pubsub-custom
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yoshihiro MIYAI
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-06-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -36,14 +36,14 @@ dependencies:
|
|
36
36
|
requirements:
|
37
37
|
- - "~>"
|
38
38
|
- !ruby/object:Gem::Version
|
39
|
-
version:
|
39
|
+
version: 2.3.1
|
40
40
|
type: :runtime
|
41
41
|
prerelease: false
|
42
42
|
version_requirements: !ruby/object:Gem::Requirement
|
43
43
|
requirements:
|
44
44
|
- - "~>"
|
45
45
|
- !ruby/object:Gem::Version
|
46
|
-
version:
|
46
|
+
version: 2.3.1
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
48
|
name: bundler
|
49
49
|
requirement: !ruby/object:Gem::Requirement
|
@@ -106,8 +106,8 @@ executables: []
|
|
106
106
|
extensions: []
|
107
107
|
extra_rdoc_files: []
|
108
108
|
files:
|
109
|
+
- ".github/workflows/test.yml"
|
109
110
|
- ".gitignore"
|
110
|
-
- ".travis.yml"
|
111
111
|
- CHANGELOG.md
|
112
112
|
- Gemfile
|
113
113
|
- LICENSE
|
@@ -117,14 +117,14 @@ files:
|
|
117
117
|
- lib/fluent/plugin/gcloud_pubsub/client.rb
|
118
118
|
- lib/fluent/plugin/in_gcloud_pubsub.rb
|
119
119
|
- lib/fluent/plugin/out_gcloud_pubsub.rb
|
120
|
+
- test/helper.rb
|
120
121
|
- test/plugin/test_in_gcloud_pubsub.rb
|
121
122
|
- test/plugin/test_out_gcloud_pubsub.rb
|
122
|
-
- test/test_helper.rb
|
123
123
|
homepage: https://github.com/mia-0032/fluent-plugin-gcloud-pubsub-custom
|
124
124
|
licenses:
|
125
125
|
- MIT
|
126
126
|
metadata: {}
|
127
|
-
post_install_message:
|
127
|
+
post_install_message:
|
128
128
|
rdoc_options: []
|
129
129
|
require_paths:
|
130
130
|
- lib
|
@@ -139,12 +139,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
139
139
|
- !ruby/object:Gem::Version
|
140
140
|
version: '0'
|
141
141
|
requirements: []
|
142
|
-
|
143
|
-
|
144
|
-
signing_key:
|
142
|
+
rubygems_version: 3.1.2
|
143
|
+
signing_key:
|
145
144
|
specification_version: 4
|
146
145
|
summary: Google Cloud Pub/Sub input/output plugin for Fluentd event collector
|
147
146
|
test_files:
|
147
|
+
- test/helper.rb
|
148
148
|
- test/plugin/test_in_gcloud_pubsub.rb
|
149
149
|
- test/plugin/test_out_gcloud_pubsub.rb
|
150
|
-
- test/test_helper.rb
|
data/.travis.yml
DELETED
@@ -1,25 +0,0 @@
|
|
1
|
-
language: ruby
|
2
|
-
|
3
|
-
rvm:
|
4
|
-
- 2.1
|
5
|
-
- 2.2
|
6
|
-
- 2.3.6
|
7
|
-
- 2.4.3
|
8
|
-
- 2.5.0
|
9
|
-
- ruby-head
|
10
|
-
|
11
|
-
gemfile:
|
12
|
-
- Gemfile
|
13
|
-
|
14
|
-
branches:
|
15
|
-
only:
|
16
|
-
- master
|
17
|
-
|
18
|
-
before_install: gem update bundler
|
19
|
-
script: bundle exec rake test
|
20
|
-
|
21
|
-
sudo: false
|
22
|
-
|
23
|
-
matrix:
|
24
|
-
allow_failures:
|
25
|
-
- rvm: ruby-head
|