fluent-plugin-gcloud-pubsub-custom 1.0.3 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/README.md +1 -0
- data/fluent-plugin-gcloud-pubsub-custom.gemspec +1 -1
- data/lib/fluent/plugin/gcloud_pubsub/client.rb +20 -11
- data/lib/fluent/plugin/out_gcloud_pubsub.rb +9 -7
- data/test/plugin/test_out_gcloud_pubsub.rb +32 -23
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a55a4279110490b7ac03da92af18e269d92d5d75af86d294a130acd4fb440364
|
4
|
+
data.tar.gz: cfa6ba338f70ae9826fc71e3733dfb6c42175f55ae7015db38fc1703c643ed33
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 00aedffe49e51e556d3b8ef86965b67614aab97d7bb06ef1032d8b826ff8d33ca85383d822430ff8c30f6638372bd2a7f2544d86d6745b9b4f89395af384e226
|
7
|
+
data.tar.gz: 0b4c66aa2418626728eebb048a4e57f1f49915d609bbf028c7979ca0ce79168e36a3d413d79301faf5ce4fb998162ae75e508b784f6764a0d08aad76a6af6d45
|
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
@@ -80,6 +80,7 @@ Use `gcloud_pubsub` output plugin.
|
|
80
80
|
- You can also use environment variable such as `GCLOUD_KEYFILE`.
|
81
81
|
- `topic` (required)
|
82
82
|
- Set topic name to publish.
|
83
|
+
- You can use placeholder in this param. See: https://docs.fluentd.org/v1.0/articles/buffer-section
|
83
84
|
- `autocreate_topic` (optional, default: `false`)
|
84
85
|
- If set to `true`, specified topic will be created when it doesn't exist.
|
85
86
|
- `max_messages` (optional, default: `1000`)
|
@@ -7,7 +7,7 @@ Gem::Specification.new do |gem|
|
|
7
7
|
gem.license = "MIT"
|
8
8
|
gem.homepage = "https://github.com/mia-0032/fluent-plugin-gcloud-pubsub-custom"
|
9
9
|
gem.summary = gem.description
|
10
|
-
gem.version = "1.0
|
10
|
+
gem.version = "1.1.0"
|
11
11
|
gem.authors = ["Yoshihiro MIYAI"]
|
12
12
|
gem.email = "msparrow17@gmail.com"
|
13
13
|
gem.has_rdoc = false
|
@@ -8,20 +8,29 @@ module Fluent
|
|
8
8
|
end
|
9
9
|
|
10
10
|
class Publisher
|
11
|
-
def initialize(project, key,
|
12
|
-
pubsub = Google::Cloud::Pubsub.new project_id: project, credentials: key
|
13
|
-
@
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
11
|
+
def initialize(project, key, autocreate_topic)
|
12
|
+
@pubsub = Google::Cloud::Pubsub.new project_id: project, credentials: key
|
13
|
+
@autocreate_topic = autocreate_topic
|
14
|
+
@topics = {}
|
15
|
+
end
|
16
|
+
|
17
|
+
def topic(topic_name)
|
18
|
+
return @topics[topic_name] if @topics.has_key? topic_name
|
19
|
+
|
20
|
+
client = @pubsub.topic topic_name
|
21
|
+
if client.nil? && @autocreate_topic
|
22
|
+
client = @pubsub.create_topic topic_name
|
23
|
+
end
|
24
|
+
if client.nil?
|
25
|
+
raise Error.new "topic:#{topic_name} does not exist."
|
20
26
|
end
|
27
|
+
|
28
|
+
@topics[topic_name] = client
|
29
|
+
client
|
21
30
|
end
|
22
31
|
|
23
|
-
def publish(messages)
|
24
|
-
|
32
|
+
def publish(topic_name, messages)
|
33
|
+
topic(topic_name).publish do |batch|
|
25
34
|
messages.each do |m|
|
26
35
|
batch.publish m
|
27
36
|
end
|
@@ -37,13 +37,13 @@ module Fluent::Plugin
|
|
37
37
|
def configure(conf)
|
38
38
|
compat_parameters_convert(conf, :buffer, :formatter)
|
39
39
|
super
|
40
|
+
placeholder_validate!(:topic, @topic)
|
40
41
|
@formatter = formatter_create
|
41
42
|
end
|
42
43
|
|
43
44
|
def start
|
44
45
|
super
|
45
|
-
@publisher = Fluent::GcloudPubSub::Publisher.new @project, @key, @
|
46
|
-
log.debug "connected topic:#{@topic} in project #{@project}"
|
46
|
+
@publisher = Fluent::GcloudPubSub::Publisher.new @project, @key, @autocreate_topic
|
47
47
|
end
|
48
48
|
|
49
49
|
def format(tag, time, record)
|
@@ -59,6 +59,8 @@ module Fluent::Plugin
|
|
59
59
|
end
|
60
60
|
|
61
61
|
def write(chunk)
|
62
|
+
topic = extract_placeholders(@topic, chunk.metadata)
|
63
|
+
|
62
64
|
messages = []
|
63
65
|
size = 0
|
64
66
|
|
@@ -68,7 +70,7 @@ module Fluent::Plugin
|
|
68
70
|
next
|
69
71
|
end
|
70
72
|
if messages.length + 1 > @max_messages || size + msg.bytesize > @max_total_size
|
71
|
-
publish messages
|
73
|
+
publish(topic, messages)
|
72
74
|
messages = []
|
73
75
|
size = 0
|
74
76
|
end
|
@@ -77,7 +79,7 @@ module Fluent::Plugin
|
|
77
79
|
end
|
78
80
|
|
79
81
|
if messages.length > 0
|
80
|
-
publish messages
|
82
|
+
publish(topic, messages)
|
81
83
|
end
|
82
84
|
rescue Fluent::GcloudPubSub::RetryableError => ex
|
83
85
|
log.warn "Retryable error occurs. Fluentd will retry.", error_message: ex.to_s, error_class: ex.class.to_s
|
@@ -90,9 +92,9 @@ module Fluent::Plugin
|
|
90
92
|
|
91
93
|
private
|
92
94
|
|
93
|
-
def publish(messages)
|
94
|
-
log.debug "send message topic:#{
|
95
|
-
@publisher.publish messages
|
95
|
+
def publish(topic, messages)
|
96
|
+
log.debug "send message topic:#{topic} length:#{messages.length} size:#{messages.map(&:bytesize).inject(:+)}"
|
97
|
+
@publisher.publish(topic, messages)
|
96
98
|
end
|
97
99
|
end
|
98
100
|
end
|
@@ -22,6 +22,10 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
|
|
22
22
|
Fluent::Test.setup
|
23
23
|
end
|
24
24
|
|
25
|
+
setup do
|
26
|
+
@time = event_time('2016-07-09 11:12:13 UTC')
|
27
|
+
end
|
28
|
+
|
25
29
|
sub_test_case 'configure' do
|
26
30
|
test 'default values are configured' do
|
27
31
|
d = create_driver(%[
|
@@ -75,9 +79,12 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
|
|
75
79
|
autocreate_topic true
|
76
80
|
])
|
77
81
|
|
82
|
+
@publisher.publish.once
|
78
83
|
@pubsub_mock.topic("topic-test").once { nil }
|
79
84
|
@pubsub_mock.create_topic("topic-test").once { @publisher }
|
80
|
-
d.run
|
85
|
+
d.run(default_tag: "test") do
|
86
|
+
d.feed(@time, {"a" => "b"})
|
87
|
+
end
|
81
88
|
end
|
82
89
|
|
83
90
|
test '40x error occurred on connecting to Pub/Sub' do
|
@@ -88,7 +95,9 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
|
|
88
95
|
end
|
89
96
|
|
90
97
|
assert_raise Google::Cloud::NotFoundError do
|
91
|
-
d.run
|
98
|
+
d.run(default_tag: "test") do
|
99
|
+
d.feed(@time, {"a" => "b"})
|
100
|
+
end
|
92
101
|
end
|
93
102
|
end
|
94
103
|
|
@@ -99,8 +108,10 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
|
|
99
108
|
raise Google::Cloud::UnavailableError.new('TEST')
|
100
109
|
end
|
101
110
|
|
102
|
-
assert_raise
|
103
|
-
d.run
|
111
|
+
assert_raise Fluent::GcloudPubSub::RetryableError do
|
112
|
+
d.run(default_tag: "test") do
|
113
|
+
d.feed(@time, {"a" => "b"})
|
114
|
+
end
|
104
115
|
end
|
105
116
|
end
|
106
117
|
|
@@ -110,7 +121,23 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
|
|
110
121
|
@pubsub_mock.topic('topic-test').once { nil }
|
111
122
|
|
112
123
|
assert_raise Fluent::GcloudPubSub::Error do
|
113
|
-
d.run
|
124
|
+
d.run(default_tag: "test") do
|
125
|
+
d.feed(@time, {"a" => "b"})
|
126
|
+
end
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
test 'messages exceeding "max_message_size" are not published' do
|
131
|
+
d = create_driver(%[
|
132
|
+
project project-test
|
133
|
+
topic topic-test
|
134
|
+
key key-test
|
135
|
+
max_message_size 1000
|
136
|
+
])
|
137
|
+
|
138
|
+
@publisher.publish.times(0)
|
139
|
+
d.run(default_tag: "test") do
|
140
|
+
d.feed(@time, {"a" => "a" * 1000})
|
114
141
|
end
|
115
142
|
end
|
116
143
|
end
|
@@ -122,10 +149,6 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
|
|
122
149
|
stub(Google::Cloud::Pubsub).new { @pubsub_mock }
|
123
150
|
end
|
124
151
|
|
125
|
-
setup do
|
126
|
-
@time = event_time('2016-07-09 11:12:13 UTC')
|
127
|
-
end
|
128
|
-
|
129
152
|
test 'messages are divided into "max_messages"' do
|
130
153
|
d = create_driver
|
131
154
|
@publisher.publish.times(2)
|
@@ -155,20 +178,6 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
|
|
155
178
|
end
|
156
179
|
end
|
157
180
|
|
158
|
-
test 'messages exceeding "max_message_size" are not published' do
|
159
|
-
d = create_driver(%[
|
160
|
-
project project-test
|
161
|
-
topic topic-test
|
162
|
-
key key-test
|
163
|
-
max_message_size 1000
|
164
|
-
])
|
165
|
-
|
166
|
-
@publisher.publish.times(0)
|
167
|
-
d.run(default_tag: "test") do
|
168
|
-
d.feed(@time, {"a" => "a" * 1000})
|
169
|
-
end
|
170
|
-
end
|
171
|
-
|
172
181
|
test 'accept "ASCII-8BIT" encoded multibyte strings' do
|
173
182
|
# on fluentd v0.14, all strings treated as "ASCII-8BIT" except specified encoding.
|
174
183
|
d = create_driver
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-gcloud-pubsub-custom
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yoshihiro MIYAI
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-03-
|
11
|
+
date: 2018-03-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|