fluent-plugin-gcloud-pubsub-custom 0.1.4 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/README.md +10 -1
- data/fluent-plugin-gcloud-pubsub-custom.gemspec +1 -1
- data/lib/fluent/plugin/in_gcloud_pubsub.rb +100 -8
- data/test/plugin/test_in_gcloud_pubsub.rb +69 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 08c5555e386df16b94ce1a8db279f952893b3f3b
|
4
|
+
data.tar.gz: e6eee873e191628778e9793a4545baf4a9d90344
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: eb8869520add948a27084fd46c58740455432f35bc29210efc77e4578901f5b10762c2b24a56eede4df04bfa54b57eaa210de5d022a11bcd5555e925939849aa
|
7
|
+
data.tar.gz: 3d2ef099388aa1a0260061e3cc134d84c9326c29202414643a2fe3612ca27fed9697b763ddd2ead9c3f68aa3ef3adb85d55b1e00de77b1b6d79e035b16e452a7
|
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
@@ -89,6 +89,9 @@ Use `gcloud_pubsub` input plugin.
|
|
89
89
|
return_immediately true
|
90
90
|
pull_interval 0.5
|
91
91
|
format json
|
92
|
+
enable_rpc true
|
93
|
+
rpc_bind 0.0.0.0
|
94
|
+
rpc_port 24680
|
92
95
|
</source>
|
93
96
|
```
|
94
97
|
|
@@ -110,11 +113,17 @@ Use `gcloud_pubsub` input plugin.
|
|
110
113
|
- See maxMessages on https://cloud.google.com/pubsub/subscriber#receiving-pull-messages
|
111
114
|
- `return_immediately` (optional, default: `true`)
|
112
115
|
- See returnImmediately on https://cloud.google.com/pubsub/subscriber#receiving-pull-messages
|
113
|
-
- If `return_immediately` is `
|
116
|
+
- If `return_immediately` is `true` and pulling message is stopped by HTTP RPC, this plugin wait `pull_interval` each pull.
|
114
117
|
- `pull_interval` (optional, default: `5.0`)
|
115
118
|
- Pulling messages by intervals of specified seconds.
|
116
119
|
- `format` (optional, default: `json`)
|
117
120
|
- Set input format. See format section in http://docs.fluentd.org/articles/in_tail
|
121
|
+
- `enable_rpc` (optional, default: `false`)
|
122
|
+
- If `true` is specified, HTTP RPC to stop or start pulling message is enabled.
|
123
|
+
- `rpc_bind` (optional, default: `0.0.0.0`)
|
124
|
+
- Bind IP address for HTTP RPC.
|
125
|
+
- `rpc_port` (optional, default: `24680`)
|
126
|
+
- Port for HTTP RPC.
|
118
127
|
|
119
128
|
## Contributing
|
120
129
|
|
@@ -7,7 +7,7 @@ Gem::Specification.new do |gem|
|
|
7
7
|
gem.license = "MIT"
|
8
8
|
gem.homepage = "https://github.com/mia-0032/fluent-plugin-gcloud-pubsub-custom"
|
9
9
|
gem.summary = gem.description
|
10
|
-
gem.version = "0.
|
10
|
+
gem.version = "0.2.0"
|
11
11
|
gem.authors = ["Yoshihiro MIYAI"]
|
12
12
|
gem.email = "msparrow17@gmail.com"
|
13
13
|
gem.has_rdoc = false
|
@@ -1,3 +1,6 @@
|
|
1
|
+
require 'json'
|
2
|
+
require 'webrick'
|
3
|
+
|
1
4
|
require 'fluent/input'
|
2
5
|
require 'fluent/parser'
|
3
6
|
|
@@ -8,14 +11,18 @@ module Fluent
|
|
8
11
|
Fluent::Plugin.register_input('gcloud_pubsub', self)
|
9
12
|
|
10
13
|
config_param :tag, :string
|
11
|
-
config_param :project, :string, :
|
12
|
-
config_param :key, :string, :
|
14
|
+
config_param :project, :string, default: nil
|
15
|
+
config_param :key, :string, default: nil
|
13
16
|
config_param :topic, :string
|
14
17
|
config_param :subscription, :string
|
15
|
-
config_param :pull_interval, :float, :
|
16
|
-
config_param :max_messages, :integer, :
|
17
|
-
config_param :return_immediately, :bool, :
|
18
|
-
config_param :format, :string, :
|
18
|
+
config_param :pull_interval, :float, default: 5.0
|
19
|
+
config_param :max_messages, :integer, default: 100
|
20
|
+
config_param :return_immediately, :bool, default: true
|
21
|
+
config_param :format, :string, default: 'json'
|
22
|
+
# for HTTP RPC
|
23
|
+
config_param :enable_rpc, :bool, default: false
|
24
|
+
config_param :rpc_bind, :string, default: '0.0.0.0'
|
25
|
+
config_param :rpc_port, :integer, default: 24680
|
19
26
|
|
20
27
|
unless method_defined?(:log)
|
21
28
|
define_method("log") { $log }
|
@@ -25,33 +32,118 @@ module Fluent
|
|
25
32
|
define_method("router") { Fluent::Engine }
|
26
33
|
end
|
27
34
|
|
35
|
+
class RPCServlet < WEBrick::HTTPServlet::AbstractServlet
|
36
|
+
class Error < StandardError; end
|
37
|
+
|
38
|
+
def initialize(server, plugin)
|
39
|
+
super
|
40
|
+
@plugin = plugin
|
41
|
+
end
|
42
|
+
|
43
|
+
def do_GET(req, res)
|
44
|
+
begin
|
45
|
+
code, header, body = process(req, res)
|
46
|
+
rescue
|
47
|
+
code, header, body = render_json(500, {
|
48
|
+
'ok' => false,
|
49
|
+
'message' => 'Internal Server Error',
|
50
|
+
'error' => "#{$!}",
|
51
|
+
'backtrace'=> $!.backtrace
|
52
|
+
})
|
53
|
+
end
|
54
|
+
|
55
|
+
res.status = code
|
56
|
+
header.each_pair {|k,v|
|
57
|
+
res[k] = v
|
58
|
+
}
|
59
|
+
res.body = body
|
60
|
+
end
|
61
|
+
|
62
|
+
def render_json(code, obj)
|
63
|
+
[code, {'Content-Type' => 'application/json'}, obj.to_json]
|
64
|
+
end
|
65
|
+
|
66
|
+
def process(req, res)
|
67
|
+
case req.path_info
|
68
|
+
when '/stop'
|
69
|
+
@plugin.stop_pull
|
70
|
+
when '/start'
|
71
|
+
@plugin.start_pull
|
72
|
+
else
|
73
|
+
raise Error.new "Invalid path_info: #{req.path_info}"
|
74
|
+
end
|
75
|
+
render_json(200, {'ok' => true})
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
28
79
|
def configure(conf)
|
29
80
|
super
|
81
|
+
@rpc_srv = nil
|
82
|
+
@rpc_thread = nil
|
83
|
+
@stop_pull = false
|
84
|
+
|
30
85
|
@parser = Plugin.new_parser(@format)
|
31
86
|
@parser.configure(conf)
|
32
87
|
end
|
33
88
|
|
34
89
|
def start
|
35
90
|
super
|
91
|
+
start_rpc if @enable_rpc
|
92
|
+
|
36
93
|
@subscriber = Fluent::GcloudPubSub::Subscriber.new @project, @key, @topic, @subscription
|
37
94
|
log.debug "connected subscription:#{@subscription} in project #{@project}"
|
95
|
+
|
38
96
|
@stop_subscribing = false
|
39
97
|
@subscribe_thread = Thread.new(&method(:subscribe))
|
40
98
|
end
|
41
99
|
|
42
100
|
def shutdown
|
43
101
|
super
|
102
|
+
if @rpc_srv
|
103
|
+
@rpc_srv.shutdown
|
104
|
+
@rpc_srv = nil
|
105
|
+
end
|
106
|
+
if @rpc_thread
|
107
|
+
@rpc_thread.join
|
108
|
+
@rpc_thread = nil
|
109
|
+
end
|
44
110
|
@stop_subscribing = true
|
45
111
|
@subscribe_thread.join
|
46
112
|
end
|
47
113
|
|
114
|
+
def stop_pull
|
115
|
+
@stop_pull = true
|
116
|
+
log.info "stop pull from subscription:#{@subscription}"
|
117
|
+
end
|
118
|
+
|
119
|
+
def start_pull
|
120
|
+
@stop_pull = false
|
121
|
+
log.info "start pull from subscription:#{@subscription}"
|
122
|
+
end
|
123
|
+
|
48
124
|
private
|
49
125
|
|
126
|
+
def start_rpc
|
127
|
+
log.info "listening http rpc server on http://#{@rpc_bind}:#{@rpc_port}/"
|
128
|
+
@rpc_srv = WEBrick::HTTPServer.new(
|
129
|
+
{
|
130
|
+
BindAddress: @rpc_bind,
|
131
|
+
Port: @rpc_port,
|
132
|
+
Logger: WEBrick::Log.new(STDERR, WEBrick::Log::FATAL),
|
133
|
+
AccessLog: []
|
134
|
+
}
|
135
|
+
)
|
136
|
+
@rpc_srv.mount('/api/in_gcloud_pubsub/pull/', RPCServlet, self)
|
137
|
+
@rpc_thread = Thread.new {
|
138
|
+
@rpc_srv.start
|
139
|
+
}
|
140
|
+
end
|
141
|
+
|
50
142
|
def subscribe
|
51
143
|
until @stop_subscribing
|
52
|
-
_subscribe
|
144
|
+
_subscribe unless @stop_pull
|
53
145
|
|
54
|
-
if @return_immediately
|
146
|
+
if @return_immediately || @stop_pull
|
55
147
|
sleep @pull_interval
|
56
148
|
end
|
57
149
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
require 'net/http'
|
2
|
+
|
1
3
|
require_relative "../test_helper"
|
2
4
|
|
3
5
|
class GcloudPubSubInputTest < Test::Unit::TestCase
|
@@ -10,10 +12,19 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
|
|
10
12
|
format json
|
11
13
|
]
|
12
14
|
|
15
|
+
DEFAULT_HOST = '127.0.0.1'
|
16
|
+
DEFAULT_PORT = 24680
|
17
|
+
|
13
18
|
def create_driver(conf=CONFIG)
|
14
19
|
Fluent::Test::InputTestDriver.new(Fluent::GcloudPubSubInput).configure(conf)
|
15
20
|
end
|
16
21
|
|
22
|
+
def http_get(path)
|
23
|
+
http = Net::HTTP.new(DEFAULT_HOST, DEFAULT_PORT)
|
24
|
+
req = Net::HTTP::Get.new(path, {'Content-Type' => 'application/x-www-form-urlencoded'})
|
25
|
+
http.request(req)
|
26
|
+
end
|
27
|
+
|
17
28
|
setup do
|
18
29
|
Fluent::Test.setup
|
19
30
|
end
|
@@ -30,6 +41,9 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
|
|
30
41
|
return_immediately true
|
31
42
|
pull_interval 2
|
32
43
|
format ltsv
|
44
|
+
enable_rpc true
|
45
|
+
rpc_bind 127.0.0.1
|
46
|
+
rpc_port 24681
|
33
47
|
])
|
34
48
|
|
35
49
|
assert_equal('test', d.instance.tag)
|
@@ -41,6 +55,9 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
|
|
41
55
|
assert_equal(1000, d.instance.max_messages)
|
42
56
|
assert_equal(true, d.instance.return_immediately)
|
43
57
|
assert_equal('ltsv', d.instance.format)
|
58
|
+
assert_equal(true, d.instance.enable_rpc)
|
59
|
+
assert_equal('127.0.0.1', d.instance.rpc_bind)
|
60
|
+
assert_equal(24681, d.instance.rpc_port)
|
44
61
|
end
|
45
62
|
|
46
63
|
test 'default values are configured' do
|
@@ -49,6 +66,9 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
|
|
49
66
|
assert_equal(100, d.instance.max_messages)
|
50
67
|
assert_equal(true, d.instance.return_immediately)
|
51
68
|
assert_equal('json', d.instance.format)
|
69
|
+
assert_equal(false, d.instance.enable_rpc)
|
70
|
+
assert_equal('0.0.0.0', d.instance.rpc_bind)
|
71
|
+
assert_equal(24680, d.instance.rpc_port)
|
52
72
|
end
|
53
73
|
end
|
54
74
|
|
@@ -139,5 +159,54 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
|
|
139
159
|
assert_equal(0.5, d.instance.pull_interval)
|
140
160
|
assert_equal(true, d.emits.empty?)
|
141
161
|
end
|
162
|
+
|
163
|
+
test 'stop by http rpc' do
|
164
|
+
messages = Array.new(1, DummyMessage.new)
|
165
|
+
@subscriber.pull(immediate: true, max: 100).once { messages }
|
166
|
+
@subscriber.acknowledge(messages).once
|
167
|
+
|
168
|
+
d = create_driver("#{CONFIG}\npull_interval 1.0\nenable_rpc true")
|
169
|
+
assert_equal(false, d.instance.instance_variable_get(:@stop_pull))
|
170
|
+
|
171
|
+
d.run {
|
172
|
+
http_get('/api/in_gcloud_pubsub/pull/stop')
|
173
|
+
sleep 0.75
|
174
|
+
# d.run sleeps 0.5 sec
|
175
|
+
}
|
176
|
+
emits = d.emits
|
177
|
+
|
178
|
+
assert_equal(1, emits.length)
|
179
|
+
assert_equal(true, d.instance.instance_variable_get(:@stop_pull))
|
180
|
+
|
181
|
+
emits.each do |tag, time, record|
|
182
|
+
assert_equal("test", tag)
|
183
|
+
assert_equal({"foo" => "bar"}, record)
|
184
|
+
end
|
185
|
+
end
|
186
|
+
|
187
|
+
test 'start by http rpc' do
|
188
|
+
messages = Array.new(1, DummyMessage.new)
|
189
|
+
@subscriber.pull(immediate: true, max: 100).at_least(1) { messages }
|
190
|
+
@subscriber.acknowledge(messages).at_least(1)
|
191
|
+
|
192
|
+
d = create_driver("#{CONFIG}\npull_interval 1.0\nenable_rpc true")
|
193
|
+
d.instance.stop_pull
|
194
|
+
assert_equal(true, d.instance.instance_variable_get(:@stop_pull))
|
195
|
+
|
196
|
+
d.run {
|
197
|
+
http_get('/api/in_gcloud_pubsub/pull/start')
|
198
|
+
sleep 0.75
|
199
|
+
# d.run sleeps 0.5 sec
|
200
|
+
}
|
201
|
+
emits = d.emits
|
202
|
+
|
203
|
+
assert_equal(true, emits.length > 0)
|
204
|
+
assert_equal(false, d.instance.instance_variable_get(:@stop_pull))
|
205
|
+
|
206
|
+
emits.each do |tag, time, record|
|
207
|
+
assert_equal("test", tag)
|
208
|
+
assert_equal({"foo" => "bar"}, record)
|
209
|
+
end
|
210
|
+
end
|
142
211
|
end
|
143
212
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-gcloud-pubsub-custom
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yoshihiro MIYAI
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-10-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|