fluent-plugin-gcloud-pubsub-custom 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +13 -0
- data/README.md +2 -12
- data/fluent-plugin-gcloud-pubsub-custom.gemspec +1 -1
- data/lib/fluent/plugin/in_gcloud_pubsub.rb +32 -20
- data/test/plugin/test_in_gcloud_pubsub.rb +89 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1c24f666eacd9d35bb07d5eb1aad175fd0c549b3
|
4
|
+
data.tar.gz: 888acd31eb8ca1f74c689c39b21f915f8b9034ab
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f17fc4b9578f08f4181501a0be689db36dcad82b0474b06646f99edc34f1850eb2273efce945ce098a90257cdd9ded87449a93dea5fc0ca5449b30ba3ddc843f
|
7
|
+
data.tar.gz: 8a937780624f4c02ff5f5d39bc6950a28d8561b1b5494cffb19d0669a713f82f8b7b4cf2789c727ca450d933c9a483275781301aa210d6549e4cb485523505f1
|
data/CHANGELOG.md
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
## ChangeLog
|
2
|
+
|
3
|
+
- Release 0.1.3 - 2016/09/17
|
4
|
+
- Fix error handling and add debug logging in input plugin
|
5
|
+
- Release 0.1.2 - 2016/09/11
|
6
|
+
- Change default max message size and add debug message
|
7
|
+
- Release 0.1.1 - 2016/08/27
|
8
|
+
- Bump up google-cloud-pubsub (gcloud-ruby) to 0.20
|
9
|
+
- Release 0.1.0 - 2016/08/22
|
10
|
+
- Use formatter / parser plugin and add format configuration
|
11
|
+
- Bump up gcloud-ruby to 0.12
|
12
|
+
- Remove dependency on lightening buffer
|
13
|
+
- Fix error caused by Pub/Sub quotas
|
data/README.md
CHANGED
@@ -41,7 +41,7 @@ Use `gcloud_pubsub` output plugin.
|
|
41
41
|
topic <YOUR TOPIC>
|
42
42
|
autocreate_topic false
|
43
43
|
max_messages 1000
|
44
|
-
max_total_size
|
44
|
+
max_total_size 9800000
|
45
45
|
buffer_type file
|
46
46
|
buffer_path /path/to/your/buffer
|
47
47
|
flush_interval 1s
|
@@ -110,22 +110,12 @@ Use `gcloud_pubsub` input plugin.
|
|
110
110
|
- See maxMessages on https://cloud.google.com/pubsub/subscriber#receiving-pull-messages
|
111
111
|
- `return_immediately` (optional, default: `true`)
|
112
112
|
- See returnImmediately on https://cloud.google.com/pubsub/subscriber#receiving-pull-messages
|
113
|
-
- If `return_immediately` is `
|
113
|
+
- If `return_immediately` is `false`, this plugin ignore `pull_interval`.
|
114
114
|
- `pull_interval` (optional, default: `5`)
|
115
115
|
- Pulling messages by intervals of specified seconds.
|
116
116
|
- `format` (optional, default: `json`)
|
117
117
|
- Set input format. See format section in http://docs.fluentd.org/articles/in_tail
|
118
118
|
|
119
|
-
## ChangeLog
|
120
|
-
|
121
|
-
- Release 0.1.1 - 2016/08/27
|
122
|
-
- Bump up google-cloud-pubsub (gcloud-ruby) to 0.20
|
123
|
-
- Release 0.1.0 - 2016/08/22
|
124
|
-
- Use formatter / parser plugin and add format configuration
|
125
|
-
- Bump up gcloud-ruby to 0.12
|
126
|
-
- Remove dependency on lightening buffer
|
127
|
-
- Fix error caused by Pub/Sub quotas
|
128
|
-
|
129
119
|
## Contributing
|
130
120
|
|
131
121
|
1. Fork it
|
@@ -7,7 +7,7 @@ Gem::Specification.new do |gem|
|
|
7
7
|
gem.license = "MIT"
|
8
8
|
gem.homepage = "https://github.com/mia-0032/fluent-plugin-gcloud-pubsub-custom"
|
9
9
|
gem.summary = gem.description
|
10
|
-
gem.version = "0.1.
|
10
|
+
gem.version = "0.1.3"
|
11
11
|
gem.authors = ["Yoshihiro MIYAI"]
|
12
12
|
gem.email = "msparrow17@gmail.com"
|
13
13
|
gem.has_rdoc = false
|
@@ -49,28 +49,40 @@ module Fluent
|
|
49
49
|
|
50
50
|
def subscribe
|
51
51
|
until @stop_subscribing
|
52
|
-
|
53
|
-
|
54
|
-
if messages.length > 0
|
55
|
-
es = parse_messages(messages)
|
56
|
-
unless es.empty?
|
57
|
-
begin
|
58
|
-
router.emit_stream(@tag, es)
|
59
|
-
rescue
|
60
|
-
# ignore errors. Engine shows logs and backtraces.
|
61
|
-
end
|
62
|
-
@subscriber.acknowledge messages
|
63
|
-
log.debug "#{messages.length} message(s) processed"
|
64
|
-
end
|
65
|
-
end
|
52
|
+
_subscribe
|
66
53
|
|
67
54
|
if @return_immediately
|
68
55
|
sleep @pull_interval
|
69
56
|
end
|
70
57
|
end
|
71
|
-
rescue
|
72
|
-
log.error "unexpected error", :
|
73
|
-
log.error_backtrace
|
58
|
+
rescue => ex
|
59
|
+
log.error "unexpected error", error_message: ex.to_s, error_class: ex.class.to_s
|
60
|
+
log.error_backtrace ex.backtrace
|
61
|
+
end
|
62
|
+
|
63
|
+
def _subscribe
|
64
|
+
messages = @subscriber.pull @return_immediately, @max_messages
|
65
|
+
if messages.length == 0
|
66
|
+
log.debug "no messages are pulled"
|
67
|
+
return
|
68
|
+
end
|
69
|
+
|
70
|
+
es = parse_messages(messages)
|
71
|
+
if es.empty?
|
72
|
+
log.warn "#{messages.length} message(s) are pulled, but no messages are parsed"
|
73
|
+
return
|
74
|
+
end
|
75
|
+
|
76
|
+
begin
|
77
|
+
router.emit_stream(@tag, es)
|
78
|
+
rescue
|
79
|
+
# ignore errors. Engine shows logs and backtraces.
|
80
|
+
end
|
81
|
+
@subscriber.acknowledge messages
|
82
|
+
log.debug "#{messages.length} message(s) processed"
|
83
|
+
rescue => ex
|
84
|
+
log.error "unexpected error", error_message: ex.to_s, error_class: ex.class.to_s
|
85
|
+
log.error_backtrace ex.backtrace
|
74
86
|
end
|
75
87
|
|
76
88
|
def parse_messages(messages)
|
@@ -90,9 +102,9 @@ module Fluent
|
|
90
102
|
log.warn "pattern not match: #{line.inspect}"
|
91
103
|
end
|
92
104
|
}
|
93
|
-
rescue =>
|
94
|
-
log.warn line.dump, :
|
95
|
-
log.
|
105
|
+
rescue => ex
|
106
|
+
log.warn line.dump, error_message: ex.to_s, error_class: ex.class.to_s
|
107
|
+
log.warn_backtrace ex.backtrace
|
96
108
|
end
|
97
109
|
end
|
98
110
|
end
|
@@ -4,6 +4,7 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
|
|
4
4
|
CONFIG = %[
|
5
5
|
tag test
|
6
6
|
project project-test
|
7
|
+
topic topic-test
|
7
8
|
subscription subscription-test
|
8
9
|
key key-test
|
9
10
|
format json
|
@@ -44,11 +45,98 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
|
|
44
45
|
|
45
46
|
test 'default values are configured' do
|
46
47
|
d = create_driver
|
47
|
-
assert_equal(nil, d.instance.topic)
|
48
48
|
assert_equal(5, d.instance.pull_interval)
|
49
49
|
assert_equal(100, d.instance.max_messages)
|
50
50
|
assert_equal(true, d.instance.return_immediately)
|
51
51
|
assert_equal('json', d.instance.format)
|
52
52
|
end
|
53
53
|
end
|
54
|
+
|
55
|
+
sub_test_case 'emit' do
|
56
|
+
class DummyMsgData
|
57
|
+
def data
|
58
|
+
return '{"foo": "bar"}'
|
59
|
+
end
|
60
|
+
end
|
61
|
+
class DummyMessage
|
62
|
+
def message
|
63
|
+
DummyMsgData.new
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
setup do
|
68
|
+
@subscriber = mock!
|
69
|
+
@topic_mock = mock!.subscription('subscription-test') { @subscriber }
|
70
|
+
@pubsub_mock = mock!.topic('topic-test') { @topic_mock }
|
71
|
+
@gcloud_mock = mock!.pubsub { @pubsub_mock }
|
72
|
+
stub(Google::Cloud).new { @gcloud_mock }
|
73
|
+
end
|
74
|
+
|
75
|
+
test 'empty' do
|
76
|
+
@subscriber.pull(immediate: true, max: 100).once { [] }
|
77
|
+
@subscriber.acknowledge.times(0)
|
78
|
+
|
79
|
+
d = create_driver
|
80
|
+
d.run {
|
81
|
+
# d.run sleeps 0.5 sec
|
82
|
+
}
|
83
|
+
|
84
|
+
assert_equal(true, d.emits.empty?)
|
85
|
+
end
|
86
|
+
|
87
|
+
test 'simple' do
|
88
|
+
messages = Array.new(1, DummyMessage.new)
|
89
|
+
@subscriber.pull(immediate: true, max: 100).once { messages }
|
90
|
+
@subscriber.acknowledge(messages).once
|
91
|
+
|
92
|
+
d = create_driver
|
93
|
+
d.run {
|
94
|
+
# d.run sleeps 0.5 sec
|
95
|
+
}
|
96
|
+
emits = d.emits
|
97
|
+
|
98
|
+
assert_equal(1, emits.length)
|
99
|
+
emits.each do |tag, time, record|
|
100
|
+
assert_equal("test", tag)
|
101
|
+
assert_equal({"foo" => "bar"}, record)
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
test 'invalid messages' do
|
106
|
+
class DummyInvalidMsgData
|
107
|
+
def data
|
108
|
+
return 'foo:bar'
|
109
|
+
end
|
110
|
+
end
|
111
|
+
class DummyInvalidMessage
|
112
|
+
def message
|
113
|
+
DummyInvalidMsgData.new
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
messages = Array.new(1, DummyInvalidMessage.new)
|
118
|
+
@subscriber.pull(immediate: true, max: 100).once { messages }
|
119
|
+
@subscriber.acknowledge.times(0)
|
120
|
+
|
121
|
+
d = create_driver
|
122
|
+
d.run {
|
123
|
+
# d.run sleeps 0.5 sec
|
124
|
+
}
|
125
|
+
assert_equal(true, d.emits.empty?)
|
126
|
+
end
|
127
|
+
|
128
|
+
test 'retry if raised error' do
|
129
|
+
class UnknownError < StandardError
|
130
|
+
end
|
131
|
+
@subscriber.pull(immediate: true, max: 100).twice { raise UnknownError.new('test') }
|
132
|
+
@subscriber.acknowledge.times(0)
|
133
|
+
|
134
|
+
d = create_driver(CONFIG + 'pull_interval 1')
|
135
|
+
d.run {
|
136
|
+
sleep 1 # + 0.5s
|
137
|
+
}
|
138
|
+
|
139
|
+
assert_equal(true, d.emits.empty?)
|
140
|
+
end
|
141
|
+
end
|
54
142
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-gcloud-pubsub-custom
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yoshihiro MIYAI
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-09-
|
11
|
+
date: 2016-09-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -108,6 +108,7 @@ extra_rdoc_files: []
|
|
108
108
|
files:
|
109
109
|
- ".gitignore"
|
110
110
|
- ".travis.yml"
|
111
|
+
- CHANGELOG.md
|
111
112
|
- Gemfile
|
112
113
|
- Gemfile.v0.12
|
113
114
|
- LICENSE
|