fluent-plugin-gcloud-pubsub-custom 0.1.2 → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 2fd80971b144486ed5200c6e8a3680275f3b2af0
4
- data.tar.gz: fcddbba7b68cb1ad6ce4fc72b9bce3df3fce2688
3
+ metadata.gz: 1c24f666eacd9d35bb07d5eb1aad175fd0c549b3
4
+ data.tar.gz: 888acd31eb8ca1f74c689c39b21f915f8b9034ab
5
5
  SHA512:
6
- metadata.gz: bb4c1f75828bb55177f1a35a159e61925c45afb6cb6ecc07d5d6c29dd4215b002289beb293dd2cdf94255444f44469fecd4ede53571b291e0c5d0d732460b964
7
- data.tar.gz: 4526515a2b03759ca0c306288d7940a0fdac4b12f80809fed493e10a08c64d9edd43f7ae29a8ff41788dfcc5d9aaafc45a28fb420e54038e02c4efa4e376ff0c
6
+ metadata.gz: f17fc4b9578f08f4181501a0be689db36dcad82b0474b06646f99edc34f1850eb2273efce945ce098a90257cdd9ded87449a93dea5fc0ca5449b30ba3ddc843f
7
+ data.tar.gz: 8a937780624f4c02ff5f5d39bc6950a28d8561b1b5494cffb19d0669a713f82f8b7b4cf2789c727ca450d933c9a483275781301aa210d6549e4cb485523505f1
data/CHANGELOG.md ADDED
@@ -0,0 +1,13 @@
1
+ ## ChangeLog
2
+
3
+ - Release 0.1.3 - 2016/09/17
4
+ - Fix error handling and add debug logging in input plugin
5
+ - Release 0.1.2 - 2016/09/11
6
+ - Change default max message size and add debug message
7
+ - Release 0.1.1 - 2016/08/27
8
+ - Bump up google-cloud-pubsub (gcloud-ruby) to 0.20
9
+ - Release 0.1.0 - 2016/08/22
10
+ - Use formatter / parser plugin and add format configuration
11
+ - Bump up gcloud-ruby to 0.12
12
+ - Remove dependency on lightening buffer
13
+ - Fix error caused by Pub/Sub quotas
data/README.md CHANGED
@@ -41,7 +41,7 @@ Use `gcloud_pubsub` output plugin.
41
41
  topic <YOUR TOPIC>
42
42
  autocreate_topic false
43
43
  max_messages 1000
44
- max_total_size 10000000
44
+ max_total_size 9800000
45
45
  buffer_type file
46
46
  buffer_path /path/to/your/buffer
47
47
  flush_interval 1s
@@ -110,22 +110,12 @@ Use `gcloud_pubsub` input plugin.
110
110
  - See maxMessages on https://cloud.google.com/pubsub/subscriber#receiving-pull-messages
111
111
  - `return_immediately` (optional, default: `true`)
112
112
  - See returnImmediately on https://cloud.google.com/pubsub/subscriber#receiving-pull-messages
113
- - If `return_immediately` is `true`, this plugin ignore `pull_interval`.
113
+ - If `return_immediately` is `false`, this plugin ignore `pull_interval`.
114
114
  - `pull_interval` (optional, default: `5`)
115
115
  - Pulling messages by intervals of specified seconds.
116
116
  - `format` (optional, default: `json`)
117
117
  - Set input format. See format section in http://docs.fluentd.org/articles/in_tail
118
118
 
119
- ## ChangeLog
120
-
121
- - Release 0.1.1 - 2016/08/27
122
- - Bump up google-cloud-pubsub (gcloud-ruby) to 0.20
123
- - Release 0.1.0 - 2016/08/22
124
- - Use formatter / parser plugin and add format configuration
125
- - Bump up gcloud-ruby to 0.12
126
- - Remove dependency on lightening buffer
127
- - Fix error caused by Pub/Sub quotas
128
-
129
119
  ## Contributing
130
120
 
131
121
  1. Fork it
@@ -7,7 +7,7 @@ Gem::Specification.new do |gem|
7
7
  gem.license = "MIT"
8
8
  gem.homepage = "https://github.com/mia-0032/fluent-plugin-gcloud-pubsub-custom"
9
9
  gem.summary = gem.description
10
- gem.version = "0.1.2"
10
+ gem.version = "0.1.3"
11
11
  gem.authors = ["Yoshihiro MIYAI"]
12
12
  gem.email = "msparrow17@gmail.com"
13
13
  gem.has_rdoc = false
@@ -49,28 +49,40 @@ module Fluent
49
49
 
50
50
  def subscribe
51
51
  until @stop_subscribing
52
- messages = @subscriber.pull @return_immediately, @max_messages
53
-
54
- if messages.length > 0
55
- es = parse_messages(messages)
56
- unless es.empty?
57
- begin
58
- router.emit_stream(@tag, es)
59
- rescue
60
- # ignore errors. Engine shows logs and backtraces.
61
- end
62
- @subscriber.acknowledge messages
63
- log.debug "#{messages.length} message(s) processed"
64
- end
65
- end
52
+ _subscribe
66
53
 
67
54
  if @return_immediately
68
55
  sleep @pull_interval
69
56
  end
70
57
  end
71
- rescue
72
- log.error "unexpected error", :error=>$!.to_s
73
- log.error_backtrace
58
+ rescue => ex
59
+ log.error "unexpected error", error_message: ex.to_s, error_class: ex.class.to_s
60
+ log.error_backtrace ex.backtrace
61
+ end
62
+
63
+ def _subscribe
64
+ messages = @subscriber.pull @return_immediately, @max_messages
65
+ if messages.length == 0
66
+ log.debug "no messages are pulled"
67
+ return
68
+ end
69
+
70
+ es = parse_messages(messages)
71
+ if es.empty?
72
+ log.warn "#{messages.length} message(s) are pulled, but no messages are parsed"
73
+ return
74
+ end
75
+
76
+ begin
77
+ router.emit_stream(@tag, es)
78
+ rescue
79
+ # ignore errors. Engine shows logs and backtraces.
80
+ end
81
+ @subscriber.acknowledge messages
82
+ log.debug "#{messages.length} message(s) processed"
83
+ rescue => ex
84
+ log.error "unexpected error", error_message: ex.to_s, error_class: ex.class.to_s
85
+ log.error_backtrace ex.backtrace
74
86
  end
75
87
 
76
88
  def parse_messages(messages)
@@ -90,9 +102,9 @@ module Fluent
90
102
  log.warn "pattern not match: #{line.inspect}"
91
103
  end
92
104
  }
93
- rescue => e
94
- log.warn line.dump, :error => e.to_s
95
- log.debug_backtrace(e.backtrace)
105
+ rescue => ex
106
+ log.warn line.dump, error_message: ex.to_s, error_class: ex.class.to_s
107
+ log.warn_backtrace ex.backtrace
96
108
  end
97
109
  end
98
110
  end
@@ -4,6 +4,7 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
4
4
  CONFIG = %[
5
5
  tag test
6
6
  project project-test
7
+ topic topic-test
7
8
  subscription subscription-test
8
9
  key key-test
9
10
  format json
@@ -44,11 +45,98 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
44
45
 
45
46
  test 'default values are configured' do
46
47
  d = create_driver
47
- assert_equal(nil, d.instance.topic)
48
48
  assert_equal(5, d.instance.pull_interval)
49
49
  assert_equal(100, d.instance.max_messages)
50
50
  assert_equal(true, d.instance.return_immediately)
51
51
  assert_equal('json', d.instance.format)
52
52
  end
53
53
  end
54
+
55
+ sub_test_case 'emit' do
56
+ class DummyMsgData
57
+ def data
58
+ return '{"foo": "bar"}'
59
+ end
60
+ end
61
+ class DummyMessage
62
+ def message
63
+ DummyMsgData.new
64
+ end
65
+ end
66
+
67
+ setup do
68
+ @subscriber = mock!
69
+ @topic_mock = mock!.subscription('subscription-test') { @subscriber }
70
+ @pubsub_mock = mock!.topic('topic-test') { @topic_mock }
71
+ @gcloud_mock = mock!.pubsub { @pubsub_mock }
72
+ stub(Google::Cloud).new { @gcloud_mock }
73
+ end
74
+
75
+ test 'empty' do
76
+ @subscriber.pull(immediate: true, max: 100).once { [] }
77
+ @subscriber.acknowledge.times(0)
78
+
79
+ d = create_driver
80
+ d.run {
81
+ # d.run sleeps 0.5 sec
82
+ }
83
+
84
+ assert_equal(true, d.emits.empty?)
85
+ end
86
+
87
+ test 'simple' do
88
+ messages = Array.new(1, DummyMessage.new)
89
+ @subscriber.pull(immediate: true, max: 100).once { messages }
90
+ @subscriber.acknowledge(messages).once
91
+
92
+ d = create_driver
93
+ d.run {
94
+ # d.run sleeps 0.5 sec
95
+ }
96
+ emits = d.emits
97
+
98
+ assert_equal(1, emits.length)
99
+ emits.each do |tag, time, record|
100
+ assert_equal("test", tag)
101
+ assert_equal({"foo" => "bar"}, record)
102
+ end
103
+ end
104
+
105
+ test 'invalid messages' do
106
+ class DummyInvalidMsgData
107
+ def data
108
+ return 'foo:bar'
109
+ end
110
+ end
111
+ class DummyInvalidMessage
112
+ def message
113
+ DummyInvalidMsgData.new
114
+ end
115
+ end
116
+
117
+ messages = Array.new(1, DummyInvalidMessage.new)
118
+ @subscriber.pull(immediate: true, max: 100).once { messages }
119
+ @subscriber.acknowledge.times(0)
120
+
121
+ d = create_driver
122
+ d.run {
123
+ # d.run sleeps 0.5 sec
124
+ }
125
+ assert_equal(true, d.emits.empty?)
126
+ end
127
+
128
+ test 'retry if raised error' do
129
+ class UnknownError < StandardError
130
+ end
131
+ @subscriber.pull(immediate: true, max: 100).twice { raise UnknownError.new('test') }
132
+ @subscriber.acknowledge.times(0)
133
+
134
+ d = create_driver(CONFIG + 'pull_interval 1')
135
+ d.run {
136
+ sleep 1 # + 0.5s
137
+ }
138
+
139
+ assert_equal(true, d.emits.empty?)
140
+ end
141
+ end
54
142
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-gcloud-pubsub-custom
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.2
4
+ version: 0.1.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Yoshihiro MIYAI
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-09-11 00:00:00.000000000 Z
11
+ date: 2016-09-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -108,6 +108,7 @@ extra_rdoc_files: []
108
108
  files:
109
109
  - ".gitignore"
110
110
  - ".travis.yml"
111
+ - CHANGELOG.md
111
112
  - Gemfile
112
113
  - Gemfile.v0.12
113
114
  - LICENSE