fluent-plugin-gcloud-pubsub-custom 0.3.1 → 0.3.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +7 -1
- data/README.md +7 -1
- data/fluent-plugin-gcloud-pubsub-custom.gemspec +1 -1
- data/lib/fluent/plugin/in_gcloud_pubsub.rb +36 -30
- data/lib/fluent/plugin/out_gcloud_pubsub.rb +14 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1b951f60017ac6da9a7cab1d745b4d9f52ffafe1
|
4
|
+
data.tar.gz: eae153dca1b2b632c0bddfa196cd19949eff6aae
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b58c8a1285d86cb1f961aaca2042c9f0c6e9e54aef2d893149d6a3f53c825a2200cdcce574de68ffc38a4b6e46029ce87581ce72595e475cf2d9d5328be90e66
|
7
|
+
data.tar.gz: 73ce9c108180b311d839b87f68ba8a0309cf2d681cabbf6a4f5beb2503450e41a7cdcc147c2c7bd5d4b89fa40ba5c8743c382013072cf9c65e1b310d4a96142e
|
data/CHANGELOG.md
CHANGED
@@ -1,9 +1,15 @@
|
|
1
1
|
## ChangeLog
|
2
2
|
|
3
|
+
### Release 0.3.2 - 2016/11/13
|
4
|
+
|
5
|
+
- Add plugin param desc
|
6
|
+
- Input plugin
|
7
|
+
- Improve handling to acknowledge messages
|
8
|
+
|
3
9
|
### Release 0.3.1 - 2016/11/03
|
4
10
|
|
5
11
|
- Output plugin
|
6
|
-
- Improve error handling
|
12
|
+
- Improve error handling
|
7
13
|
|
8
14
|
### Release 0.3.0 - 2016/10/30
|
9
15
|
|
data/README.md
CHANGED
@@ -27,6 +27,12 @@ Install by gem:
|
|
27
27
|
$ gem install fluent-plugin-gcloud-pubsub-custom
|
28
28
|
```
|
29
29
|
|
30
|
+
**Caution**
|
31
|
+
|
32
|
+
This plugin doesn't work in [td-agent](http://docs.fluentd.org/articles/install-by-rpm).
|
33
|
+
|
34
|
+
Please use in [Fluentd installed by gem](http://docs.fluentd.org/articles/install-by-gem).
|
35
|
+
|
30
36
|
## Configuration
|
31
37
|
|
32
38
|
### Publish messages
|
@@ -51,7 +57,7 @@ Use `gcloud_pubsub` output plugin.
|
|
51
57
|
```
|
52
58
|
|
53
59
|
- `project` (optional)
|
54
|
-
- Set your GCP project
|
60
|
+
- Set your GCP project.
|
55
61
|
- Running fluentd on GCP, you don't have to specify.
|
56
62
|
- You can also use environment variable such as `GCLOUD_PROJECT`.
|
57
63
|
- `key` (optional)
|
@@ -7,7 +7,7 @@ Gem::Specification.new do |gem|
|
|
7
7
|
gem.license = "MIT"
|
8
8
|
gem.homepage = "https://github.com/mia-0032/fluent-plugin-gcloud-pubsub-custom"
|
9
9
|
gem.summary = gem.description
|
10
|
-
gem.version = "0.3.
|
10
|
+
gem.version = "0.3.2"
|
11
11
|
gem.authors = ["Yoshihiro MIYAI"]
|
12
12
|
gem.email = "msparrow17@gmail.com"
|
13
13
|
gem.has_rdoc = false
|
@@ -10,20 +10,43 @@ module Fluent
|
|
10
10
|
class GcloudPubSubInput < Input
|
11
11
|
Fluent::Plugin.register_input('gcloud_pubsub', self)
|
12
12
|
|
13
|
+
class << self
|
14
|
+
unless method_defined?(:desc)
|
15
|
+
def desc(description)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
class FailedParseError < StandardError
|
21
|
+
end
|
22
|
+
|
23
|
+
desc 'Set tag of messages.'
|
13
24
|
config_param :tag, :string
|
25
|
+
desc 'Set your GCP project.'
|
14
26
|
config_param :project, :string, default: nil
|
27
|
+
desc 'Set your credential file path.'
|
15
28
|
config_param :key, :string, default: nil
|
29
|
+
desc 'Set topic name to pull.'
|
16
30
|
config_param :topic, :string
|
31
|
+
desc 'Set subscription name to pull.'
|
17
32
|
config_param :subscription, :string
|
33
|
+
desc 'Pulling messages by intervals of specified seconds.'
|
18
34
|
config_param :pull_interval, :float, default: 5.0
|
35
|
+
desc 'Max messages pulling at once.'
|
19
36
|
config_param :max_messages, :integer, default: 100
|
37
|
+
desc 'Setting `true`, keepalive connection to wait for new messages.'
|
20
38
|
config_param :return_immediately, :bool, default: true
|
39
|
+
desc 'Set number of threads to pull messages.'
|
21
40
|
config_param :pull_threads, :integer, default: 1
|
41
|
+
desc 'Set input format.'
|
22
42
|
config_param :format, :string, default: 'json'
|
23
43
|
# for HTTP RPC
|
44
|
+
desc 'If `true` is specified, HTTP RPC to stop or start pulling message is enabled.'
|
24
45
|
config_param :enable_rpc, :bool, default: false
|
25
|
-
|
26
|
-
config_param :
|
46
|
+
desc 'Bind IP address for HTTP RPC.'
|
47
|
+
config_param :rpc_bind, :string, default: '0.0.0.0'
|
48
|
+
desc 'Port for HTTP RPC.'
|
49
|
+
config_param :rpc_port, :integer, default: 24680
|
27
50
|
|
28
51
|
unless method_defined?(:log)
|
29
52
|
define_method("log") { $log }
|
@@ -163,44 +186,27 @@ module Fluent
|
|
163
186
|
return
|
164
187
|
end
|
165
188
|
|
166
|
-
|
167
|
-
if es.empty?
|
168
|
-
log.warn "#{messages.length} message(s) are pulled, but no messages are parsed"
|
169
|
-
return
|
170
|
-
end
|
171
|
-
|
172
|
-
begin
|
173
|
-
router.emit_stream(@tag, es)
|
174
|
-
rescue
|
175
|
-
# ignore errors. Engine shows logs and backtraces.
|
176
|
-
end
|
189
|
+
process messages
|
177
190
|
@subscriber.acknowledge messages
|
191
|
+
|
178
192
|
log.debug "#{messages.length} message(s) processed"
|
179
193
|
rescue => ex
|
180
194
|
log.error "unexpected error", error_message: ex.to_s, error_class: ex.class.to_s
|
181
195
|
log.error_backtrace ex.backtrace
|
182
196
|
end
|
183
197
|
|
184
|
-
def
|
198
|
+
def process(messages)
|
185
199
|
es = MultiEventStream.new
|
186
200
|
messages.each do |m|
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
line = line.chomp # remove \n
|
194
|
-
@parser.parse(line) { |time, record|
|
195
|
-
if time && record
|
196
|
-
es.add(time, record)
|
197
|
-
else
|
198
|
-
log.warn "pattern not match: #{line.inspect}"
|
201
|
+
@parser.parse(m.message.data.chomp) do |time, record|
|
202
|
+
if time && record
|
203
|
+
es.add(time, record)
|
204
|
+
else
|
205
|
+
raise FailedParseError.new "pattern not match: #{line.inspect}"
|
206
|
+
end
|
199
207
|
end
|
200
|
-
|
201
|
-
|
202
|
-
log.warn line.dump, error_message: ex.to_s, error_class: ex.class.to_s
|
203
|
-
log.warn_backtrace ex.backtrace
|
208
|
+
end
|
209
|
+
router.emit_stream(@tag, es)
|
204
210
|
end
|
205
211
|
end
|
206
212
|
end
|
@@ -6,12 +6,26 @@ module Fluent
|
|
6
6
|
class GcloudPubSubOutput < BufferedOutput
|
7
7
|
Fluent::Plugin.register_output('gcloud_pubsub', self)
|
8
8
|
|
9
|
+
class << self
|
10
|
+
unless method_defined?(:desc)
|
11
|
+
def desc(description)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
desc 'Set your GCP project.'
|
9
17
|
config_param :project, :string, :default => nil
|
18
|
+
desc 'Set your credential file path.'
|
10
19
|
config_param :key, :string, :default => nil
|
20
|
+
desc 'Set topic name to publish.'
|
11
21
|
config_param :topic, :string
|
22
|
+
desc "If set to `true`, specified topic will be created when it doesn't exist."
|
12
23
|
config_param :autocreate_topic, :bool, :default => false
|
24
|
+
desc 'Publishing messages count per request to Cloud Pub/Sub.'
|
13
25
|
config_param :max_messages, :integer, :default => 1000
|
26
|
+
desc 'Publishing messages bytesize per request to Cloud Pub/Sub.'
|
14
27
|
config_param :max_total_size, :integer, :default => 9800000 # 9.8MB
|
28
|
+
desc 'Set output format.'
|
15
29
|
config_param :format, :string, :default => 'json'
|
16
30
|
|
17
31
|
unless method_defined?(:log)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-gcloud-pubsub-custom
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yoshihiro MIYAI
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-11-
|
11
|
+
date: 2016-11-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|