fluent-plugin-gcloud-pubsub 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 567b6fb76121f3492d47a459a3de2a29d535afa6
4
+ data.tar.gz: 290f0a7dab1ca7206083561df0645905dacd9cdc
5
+ SHA512:
6
+ metadata.gz: bf695bd0c8812e5bcb6dd1472a382b5617bff728be5827e01d2652c3a2fe84d37e10be7849ed2a5ec4e6ea9e4b74ac091bedca3055f2ff99f1cab849cacd5e62
7
+ data.tar.gz: 4dd54deb86ccda01acd2258169ecadb6014f7e453366f06632ee28d74fe516ec94bd5d9f0ae867d70fac5c968a55fc6c73da08928679955b9a28d1708660e5d6
@@ -0,0 +1,11 @@
1
+ ~*
2
+ #*
3
+ *~
4
+ [._]*.s[a-w][a-z]
5
+ .DS_Store
6
+
7
+ *.gem
8
+ .bundle
9
+ Gemfile.lock
10
+ vendor
11
+ .ruby-version
@@ -0,0 +1,16 @@
1
+ language: ruby
2
+
3
+ rvm:
4
+ - 1.9.3
5
+ - 2.0
6
+ - 2.1
7
+ - 2.2.2
8
+
9
+ gemfile:
10
+ - Gemfile
11
+
12
+ branches:
13
+ only:
14
+ - master
15
+
16
+ script: bundle exec rake test
data/Gemfile ADDED
@@ -0,0 +1,3 @@
1
+ source "http://rubygems.org"
2
+
3
+ gemspec
data/LICENSE ADDED
@@ -0,0 +1,22 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2015 mdoi
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
22
+
@@ -0,0 +1,73 @@
1
+ # fluent-plugin-gcloud-pubsub
2
+ [![Build Status](https://travis-ci.org/mdoi/fluent-plugin-gcloud-pubsub.svg?branch=master)](https://travis-ci.org/mdoi/fluent-plugin-gcloud-pubsub)
3
+
4
+
5
+ ## Overview
6
+ [Cloud Pub/Sub](https://cloud.google.com/pubsub/) Input/Output plugin for [Fluentd](http://www.fluentd.org/) using [gcloud](https://googlecloudplatform.github.io/gcloud-ruby/) gem
7
+
8
+ - Publish BufferdOutput chunk into Cloud Pub/Sub using [batch publishing](http://googlecloudplatform.github.io/gcloud-ruby/docs/v0.2.0/Gcloud/Pubsub/Topic.html#method-i-publish)
9
+ - [Pull](http://googlecloudplatform.github.io/gcloud-ruby/docs/v0.2.0/Gcloud/Pubsub/Subscription.html#method-i-pull) messages from Cloud Pub/Sub
10
+
11
+ ## Preparation
12
+ - Create a project on Google Developer Console
13
+ - Add a topic of Cloud Pub/Sub to the project
14
+ - Add a pull style subscription to the topic
15
+ - Download your credential (json)
16
+
17
+ ## Publish messages
18
+
19
+ Use `out_gcloud_pubsub`.
20
+
21
+ ### Configuration
22
+ publish dummy json data like `{"message": "dummy", "value": 0}\n{"message": "dummy", "value": 1}\n ...`.
23
+
24
+ ```
25
+ <source>
26
+ type dummy
27
+ tag example.publish
28
+ rate 100
29
+ auto_increment_key value
30
+ </source>
31
+
32
+ <match example.publish>
33
+ type gcloud_pubsub
34
+ project <YOUR PROJECT>
35
+ topic <YOUR TOPIC>
36
+ key <YOUR KEY>
37
+ flush_interval 10
38
+ </match>
39
+ ```
40
+
41
+
42
+ ## Pull messages
43
+ Use `in_gcloud_pubsub`.
44
+
45
+ ### Configuration
46
+ Pull json data from Cloud Pub/Sub
47
+
48
+ ```
49
+ <source>
50
+ type gcloud_pubsub
51
+ tag example.pull
52
+ project <YOUR PROJECT>
53
+ topic <YOUR TOPIC>
54
+ subscription <YOUR SUBSCRIPTION>
55
+ key <YOUR KEY>
56
+ max_messages 1000
57
+ return_immediately true
58
+ pull_interval 2
59
+ format json
60
+ </source>
61
+
62
+ <match example.pull>
63
+ type stdout
64
+ </match>
65
+ ```
66
+
67
+ - `max_messages`
68
+ - see maxMessages on https://cloud.google.com/pubsub/subscriber
69
+
70
+ - `return_immediately`
71
+ - see returnImmediately on https://cloud.google.com/pubsub/subscriber
72
+ - When `return_immediately` is true, this plugin ignore pull_interval
73
+
@@ -0,0 +1,12 @@
1
+ require 'bundler'
2
+ Bundler::GemHelper.install_tasks
3
+
4
+ require 'rake/testtask'
5
+
6
+ Rake::TestTask.new(:test) do |test|
7
+ test.libs << 'lib' << 'test'
8
+ test.test_files = FileList['test/plugin/test_*.rb']
9
+ test.verbose = true
10
+ end
11
+
12
+ task :default => [:build]
@@ -0,0 +1,26 @@
1
+ # encoding: utf-8
2
+ $:.push File.expand_path('../lib', __FILE__)
3
+
4
+ Gem::Specification.new do |gem|
5
+ gem.name = "fluent-plugin-gcloud-pubsub"
6
+ gem.description = "Google Cloud Pub/Sub input/output plugin for Fluentd event collector"
7
+ gem.license = "MIT"
8
+ gem.homepage = "https://github.com/mdoi/fluent-plugin-gcloud-pubsub"
9
+ gem.summary = gem.description
10
+ gem.version = "0.0.1"
11
+ gem.authors = ["Masayuki DOI"]
12
+ gem.email = "dotquasar@gmail.com"
13
+ gem.has_rdoc = false
14
+ gem.files = `git ls-files`.split("\n")
15
+ gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
16
+ gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
17
+ gem.require_paths = ['lib']
18
+
19
+ gem.add_runtime_dependency "fluentd"
20
+ gem.add_runtime_dependency "gcloud", ">= 0.2.0"
21
+ gem.add_runtime_dependency "fluent-plugin-buffer-lightening", ">= 0.0.2"
22
+
23
+ gem.add_development_dependency "bundler"
24
+ gem.add_development_dependency "rake"
25
+ gem.add_development_dependency "test-unit"
26
+ end
@@ -0,0 +1,111 @@
1
+ require 'gcloud'
2
+
3
+ module Fluent
4
+ class GcloudPubSubInput < Fluent::Input
5
+ Fluent::Plugin.register_input('gcloud_pubsub', self)
6
+
7
+ config_param :tag, :string
8
+ config_param :project, :string, :default => nil
9
+ config_param :topic, :string, :default => nil
10
+ config_param :subscription, :string, :default => nil
11
+ config_param :key, :string, :default => nil
12
+ config_param :pull_interval, :integer, :default => 5
13
+ config_param :max_messages, :integer, :default => 100
14
+ config_param :return_immediately, :bool, :default => true
15
+
16
+ unless method_defined?(:log)
17
+ define_method("log") { $log }
18
+ end
19
+
20
+ unless method_defined?(:router)
21
+ define_method("router") { Fluent::Engine }
22
+ end
23
+
24
+ def configure(conf)
25
+ super
26
+
27
+ raise Fluent::ConfigError, "'project' must be specified." unless @project
28
+ raise Fluent::ConfigError, "'topic' must be specified." unless @topic
29
+ raise Fluent::ConfigError, "'subscription' must be specified." unless @subscription
30
+ raise Fluent::ConfigError, "'key' must be specified." unless @key
31
+
32
+ configure_parser(conf)
33
+ end
34
+
35
+ def configure_parser(conf)
36
+ @parser = TextParser.new
37
+ @parser.configure(conf)
38
+ end
39
+
40
+ def start
41
+ super
42
+
43
+ pubsub = (Gcloud.new @project, @key).pubsub
44
+ topic = pubsub.topic @topic
45
+ @client = topic.subscription @subscription
46
+ @stop_subscribing = false
47
+ @subscribe_thread = Thread.new(&method(:subscribe))
48
+ end
49
+
50
+ def shutdown
51
+ super
52
+
53
+ @stop_subscribing = true
54
+ @subscribe_thread.join
55
+ end
56
+
57
+ private
58
+ def configure_parser(conf)
59
+ @parser = TextParser.new
60
+ @parser.configure(conf)
61
+ end
62
+
63
+ def subscribe
64
+ until @stop_subscribing
65
+ messages = @client.pull max: @max_messages, immediate: @return_immediately
66
+
67
+ if messages.length > 0
68
+ es = parse_messages(messages)
69
+ unless es.empty?
70
+ begin
71
+ router.emit_stream(@tag, es)
72
+ rescue
73
+ # ignore errors. Engine shows logs and backtraces.
74
+ end
75
+ @client.acknowledge messages
76
+ log.debug "#{messages.length} message(s) processed"
77
+ end
78
+ end
79
+
80
+ if @return_immediately
81
+ sleep @pull_interval
82
+ end
83
+ end
84
+ rescue
85
+ log.error "unexpected error", :error=>$!.to_s
86
+ log.error_backtrace
87
+ end
88
+
89
+ def parse_messages(messages)
90
+ es = MultiEventStream.new
91
+ messages.each do |m|
92
+ convert_line_to_event(m.message.data, es)
93
+ end
94
+ es
95
+ end
96
+
97
+ def convert_line_to_event(line, es)
98
+ line.chomp! # remove \n
99
+ @parser.parse(line) { |time, record|
100
+ if time && record
101
+ es.add(time, record)
102
+ else
103
+ log.warn "pattern not match: #{line.inspect}"
104
+ end
105
+ }
106
+ rescue => e
107
+ log.warn line.dump, :error => e.to_s
108
+ log.debug_backtrace(e.backtrace)
109
+ end
110
+ end
111
+ end
@@ -0,0 +1,64 @@
1
+ require 'gcloud'
2
+
3
+ module Fluent
4
+ class GcloudPubSubOutput < Fluent::BufferedOutput
5
+ Fluent::Plugin.register_output('gcloud_pubsub', self)
6
+
7
+ config_set_default :buffer_type, 'lightening'
8
+ config_set_default :flush_interval, 1
9
+ config_set_default :try_flush_interval, 0.05
10
+ config_set_default :buffer_chunk_records_limit, 900
11
+ config_set_default :buffer_chunk_limit, 9437184
12
+ config_set_default :buffer_queue_limit, 64
13
+
14
+ config_param :project, :string, :default => nil
15
+ config_param :topic, :string, :default => nil
16
+ config_param :key, :string, :default => nil
17
+
18
+ unless method_defined?(:log)
19
+ define_method("log") { $log }
20
+ end
21
+
22
+ unless method_defined?(:router)
23
+ define_method("router") { Fluent::Engine }
24
+ end
25
+
26
+ def configure(conf)
27
+ super
28
+
29
+ raise Fluent::ConfigError, "'project' must be specified." unless @project
30
+ raise Fluent::ConfigError, "'topic' must be specified." unless @topic
31
+ raise Fluent::ConfigError, "'key' must be specified." unless @key
32
+ end
33
+
34
+ def start
35
+ super
36
+
37
+ pubsub = (Gcloud.new @project, @key).pubsub
38
+ @client = pubsub.topic @topic
39
+ end
40
+
41
+ def format(tag, time, record)
42
+ [tag, time, record].to_msgpack
43
+ end
44
+
45
+ def write(chunk)
46
+ messages = []
47
+
48
+ chunk.msgpack_each do |tag, time, record|
49
+ messages << record.to_json
50
+ end
51
+
52
+ if messages.length > 0
53
+ @client.publish do |batch|
54
+ messages.each do |m|
55
+ batch.publish m
56
+ end
57
+ end
58
+ end
59
+ rescue
60
+ log.error "unexpected error", :error=>$!.to_s
61
+ log.error_backtrace
62
+ end
63
+ end
64
+ end
@@ -0,0 +1,35 @@
1
+ require_relative "../test_helper"
2
+
3
+
4
+ class GcloudPubSubInputTest < Test::Unit::TestCase
5
+ def setup
6
+ Fluent::Test.setup
7
+ end
8
+
9
+ def create_driver(conf=CONFIG)
10
+ Fluent::Test::InputTestDriver.new(Fluent::GcloudPubSubInput).configure(conf)
11
+ end
12
+
13
+ def test_configure
14
+ d = create_driver(<<-EOC)
15
+ type gcloud_pubsub
16
+ tag test
17
+ project project-test
18
+ topic topic-test
19
+ subscription subscription-test
20
+ key key-test
21
+ max_messages 1000
22
+ return_immediately true
23
+ pull_interval 2
24
+ format json
25
+ EOC
26
+
27
+ assert_equal('test', d.instance.tag)
28
+ assert_equal('project-test', d.instance.project)
29
+ assert_equal('topic-test', d.instance.topic)
30
+ assert_equal('subscription-test', d.instance.subscription)
31
+ assert_equal('key-test', d.instance.key)
32
+ assert_equal(1000, d.instance.max_messages)
33
+ assert_equal(true, d.instance.return_immediately)
34
+ end
35
+ end
@@ -0,0 +1,27 @@
1
+ require_relative "../test_helper"
2
+
3
+
4
+ class GcloudPubSubOutputTest < Test::Unit::TestCase
5
+ def setup
6
+ Fluent::Test.setup
7
+ end
8
+
9
+ def create_driver(conf=CONFIG)
10
+ Fluent::Test::BufferedOutputTestDriver.new(Fluent::GcloudPubSubOutput).configure(conf)
11
+ end
12
+
13
+ def test_configure
14
+ d = create_driver(<<-EOC)
15
+ type gcloud_pubsub
16
+ project project-test
17
+ topic topic-test
18
+ key key-test
19
+ flush_interval 1
20
+ EOC
21
+
22
+ assert_equal('project-test', d.instance.project)
23
+ assert_equal('topic-test', d.instance.topic)
24
+ assert_equal('key-test', d.instance.key)
25
+ assert_equal(1, d.instance.flush_interval)
26
+ end
27
+ end
@@ -0,0 +1,30 @@
1
+ require 'rubygems'
2
+ require 'bundler'
3
+ begin
4
+ Bundler.setup(:default, :development)
5
+ rescue Bundler::BundlerError => e
6
+ $stderr.puts e.message
7
+ $stderr.puts "Run `bundle install` to install missing gems"
8
+ exit e.status_code
9
+ end
10
+ require 'test/unit'
11
+
12
+ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
13
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
14
+
15
+ require 'fluent/test'
16
+ unless ENV.has_key?('VERBOSE')
17
+ nulllogger = Object.new
18
+ nulllogger.instance_eval {|obj|
19
+ def method_missing(method, *args)
20
+ # pass
21
+ end
22
+ }
23
+ $log = nulllogger
24
+ end
25
+
26
+ require 'fluent/plugin/in_gcloud_pubsub'
27
+ require 'fluent/plugin/out_gcloud_pubsub'
28
+
29
+ class Test::Unit::TestCase
30
+ end
metadata ADDED
@@ -0,0 +1,142 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-gcloud-pubsub
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Masayuki DOI
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2015-08-21 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: fluentd
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: gcloud
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: 0.2.0
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: 0.2.0
41
+ - !ruby/object:Gem::Dependency
42
+ name: fluent-plugin-buffer-lightening
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: 0.0.2
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: 0.0.2
55
+ - !ruby/object:Gem::Dependency
56
+ name: bundler
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
69
+ - !ruby/object:Gem::Dependency
70
+ name: rake
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: test-unit
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - ">="
88
+ - !ruby/object:Gem::Version
89
+ version: '0'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
97
+ description: Google Cloud Pub/Sub input/output plugin for Fluentd event collector
98
+ email: dotquasar@gmail.com
99
+ executables: []
100
+ extensions: []
101
+ extra_rdoc_files: []
102
+ files:
103
+ - ".gitignore"
104
+ - ".travis.yml"
105
+ - Gemfile
106
+ - LICENSE
107
+ - README.md
108
+ - Rakefile
109
+ - fluent-plugin-gcloud-pubsub.gemspec
110
+ - lib/fluent/plugin/in_gcloud_pubsub.rb
111
+ - lib/fluent/plugin/out_gcloud_pubsub.rb
112
+ - test/plugin/test_in_gcloud_pubsub.rb
113
+ - test/plugin/test_out_gcloud_pubsub.rb
114
+ - test/test_helper.rb
115
+ homepage: https://github.com/mdoi/fluent-plugin-gcloud-pubsub
116
+ licenses:
117
+ - MIT
118
+ metadata: {}
119
+ post_install_message:
120
+ rdoc_options: []
121
+ require_paths:
122
+ - lib
123
+ required_ruby_version: !ruby/object:Gem::Requirement
124
+ requirements:
125
+ - - ">="
126
+ - !ruby/object:Gem::Version
127
+ version: '0'
128
+ required_rubygems_version: !ruby/object:Gem::Requirement
129
+ requirements:
130
+ - - ">="
131
+ - !ruby/object:Gem::Version
132
+ version: '0'
133
+ requirements: []
134
+ rubyforge_project:
135
+ rubygems_version: 2.4.5
136
+ signing_key:
137
+ specification_version: 4
138
+ summary: Google Cloud Pub/Sub input/output plugin for Fluentd event collector
139
+ test_files:
140
+ - test/plugin/test_in_gcloud_pubsub.rb
141
+ - test/plugin/test_out_gcloud_pubsub.rb
142
+ - test/test_helper.rb