fluent-plugin-gcloud-pubsub 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 567b6fb76121f3492d47a459a3de2a29d535afa6
+  data.tar.gz: 290f0a7dab1ca7206083561df0645905dacd9cdc
+SHA512:
+  metadata.gz: bf695bd0c8812e5bcb6dd1472a382b5617bff728be5827e01d2652c3a2fe84d37e10be7849ed2a5ec4e6ea9e4b74ac091bedca3055f2ff99f1cab849cacd5e62
+  data.tar.gz: 4dd54deb86ccda01acd2258169ecadb6014f7e453366f06632ee28d74fe516ec94bd5d9f0ae867d70fac5c968a55fc6c73da08928679955b9a28d1708660e5d6

data/.gitignore

@@ -0,0 +1,11 @@
+~*
+#*
+*~
+[._]*.s[a-w][a-z]
+.DS_Store
+
+*.gem
+.bundle
+Gemfile.lock
+vendor
+.ruby-version

data/.travis.yml

@@ -0,0 +1,16 @@
+language: ruby
+
+rvm:
+  - 1.9.3
+  - 2.0
+  - 2.1
+  - 2.2.2
+
+gemfile:
+ - Gemfile
+
+branches:
+  only:
+    - master
+
+script: bundle exec rake test

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 mdoi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

data/README.md

@@ -0,0 +1,73 @@
+# fluent-plugin-gcloud-pubsub
+[![Build Status](https://travis-ci.org/mdoi/fluent-plugin-gcloud-pubsub.svg?branch=master)](https://travis-ci.org/mdoi/fluent-plugin-gcloud-pubsub)
+
+
+## Overview
+[Cloud Pub/Sub](https://cloud.google.com/pubsub/) Input/Output plugin for [Fluentd](http://www.fluentd.org/) using [gcloud](https://googlecloudplatform.github.io/gcloud-ruby/) gem
+
+- Publish BufferdOutput chunk into Cloud Pub/Sub using [batch publishing](http://googlecloudplatform.github.io/gcloud-ruby/docs/v0.2.0/Gcloud/Pubsub/Topic.html#method-i-publish)
+- [Pull](http://googlecloudplatform.github.io/gcloud-ruby/docs/v0.2.0/Gcloud/Pubsub/Subscription.html#method-i-pull) messages from Cloud Pub/Sub
+
+## Preparation
+- Create a project on Google Developer Console
+- Add a topic of Cloud Pub/Sub to the project
+- Add a pull style subscription to the topic
+- Download your credential (json) 
+
+## Publish messages
+
+Use `out_gcloud_pubsub`.
+
+### Configuration
+publish dummy json data like `{"message": "dummy", "value": 0}\n{"message": "dummy", "value": 1}\n ...`.
+
+```
+<source>
+  type dummy
+  tag example.publish
+  rate 100 
+  auto_increment_key value
+</source>
+
+<match example.publish>
+  type gcloud_pubsub
+  project <YOUR PROJECT>
+  topic <YOUR TOPIC>
+  key <YOUR KEY>
+  flush_interval 10
+</match>
+```
+
+
+## Pull messages
+Use `in_gcloud_pubsub`.
+
+### Configuration
+Pull json data from Cloud Pub/Sub
+
+```
+<source>
+  type gcloud_pubsub
+  tag example.pull
+  project <YOUR PROJECT>
+  topic <YOUR TOPIC>
+  subscription <YOUR SUBSCRIPTION>
+  key <YOUR KEY>
+  max_messages 1000
+  return_immediately true
+  pull_interval 2
+  format json
+</source>
+
+<match example.pull>
+  type stdout
+</match>
+```
+
+- `max_messages`
+ - see maxMessages on https://cloud.google.com/pubsub/subscriber
+
+- `return_immediately`
+ - see returnImmediately on https://cloud.google.com/pubsub/subscriber
+ - When `return_immediately` is true, this plugin ignore pull_interval
+

data/Rakefile

@@ -0,0 +1,12 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/plugin/test_*.rb']
+  test.verbose = true
+end
+
+task :default => [:build]

data/fluent-plugin-gcloud-pubsub.gemspec

@@ -0,0 +1,26 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-gcloud-pubsub"
+  gem.description = "Google Cloud Pub/Sub input/output plugin for Fluentd event collector"
+  gem.license     = "MIT"
+  gem.homepage    = "https://github.com/mdoi/fluent-plugin-gcloud-pubsub"
+  gem.summary     = gem.description
+  gem.version     = "0.0.1"
+  gem.authors     = ["Masayuki DOI"]
+  gem.email       = "dotquasar@gmail.com"
+  gem.has_rdoc    = false
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_runtime_dependency "fluentd"
+  gem.add_runtime_dependency "gcloud", ">= 0.2.0"
+  gem.add_runtime_dependency "fluent-plugin-buffer-lightening", ">= 0.0.2"
+
+  gem.add_development_dependency "bundler"
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "test-unit"
+end

data/lib/fluent/plugin/in_gcloud_pubsub.rb

@@ -0,0 +1,111 @@
+require 'gcloud'
+
+module Fluent
+  class GcloudPubSubInput < Fluent::Input
+    Fluent::Plugin.register_input('gcloud_pubsub', self)
+
+    config_param :tag,                :string
+    config_param :project,            :string,  :default => nil
+    config_param :topic,              :string,  :default => nil
+    config_param :subscription,       :string,  :default => nil
+    config_param :key,                :string,  :default => nil
+    config_param :pull_interval,      :integer, :default => 5
+    config_param :max_messages,       :integer, :default => 100
+    config_param :return_immediately, :bool,    :default => true
+
+    unless method_defined?(:log)
+      define_method("log") { $log }
+    end
+
+    unless method_defined?(:router)
+      define_method("router") { Fluent::Engine }
+    end
+
+    def configure(conf)
+      super
+
+      raise Fluent::ConfigError, "'project' must be specified." unless @project
+      raise Fluent::ConfigError, "'topic' must be specified." unless @topic
+      raise Fluent::ConfigError, "'subscription' must be specified." unless @subscription
+      raise Fluent::ConfigError, "'key' must be specified." unless @key
+
+      configure_parser(conf)
+    end
+
+    def configure_parser(conf)
+      @parser = TextParser.new
+      @parser.configure(conf)
+    end
+
+    def start
+      super
+
+      pubsub = (Gcloud.new @project, @key).pubsub
+      topic = pubsub.topic @topic
+      @client = topic.subscription @subscription
+      @stop_subscribing = false
+      @subscribe_thread = Thread.new(&method(:subscribe))
+    end
+
+    def shutdown
+      super
+
+      @stop_subscribing = true
+      @subscribe_thread.join
+    end
+
+    private
+    def configure_parser(conf)
+      @parser = TextParser.new
+      @parser.configure(conf)
+    end
+
+    def subscribe
+      until @stop_subscribing
+        messages = @client.pull max: @max_messages, immediate: @return_immediately
+
+        if messages.length > 0
+          es = parse_messages(messages)
+          unless es.empty?
+            begin
+              router.emit_stream(@tag, es)
+            rescue
+              # ignore errors. Engine shows logs and backtraces.
+            end
+            @client.acknowledge messages
+            log.debug "#{messages.length} message(s) processed"
+          end
+        end
+
+        if @return_immediately
+          sleep @pull_interval
+        end
+      end
+    rescue
+      log.error "unexpected error", :error=>$!.to_s
+      log.error_backtrace
+    end
+
+    def parse_messages(messages)
+      es = MultiEventStream.new
+      messages.each do |m|
+        convert_line_to_event(m.message.data, es)
+      end
+      es
+    end
+
+    def convert_line_to_event(line, es)
+      line.chomp!  # remove \n
+      @parser.parse(line) { |time, record|
+        if time && record
+          es.add(time, record)
+        else
+          log.warn "pattern not match: #{line.inspect}"
+        end
+      }
+    rescue => e
+      log.warn line.dump, :error => e.to_s
+      log.debug_backtrace(e.backtrace)
+    end
+  end
+end

data/lib/fluent/plugin/out_gcloud_pubsub.rb

@@ -0,0 +1,64 @@
+require 'gcloud'
+
+module Fluent
+  class GcloudPubSubOutput < Fluent::BufferedOutput
+    Fluent::Plugin.register_output('gcloud_pubsub', self)
+
+    config_set_default :buffer_type,                'lightening'
+    config_set_default :flush_interval,             1
+    config_set_default :try_flush_interval,         0.05
+    config_set_default :buffer_chunk_records_limit, 900
+    config_set_default :buffer_chunk_limit,         9437184
+    config_set_default :buffer_queue_limit,         64
+
+    config_param :project,            :string,  :default => nil
+    config_param :topic,              :string,  :default => nil
+    config_param :key,                :string,  :default => nil
+
+    unless method_defined?(:log)
+      define_method("log") { $log }
+    end
+
+    unless method_defined?(:router)
+      define_method("router") { Fluent::Engine }
+    end
+
+    def configure(conf)
+      super
+
+      raise Fluent::ConfigError, "'project' must be specified." unless @project
+      raise Fluent::ConfigError, "'topic' must be specified." unless @topic
+      raise Fluent::ConfigError, "'key' must be specified." unless @key
+    end
+
+    def start
+      super
+
+      pubsub = (Gcloud.new @project, @key).pubsub
+      @client = pubsub.topic @topic
+    end
+
+    def format(tag, time, record)
+      [tag, time, record].to_msgpack
+    end
+
+    def write(chunk)
+      messages = []
+
+      chunk.msgpack_each do |tag, time, record|
+        messages << record.to_json
+      end
+
+      if messages.length > 0
+        @client.publish do |batch|
+          messages.each do |m|
+            batch.publish m
+          end
+        end
+      end
+    rescue
+      log.error "unexpected error", :error=>$!.to_s
+      log.error_backtrace
+    end
+  end
+end

data/test/plugin/test_in_gcloud_pubsub.rb

@@ -0,0 +1,35 @@
+require_relative "../test_helper"
+
+
+class GcloudPubSubInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf=CONFIG)
+    Fluent::Test::InputTestDriver.new(Fluent::GcloudPubSubInput).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver(<<-EOC)
+      type gcloud_pubsub
+      tag test
+      project project-test
+      topic topic-test
+      subscription subscription-test
+      key key-test
+      max_messages 1000
+      return_immediately true
+      pull_interval 2
+      format json
+    EOC
+
+    assert_equal('test', d.instance.tag)
+    assert_equal('project-test', d.instance.project)
+    assert_equal('topic-test', d.instance.topic)
+    assert_equal('subscription-test', d.instance.subscription)
+    assert_equal('key-test', d.instance.key)
+    assert_equal(1000, d.instance.max_messages)
+    assert_equal(true, d.instance.return_immediately)
+  end
+end

data/test/plugin/test_out_gcloud_pubsub.rb

@@ -0,0 +1,27 @@
+require_relative "../test_helper"
+
+
+class GcloudPubSubOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf=CONFIG)
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::GcloudPubSubOutput).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver(<<-EOC)
+      type gcloud_pubsub
+      project project-test
+      topic topic-test
+      key key-test
+      flush_interval 1
+    EOC
+
+    assert_equal('project-test', d.instance.project)
+    assert_equal('topic-test', d.instance.topic)
+    assert_equal('key-test', d.instance.key)
+    assert_equal(1, d.instance.flush_interval)
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,30 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'fluent/test'
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/plugin/in_gcloud_pubsub'
+require 'fluent/plugin/out_gcloud_pubsub'
+
+class Test::Unit::TestCase
+end

metadata

@@ -0,0 +1,142 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-gcloud-pubsub
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Masayuki DOI
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-08-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: gcloud
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+- !ruby/object:Gem::Dependency
+  name: fluent-plugin-buffer-lightening
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.2
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Google Cloud Pub/Sub input/output plugin for Fluentd event collector
+email: dotquasar@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- fluent-plugin-gcloud-pubsub.gemspec
+- lib/fluent/plugin/in_gcloud_pubsub.rb
+- lib/fluent/plugin/out_gcloud_pubsub.rb
+- test/plugin/test_in_gcloud_pubsub.rb
+- test/plugin/test_out_gcloud_pubsub.rb
+- test/test_helper.rb
+homepage: https://github.com/mdoi/fluent-plugin-gcloud-pubsub
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Google Cloud Pub/Sub input/output plugin for Fluentd event collector
+test_files:
+- test/plugin/test_in_gcloud_pubsub.rb
+- test/plugin/test_out_gcloud_pubsub.rb
+- test/test_helper.rb