fluent-plugin-gcloud-pubsub-custom 0.1.4 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8c9f2243c6c609552b125ea53d060467197681fe
-  data.tar.gz: 42ebb7ee74c628d14954f1fccc713be595b9a577
+  metadata.gz: 08c5555e386df16b94ce1a8db279f952893b3f3b
+  data.tar.gz: e6eee873e191628778e9793a4545baf4a9d90344
 SHA512:
-  metadata.gz: af2b11d0ecda25cdc2a6b6db2a0a640277ce0a69c722d0899da60a6306f3fb3c00a3676113942a3c4ef7c3c6c5c77673deea7462fa0ac882e3b95d1cb4a797f2
-  data.tar.gz: adc7c0a09c842e96540e1638c36ab3dec913f2cebc4ed73a599df0f4d74e85a94f31f97da99745787649d4555b557859476f1e5e58925bc8513b84444f72e717
+  metadata.gz: eb8869520add948a27084fd46c58740455432f35bc29210efc77e4578901f5b10762c2b24a56eede4df04bfa54b57eaa210de5d022a11bcd5555e925939849aa
+  data.tar.gz: 3d2ef099388aa1a0260061e3cc134d84c9326c29202414643a2fe3612ca27fed9697b763ddd2ead9c3f68aa3ef3adb85d55b1e00de77b1b6d79e035b16e452a7

data/CHANGELOG.md

@@ -1,5 +1,8 @@
 ## ChangeLog
 
+- Release 0.2.0 - 2016/10/15
+  - Input plugin
+    - Add HTTP RPC feature
 - Release 0.1.4 - 2016/09/19
   - Input plugin
     - `pull_interval` can be specified float value

data/README.md

@@ -89,6 +89,9 @@ Use `gcloud_pubsub` input plugin.
   return_immediately true
   pull_interval 0.5
   format json
+  enable_rpc true
+  rpc_bind 0.0.0.0
+  rpc_port 24680
 </source>
 ```
 
@@ -110,11 +113,17 @@ Use `gcloud_pubsub` input plugin.
   - See maxMessages on https://cloud.google.com/pubsub/subscriber#receiving-pull-messages
 - `return_immediately` (optional, default: `true`)
   - See returnImmediately on https://cloud.google.com/pubsub/subscriber#receiving-pull-messages
-  - If `return_immediately` is `false`, this plugin ignore `pull_interval`.
+  - If `return_immediately` is `true` and pulling message is stopped by HTTP RPC, this plugin wait `pull_interval` each pull.
 - `pull_interval` (optional, default: `5.0`)
   - Pulling messages by intervals of specified seconds.
 - `format` (optional, default: `json`)
   - Set input format. See format section in http://docs.fluentd.org/articles/in_tail
+- `enable_rpc` (optional, default: `false`)
+  - If `true` is specified, HTTP RPC to stop or start pulling message is enabled.
+- `rpc_bind` (optional, default: `0.0.0.0`)
+  - Bind IP address for HTTP RPC.
+- `rpc_port` (optional, default: `24680`)
+  - Port for HTTP RPC.
 
 ## Contributing
 

data/fluent-plugin-gcloud-pubsub-custom.gemspec

@@ -7,7 +7,7 @@ Gem::Specification.new do |gem|
   gem.license     = "MIT"
   gem.homepage    = "https://github.com/mia-0032/fluent-plugin-gcloud-pubsub-custom"
   gem.summary     = gem.description
-  gem.version     = "0.1.4"
+  gem.version     = "0.2.0"
   gem.authors     = ["Yoshihiro MIYAI"]
   gem.email       = "msparrow17@gmail.com"
   gem.has_rdoc    = false

data/lib/fluent/plugin/in_gcloud_pubsub.rb

@@ -1,3 +1,6 @@
+require 'json'
+require 'webrick'
+
 require 'fluent/input'
 require 'fluent/parser'
 
@@ -8,14 +11,18 @@ module Fluent
     Fluent::Plugin.register_input('gcloud_pubsub', self)
 
     config_param :tag,                :string
-    config_param :project,            :string,  :default => nil
-    config_param :key,                :string,  :default => nil
+    config_param :project,            :string,  default: nil
+    config_param :key,                :string,  default: nil
     config_param :topic,              :string
     config_param :subscription,       :string
-    config_param :pull_interval,      :float,   :default => 5.0
-    config_param :max_messages,       :integer, :default => 100
-    config_param :return_immediately, :bool,    :default => true
-    config_param :format,             :string,  :default => 'json'
+    config_param :pull_interval,      :float,   default: 5.0
+    config_param :max_messages,       :integer, default: 100
+    config_param :return_immediately, :bool,    default: true
+    config_param :format,             :string,  default: 'json'
+    # for HTTP RPC
+    config_param :enable_rpc,         :bool,    default: false
+    config_param :rpc_bind,               :string,  default: '0.0.0.0'
+    config_param :rpc_port,               :integer, default: 24680
 
     unless method_defined?(:log)
       define_method("log") { $log }
@@ -25,33 +32,118 @@ module Fluent
       define_method("router") { Fluent::Engine }
     end
 
+    class RPCServlet < WEBrick::HTTPServlet::AbstractServlet
+      class Error < StandardError; end
+
+      def initialize(server, plugin)
+        super
+        @plugin = plugin
+      end
+
+      def do_GET(req, res)
+        begin
+          code, header, body = process(req, res)
+        rescue
+          code, header, body = render_json(500, {
+              'ok' => false,
+              'message' => 'Internal Server Error',
+              'error' => "#{$!}",
+              'backtrace'=> $!.backtrace
+          })
+        end
+
+        res.status = code
+        header.each_pair {|k,v|
+          res[k] = v
+        }
+        res.body = body
+      end
+
+      def render_json(code, obj)
+        [code, {'Content-Type' => 'application/json'}, obj.to_json]
+      end
+
+      def process(req, res)
+        case req.path_info
+        when '/stop'
+          @plugin.stop_pull
+        when '/start'
+          @plugin.start_pull
+        else
+          raise Error.new "Invalid path_info: #{req.path_info}"
+        end
+        render_json(200, {'ok' => true})
+      end
+    end
+
     def configure(conf)
       super
+      @rpc_srv = nil
+      @rpc_thread = nil
+      @stop_pull = false
+
       @parser = Plugin.new_parser(@format)
       @parser.configure(conf)
     end
 
     def start
       super
+      start_rpc if @enable_rpc
+
       @subscriber = Fluent::GcloudPubSub::Subscriber.new @project, @key, @topic, @subscription
       log.debug "connected subscription:#{@subscription} in project #{@project}"
+
       @stop_subscribing = false
       @subscribe_thread = Thread.new(&method(:subscribe))
     end
 
     def shutdown
       super
+      if @rpc_srv
+        @rpc_srv.shutdown
+        @rpc_srv = nil
+      end
+      if @rpc_thread
+        @rpc_thread.join
+        @rpc_thread = nil
+      end
       @stop_subscribing = true
       @subscribe_thread.join
     end
 
+    def stop_pull
+      @stop_pull = true
+      log.info "stop pull from subscription:#{@subscription}"
+    end
+
+    def start_pull
+      @stop_pull = false
+      log.info "start pull from subscription:#{@subscription}"
+    end
+
     private
 
+    def start_rpc
+      log.info "listening http rpc server on http://#{@rpc_bind}:#{@rpc_port}/"
+      @rpc_srv = WEBrick::HTTPServer.new(
+        {
+          BindAddress: @rpc_bind,
+          Port: @rpc_port,
+          Logger: WEBrick::Log.new(STDERR, WEBrick::Log::FATAL),
+          AccessLog: []
+        }
+      )
+      @rpc_srv.mount('/api/in_gcloud_pubsub/pull/', RPCServlet, self)
+      @rpc_thread = Thread.new {
+        @rpc_srv.start
+      }
+    end
+
     def subscribe
       until @stop_subscribing
-        _subscribe
+        _subscribe unless @stop_pull
 
-        if @return_immediately
+        if @return_immediately || @stop_pull
           sleep @pull_interval
         end
       end

data/test/plugin/test_in_gcloud_pubsub.rb

@@ -1,3 +1,5 @@
+require 'net/http'
+
 require_relative "../test_helper"
 
 class GcloudPubSubInputTest < Test::Unit::TestCase
@@ -10,10 +12,19 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
       format json
   ]
 
+  DEFAULT_HOST = '127.0.0.1'
+  DEFAULT_PORT = 24680
+
   def create_driver(conf=CONFIG)
     Fluent::Test::InputTestDriver.new(Fluent::GcloudPubSubInput).configure(conf)
   end
 
+  def http_get(path)
+    http = Net::HTTP.new(DEFAULT_HOST, DEFAULT_PORT)
+    req = Net::HTTP::Get.new(path, {'Content-Type' => 'application/x-www-form-urlencoded'})
+    http.request(req)
+  end
+
   setup do
     Fluent::Test.setup
   end
@@ -30,6 +41,9 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
         return_immediately true
         pull_interval 2
         format ltsv
+        enable_rpc true
+        rpc_bind 127.0.0.1
+        rpc_port 24681
       ])
 
       assert_equal('test', d.instance.tag)
@@ -41,6 +55,9 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
       assert_equal(1000, d.instance.max_messages)
       assert_equal(true, d.instance.return_immediately)
       assert_equal('ltsv', d.instance.format)
+      assert_equal(true, d.instance.enable_rpc)
+      assert_equal('127.0.0.1', d.instance.rpc_bind)
+      assert_equal(24681, d.instance.rpc_port)
     end
 
     test 'default values are configured' do
@@ -49,6 +66,9 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
       assert_equal(100, d.instance.max_messages)
       assert_equal(true, d.instance.return_immediately)
       assert_equal('json', d.instance.format)
+      assert_equal(false, d.instance.enable_rpc)
+      assert_equal('0.0.0.0', d.instance.rpc_bind)
+      assert_equal(24680, d.instance.rpc_port)
     end
   end
 
@@ -139,5 +159,54 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
       assert_equal(0.5, d.instance.pull_interval)
       assert_equal(true, d.emits.empty?)
     end
+
+    test 'stop by http rpc' do
+      messages = Array.new(1, DummyMessage.new)
+      @subscriber.pull(immediate: true, max: 100).once { messages }
+      @subscriber.acknowledge(messages).once
+
+      d = create_driver("#{CONFIG}\npull_interval 1.0\nenable_rpc true")
+      assert_equal(false, d.instance.instance_variable_get(:@stop_pull))
+
+      d.run {
+        http_get('/api/in_gcloud_pubsub/pull/stop')
+        sleep 0.75
+        # d.run sleeps 0.5 sec
+      }
+      emits = d.emits
+
+      assert_equal(1, emits.length)
+      assert_equal(true, d.instance.instance_variable_get(:@stop_pull))
+
+      emits.each do |tag, time, record|
+        assert_equal("test", tag)
+        assert_equal({"foo" => "bar"}, record)
+      end
+    end
+
+    test 'start by http rpc' do
+      messages = Array.new(1, DummyMessage.new)
+      @subscriber.pull(immediate: true, max: 100).at_least(1) { messages }
+      @subscriber.acknowledge(messages).at_least(1)
+
+      d = create_driver("#{CONFIG}\npull_interval 1.0\nenable_rpc true")
+      d.instance.stop_pull
+      assert_equal(true, d.instance.instance_variable_get(:@stop_pull))
+
+      d.run {
+        http_get('/api/in_gcloud_pubsub/pull/start')
+        sleep 0.75
+        # d.run sleeps 0.5 sec
+      }
+      emits = d.emits
+
+      assert_equal(true, emits.length > 0)
+      assert_equal(false, d.instance.instance_variable_get(:@stop_pull))
+
+      emits.each do |tag, time, record|
+        assert_equal("test", tag)
+        assert_equal({"foo" => "bar"}, record)
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-gcloud-pubsub-custom
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.2.0
 platform: ruby
 authors:
 - Yoshihiro MIYAI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-19 00:00:00.000000000 Z
+date: 2016-10-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd