fluent-plugin-gcloud-pubsub-custom 0.3.4 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dea78067ca4b8f4a2859ed280ad3e577fee82767
-  data.tar.gz: 3aadcbc365f28e5ec3e80037ebbb8ae7a79e5162
+  metadata.gz: a8daf09e1eeb0af707c9fc9fda10284540c73555
+  data.tar.gz: c61879a204d49e5c1e11a630e5274916d3b8e507
 SHA512:
-  metadata.gz: 9b9f6e9c9b3afb9e407f22d5cec7429e03bf196478bd513f74200da3c2449143c17997f49cf38e9bf32234ae75063cd4086c95ee828735b0108f5299132cdabe
-  data.tar.gz: f2af751a0a0c1abc468ad7d675d9d4960c856f4278e87a86e4d19fd96a824a49f6976fd3e2879586d0f42b1fbcd47dabb22008d0d6a912f6e83b116cab33c1f9
+  metadata.gz: 3de06b045e2f552038d2102c35cc127af05ee67d97138c428c5159f34db5f6f2d552215f8f3f882bab8c1c462f242bc7de21c723ee82e0f9a0e202af6802f0f9
+  data.tar.gz: 17502d7a5df5570848fcfb40de28522618825b5f31dc51674c2ded054827a0082a18b36e373ec375398a2c3ca3797dc11559755fc213996bc58fb054096c23c6

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 ## ChangeLog
 
+### Release 0.4.0 - 2017/01/21
+
+- Input plugin
+  - Add feature to use record key as tag
+
 ### Release 0.3.4 - 2017/01/03
 
 - Output plugin

data/README.md

@@ -105,6 +105,9 @@ Use `gcloud_pubsub` input plugin.
 
 - `tag` (required)
   - Set tag of messages.
+  - If `tag_key` is specified, `tag` is used as tag when record don't have specified key.
+- `tag_key` (optional)
+  - Set key to be used as tag.
 - `project` (optional)
   - Set your GCP project
   - Running fluentd on GCP, you don't have to specify.

data/fluent-plugin-gcloud-pubsub-custom.gemspec

@@ -7,7 +7,7 @@ Gem::Specification.new do |gem|
   gem.license     = "MIT"
   gem.homepage    = "https://github.com/mia-0032/fluent-plugin-gcloud-pubsub-custom"
   gem.summary     = gem.description
-  gem.version     = "0.3.4"
+  gem.version     = "0.4.0"
   gem.authors     = ["Yoshihiro MIYAI"]
   gem.email       = "msparrow17@gmail.com"
   gem.has_rdoc    = false

data/lib/fluent/plugin/in_gcloud_pubsub.rb

@@ -22,6 +22,8 @@ module Fluent
 
     desc 'Set tag of messages.'
     config_param :tag,                :string
+    desc 'Set key to be used as tag.'
+    config_param :tag_key,            :string,  default: nil
     desc 'Set your GCP project.'
     config_param :project,            :string,  default: nil
     desc 'Set your credential file path.'
@@ -108,6 +110,12 @@ module Fluent
       @rpc_thread = nil
       @stop_pull = false
 
+      @extract_tag = if @tag_key.nil?
+                       method(:static_tag)
+                     else
+                       method(:dynamic_tag)
+                     end
+
       @parser = Plugin.new_parser(@format)
       @parser.configure(conf)
     end
@@ -153,6 +161,14 @@ module Fluent
 
     private
 
+    def static_tag(record)
+      @tag
+    end
+
+    def dynamic_tag(record)
+      record.delete(@tag_key) || @tag
+    end
+
     def start_rpc
       log.info "listening http rpc server on http://#{@rpc_bind}:#{@rpc_port}/"
       @rpc_srv = WEBrick::HTTPServer.new(
@@ -201,12 +217,15 @@ module Fluent
     end
 
     def process(messages)
-      es = MultiEventStream.new
+      event_streams = Hash.new do |hsh, key|
+        hsh[key] = MultiEventStream.new
+      end
+
       messages.each do |m|
         line = m.message.data.chomp
         @parser.parse(line) do |time, record|
           if time && record
-            es.add(time, record)
+            event_streams[@extract_tag.call(record)].add(time, record)
           else
             case @parse_error_action
             when :exception
@@ -218,10 +237,12 @@ module Fluent
         end
       end
 
-      # There are some output plugins not to supposed to be called with multi-threading.
-      # Maybe remove in the future.
-      @emit_guard.synchronize do
-        router.emit_stream(@tag, es)
+      event_streams.each do |tag, es|
+        # There are some output plugins not to supposed to be called with multi-threading.
+        # Maybe remove in the future.
+        @emit_guard.synchronize do
+          router.emit_stream(tag, es)
+        end
       end
     end
   end

data/test/plugin/test_in_gcloud_pubsub.rb

@@ -99,6 +99,23 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
       end
     end
 
+    class DummyMsgDataWithTagKey
+      def initialize(tag)
+        @tag = tag
+      end
+      def data
+        return '{"foo": "bar", "test_tag_key": "' + @tag + '"}'
+      end
+    end
+    class DummyMessageWithTagKey
+      def initialize(tag)
+        @tag = tag
+      end
+      def message
+        DummyMsgDataWithTagKey.new @tag
+      end
+    end
+
     setup do
       @subscriber = mock!
       @topic_mock = mock!.subscription('subscription-test') { @subscriber }
@@ -154,6 +171,32 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
       end
     end
 
+    test 'with tag_key' do
+      messages = [
+        DummyMessageWithTagKey.new('tag1'),
+        DummyMessageWithTagKey.new('tag2'),
+        DummyMessage.new
+      ]
+      @subscriber.pull(immediate: true, max: 100).once { messages }
+      @subscriber.acknowledge(messages).once
+
+      d = create_driver("#{CONFIG}\ntag_key test_tag_key")
+      d.run {
+        # d.run sleeps 0.5 sec
+      }
+      emits = d.emits
+
+      assert_equal(3, emits.length)
+      # test tag
+      assert_equal("tag1", emits[0][0])
+      assert_equal("tag2", emits[1][0])
+      assert_equal("test", emits[2][0])
+      # test record
+      emits.each do |tag, time, record|
+        assert_equal({"foo" => "bar"}, record)
+      end
+    end
+
     test 'invalid messages with parse_error_action exception ' do
       messages = Array.new(1, DummyInvalidMessage.new)
       @subscriber.pull(immediate: true, max: 100).once { messages }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-gcloud-pubsub-custom
 version: !ruby/object:Gem::Version
-  version: 0.3.4
+  version: 0.4.0
 platform: ruby
 authors:
 - Yoshihiro MIYAI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-01-03 00:00:00.000000000 Z
+date: 2017-01-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd