fluent-plugin-gcloud-pubsub-custom 1.2.0 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 76c67cd4a1690b17057f8961cc06136afe5ae3d56adfc6dc37c49190c845368b
-  data.tar.gz: f0cf742945f94fcfc05e2364ad4881dce89c1bc6dfc723143236390ae8a90a53
+  metadata.gz: 940568295dd0772bb0e7f0fb2537c6fff1b95f6fcc2e8716b578c512529e808f
+  data.tar.gz: 9f2b817300f920541942d03a207290a833a945613611b0a25e72db52f39f9a16
 SHA512:
-  metadata.gz: 671c29ebbef19b4155100aa6bedfdb686ee2bb79310e8c5ccb553806fde095cbd6c3ea5ff225eecfc29f6607ffc270205490873e564d3b2dfc713ce190cb26c0
-  data.tar.gz: 158b3b2aa6f5f59e5fe0d8e8cfe6e1dbb2cf1e49541a6d0b8d9e21e15f0d4f8a536ea120bc76ef255a8b66c250ca1c92351bfa087c82eac00851a2f2b033307d
+  metadata.gz: 2911ef120d13bf8258981afa08d3124a01c04e88bf6af25f873c54f2eec1220f22b6473b24cfc6bbf750321933b28488c2f80a7a1b7459f9faa846044dca5e30
+  data.tar.gz: 8ce43148345570ef849d32097da05648daeae198feac5794c15cabb9272e0471d0add6409ae2a1889d2eb5e812786618eeab5c4d6317ae7eff6e05ad4d333ce0

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 ## ChangeLog
 
+### Release 1.3.0 - 2018/05/18
+
+- Make attributes available 
+
 ### Release 1.2.0 - 2018/04/24
 
 - Output plugin

data/README.md

@@ -91,6 +91,8 @@ Use `gcloud_pubsub` output plugin.
     - See https://cloud.google.com/pubsub/quotas#other_limits
 - `max_message_size` (optional, default: `4000000` = `4MB`)
   - Messages exceeding `max_message_size` are not published because Pub/Sub clients cannot receive it.
+- `attribute_keys` (optional, default: `[]`)
+  - Publishing the set fields as attributes.
 
 ### Pull messages
 
@@ -144,6 +146,8 @@ Use `gcloud_pubsub` input plugin.
   - Pulling messages by intervals of specified seconds.
 - `pull_threads` (optional, default: `1`)
   - Set number of threads to pull messages.
+- `attribute_keys` (optional, default: `[]`)
+  - Specify the key of the attribute to be emitted as the field of record.
 - `parse_error_action` (optional, default: `exception`)
   - Set error type when parsing messages fails.
     - `exception`: Raise exception. Messages are not acknowledged.

data/fluent-plugin-gcloud-pubsub-custom.gemspec

@@ -7,7 +7,7 @@ Gem::Specification.new do |gem|
   gem.license     = "MIT"
   gem.homepage    = "https://github.com/mia-0032/fluent-plugin-gcloud-pubsub-custom"
   gem.summary     = gem.description
-  gem.version     = "1.2.0"
+  gem.version     = "1.3.0"
   gem.authors     = ["Yoshihiro MIYAI"]
   gem.email       = "msparrow17@gmail.com"
   gem.has_rdoc    = false

data/lib/fluent/plugin/gcloud_pubsub/client.rb

@@ -7,6 +7,22 @@ module Fluent
     class RetryableError < Error
     end
 
+    class Message
+      attr_reader :message, :attributes
+      def initialize(message, attributes={})
+        @message = message
+        @attributes = attributes
+      end
+
+      def bytesize()
+        attr_size = 0
+        @attributes.each do |key, val|
+          attr_size += key.bytesize + val.bytesize
+        end
+        @message.bytesize + attr_size
+      end
+    end
+
     class Publisher
       def initialize(project, key, autocreate_topic)
         @pubsub = Google::Cloud::Pubsub.new project_id: project, credentials: key
@@ -32,7 +48,7 @@ module Fluent
       def publish(topic_name, messages)
         topic(topic_name).publish do |batch|
           messages.each do |m|
-            batch.publish m
+            batch.publish m.message, m.attributes
           end
         end
       rescue Google::Cloud::UnavailableError, Google::Cloud::DeadlineExceededError, Google::Cloud::InternalError => ex

data/lib/fluent/plugin/in_gcloud_pubsub.rb

@@ -37,6 +37,8 @@ module Fluent::Plugin
     config_param :return_immediately, :bool,    default: true
     desc 'Set number of threads to pull messages.'
     config_param :pull_threads,       :integer, default: 1
+    desc 'Specify the key of the attribute to be acquired as a record'
+    config_param :attribute_keys,     :array,   default: []
     desc 'Set error type when parsing messages fails.'
     config_param :parse_error_action, :enum,    default: :exception, list: [:exception, :warning]
     # for HTTP RPC
@@ -220,8 +222,13 @@ module Fluent::Plugin
 
       messages.each do |m|
         line = m.message.data.chomp
+        attributes = m.attributes
         @parser.parse(line) do |time, record|
           if time && record
+            @attribute_keys.each do |key|
+              record[key] = attributes[key]
+            end
+
             event_streams[@extract_tag.call(record)].add(time, record)
           else
             case @parse_error_action

data/lib/fluent/plugin/out_gcloud_pubsub.rb

@@ -27,6 +27,8 @@ module Fluent::Plugin
     config_param :max_total_size,     :integer, :default => 9800000  # 9.8MB
     desc 'Limit bytesize per message.'
     config_param :max_message_size,   :integer, :default => 4000000  # 4MB
+    desc 'Publishing the set field as an attribute'
+    config_param :attribute_keys,     :array,   :default => []
 
     config_section :buffer do
       config_set_default :@type, DEFAULT_BUFFER_TYPE
@@ -50,7 +52,11 @@ module Fluent::Plugin
 
     def format(tag, time, record)
       record = inject_values_to_record(tag, time, record)
-      @formatter.format(tag, time, record).to_msgpack
+      attributes = {}
+      @attribute_keys.each do |key|
+        attributes[key] = record.delete(key)
+      end
+      [@formatter.format(tag, time, record), attributes].to_msgpack
     end
 
     def formatted_to_msgpack_binary?
@@ -67,7 +73,8 @@ module Fluent::Plugin
       messages = []
       size = 0
 
-      chunk.msgpack_each do |msg|
+      chunk.msgpack_each do |msg, attr|
+        msg = Fluent::GcloudPubSub::Message.new(msg, attr)
         if msg.bytesize > @max_message_size
           log.warn 'Drop a message because its size exceeds `max_message_size`', size: msg.bytesize
           next

data/test/plugin/test_in_gcloud_pubsub.rb

@@ -18,13 +18,16 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
 
   class DummyInvalidMsgData
     def data
-      return 'foo:bar'
+      'foo:bar'
     end
   end
   class DummyInvalidMessage
     def message
       DummyInvalidMsgData.new
     end
+    def attributes
+      {"attr_1" => "a", "attr_2" => "b"}
+    end
   end
 
   def create_driver(conf=CONFIG)
@@ -53,6 +56,7 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
         return_immediately true
         pull_interval 2
         pull_threads 3
+        attribute_keys attr-test
         enable_rpc true
         rpc_bind 127.0.0.1
         rpc_port 24681
@@ -67,6 +71,7 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
       assert_equal(1000, d.instance.max_messages)
       assert_equal(true, d.instance.return_immediately)
       assert_equal(3, d.instance.pull_threads)
+      assert_equal(['attr-test'], d.instance.attribute_keys)
       assert_equal(true, d.instance.enable_rpc)
       assert_equal('127.0.0.1', d.instance.rpc_bind)
       assert_equal(24681, d.instance.rpc_port)
@@ -78,6 +83,7 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
       assert_equal(100, d.instance.max_messages)
       assert_equal(true, d.instance.return_immediately)
       assert_equal(1, d.instance.pull_threads)
+      assert_equal([], d.instance.attribute_keys)
       assert_equal(false, d.instance.enable_rpc)
       assert_equal('0.0.0.0', d.instance.rpc_bind)
       assert_equal(24680, d.instance.rpc_port)
@@ -126,13 +132,16 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
   sub_test_case 'emit' do
     class DummyMsgData
       def data
-        return '{"foo": "bar"}'
+        '{"foo": "bar"}'
       end
     end
     class DummyMessage
       def message
         DummyMsgData.new
       end
+      def attributes
+        {"attr_1" => "a", "attr_2" => "b"}
+      end
     end
 
     class DummyMsgDataWithTagKey
@@ -140,7 +149,7 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
         @tag = tag
       end
       def data
-        return '{"foo": "bar", "test_tag_key": "' + @tag + '"}'
+        '{"foo": "bar", "test_tag_key": "' + @tag + '"}'
       end
     end
     class DummyMessageWithTagKey
@@ -150,6 +159,9 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
       def message
         DummyMsgDataWithTagKey.new @tag
       end
+      def attributes
+        {"attr_1" => "a", "attr_2" => "b"}
+      end
     end
 
     setup do
@@ -235,6 +247,22 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
       assert_true d.events.empty?
     end
 
+    test 'with attributes' do
+      messages = Array.new(1, DummyMessage.new)
+      @subscriber.pull(immediate: true, max: 100).at_least(1) { messages }
+      @subscriber.acknowledge(messages).at_least(1)
+
+      d = create_driver("#{CONFIG}\nattribute_keys attr_1")
+      d.run(expect_emits: 1, timeout: 3)
+      emits = d.events
+
+      assert(1 <= emits.length)
+      emits.each do |tag, time, record|
+        assert_equal("test", tag)
+        assert_equal({"foo" => "bar", "attr_1" => "a"}, record)
+      end
+    end
+
     test 'invalid messages with parse_error_action warning' do
       messages = Array.new(1, DummyInvalidMessage.new)
       @subscriber.pull(immediate: true, max: 100).at_least(1) { messages }

data/test/plugin/test_out_gcloud_pubsub.rb

@@ -220,7 +220,7 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
       d.run(default_tag: 'test') do
         d.feed({"foo" => "bar"})
       end
-      assert_equal({"tag" => 'test', "foo" => "bar"}, JSON.parse(MessagePack.unpack(d.formatted.first)))
+      assert_equal({"tag" => 'test', "foo" => "bar"}, JSON.parse(MessagePack.unpack(d.formatted.first)[0]))
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-gcloud-pubsub-custom
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.0
 platform: ruby
 authors:
 - Yoshihiro MIYAI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-04-24 00:00:00.000000000 Z
+date: 2018-05-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd