fluent-plugin-gcloud-pubsub-custom 1.5.0 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 374946914ce3cfc2d8118fc8f0b4f98d9b3b9a588f2a44b05f540cf39f79b416
-  data.tar.gz: b593cb71d7b4305d3bb230829a7a7653038dbd33f180ba0f1d1903429203dddf
+  metadata.gz: 3bd7e53135bef7ce1f7ce3796659b6deb1ebaf6ff303755a84fdd886c4f46bf3
+  data.tar.gz: a04ddb36d5c0bceb7ac1fb8947b537ee0c38b9b26f6b5d5c53bb2ed034deb4e0
 SHA512:
-  metadata.gz: a926442360078e78350dae24fef50412bbd9fbca6bbfd5a0ede662c47c9e2b5eef815ad5da6b23e5ffe76014b22138f530768add2dbdb39d0dface069f1315f9
-  data.tar.gz: 7aa23e9902f6720b8e755031bcba63a49c69fb43d1f6600fb69d681197dbe8b25fcdbbaba48bac844ab331254c01307da5ea4c999f7cb13ca49ff65682e53ef7
+  metadata.gz: f0d6537d00021bb6d849ff7c9c9e8b4b233214826b098db649820a0a310d7d60170a8642db3b7b7acee2f83b70a392ad6a9ef8adbb47f22bc2d1be2022129d03
+  data.tar.gz: 2b742512024102940dd447f6450377684a1834c89582e38d6498957dc6219c5625afa49320f2c6c0139ff73bd7e98ad61085029b0c4794e3418c0324b0346bc0

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 ## ChangeLog
 
+### Release 1.6.0 - 2021/06/17
+
+- Output plugin
+  - Add `timeout` feature
+  - Add `attribute_key_values` feature
+
 ### Release 1.5.0 - 2021/01/26
 
 - Bump up google-cloud-pubsub to v2.3.x

data/README.md

@@ -61,6 +61,7 @@ Use `gcloud_pubsub` output plugin.
   max_total_size 9800000
   max_message_size 4000000
   endpoint <Endpoint URL>
+  timeout 60
   <buffer>
     @type memory
     flush_interval 1s
@@ -95,11 +96,15 @@ Use `gcloud_pubsub` output plugin.
 - `max_message_size` (optional, default: `4000000` = `4MB`)
   - Messages exceeding `max_message_size` are not published because Pub/Sub clients cannot receive it.
 - `attribute_keys` (optional, default: `[]`)
-  - Publishing the set fields as attributes.
+  - Publishing the set fields as attributes generated from input message.
+- `attribute_key_values` (optional, default: `{}`)
+  - Publishing the set fields as attributes generated from input params
 - `endpoint`(optional)
   - Set Pub/Sub service endpoint. For more information, see [Service Endpoints](https://cloud.google.com/pubsub/docs/reference/service_apis_overview#service_endpoints)
 - `compression` (optional, default: `nil`)
   - If set to `gzip`, messages will be compressed with gzip.
+- `timeout` (optional)
+  - Set default timeout to use in publish requests.
 
 ### Pull messages
 

data/fluent-plugin-gcloud-pubsub-custom.gemspec

@@ -7,7 +7,7 @@ Gem::Specification.new do |gem|
   gem.license     = "MIT"
   gem.homepage    = "https://github.com/mia-0032/fluent-plugin-gcloud-pubsub-custom"
   gem.summary     = gem.description
-  gem.version     = "1.5.0"
+  gem.version     = "1.6.0"
   gem.authors     = ["Yoshihiro MIYAI"]
   gem.email       = "msparrow17@gmail.com"
   gem.files       = `git ls-files`.split("\n")

data/lib/fluent/plugin/gcloud_pubsub/client.rb

@@ -24,8 +24,8 @@ module Fluent
     end
 
     class Publisher
-      def initialize(project, key, autocreate_topic, dest_project, endpoint)
-        @pubsub = Google::Cloud::Pubsub.new project_id: project, credentials: key, endpoint: endpoint
+      def initialize(project, key, autocreate_topic, dest_project, endpoint, timeout)
+        @pubsub = Google::Cloud::Pubsub.new project_id: project, credentials: key, endpoint: endpoint, timeout: timeout
         @autocreate_topic = autocreate_topic
         @dest_project = dest_project
         @topics = {}

data/lib/fluent/plugin/out_gcloud_pubsub.rb

@@ -31,12 +31,16 @@ module Fluent::Plugin
     config_param :max_total_size,     :integer, :default => 9800000  # 9.8MB
     desc 'Limit bytesize per message.'
     config_param :max_message_size,   :integer, :default => 4000000  # 4MB
-    desc 'Publishing the set field as an attribute'
+    desc 'Publishing the set field as an attribute created from input message'
     config_param :attribute_keys,     :array,   :default => []
+    desc 'Publishing the set field as an attribute created from input config params'
+    config_param :attribute_key_values,     :hash,   :default => {}
     desc 'Set service endpoint'
     config_param :endpoint, :string, :default => nil
     desc 'Compress messages'
     config_param :compression, :string, :default => nil
+    desc 'Set default timeout to use in publish requests'
+    config_param :timeout, :integer, :default => nil
 
     config_section :buffer do
       config_set_default :@type, DEFAULT_BUFFER_TYPE
@@ -60,7 +64,7 @@ module Fluent::Plugin
 
     def start
       super
-      @publisher = Fluent::GcloudPubSub::Publisher.new @project, @key, @autocreate_topic, @dest_project, @endpoint
+      @publisher = Fluent::GcloudPubSub::Publisher.new @project, @key, @autocreate_topic, @dest_project, @endpoint, @timeout
     end
 
     def format(tag, time, record)
@@ -69,6 +73,7 @@ module Fluent::Plugin
       @attribute_keys.each do |key|
         attributes[key] = record.delete(key)
       end
+      attributes.merge! @attribute_key_values
       [@compress.call(@formatter.format(tag, time, record)), attributes].to_msgpack
     end
 

data/test/plugin/test_out_gcloud_pubsub.rb

@@ -3,6 +3,8 @@ require_relative "../helper"
 require 'fluent/plugin/compressable'
 require "fluent/test/driver/output"
 require "fluent/test/helpers"
+require 'json'
+require 'msgpack'
 
 class GcloudPubSubOutputTest < Test::Unit::TestCase
   include Fluent::Test::Helpers
@@ -281,5 +283,66 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
 
       assert_equal(test_msg, JSON.parse(decompress(MessagePack.unpack(d.formatted.first)[0])))
     end
+
+    test 'with attribute_keys' do
+      d = create_driver(%[
+        project project-test
+        topic topic-test
+        key key-test
+        attribute_keys a,b
+      ])
+
+      @publisher.publish.once
+      d.run(default_tag: "test") do
+        d.feed(@time, {"a" => "foo", "b" => "bar", "c" => "baz"})
+      end
+
+      formatted = MessagePack.unpack(d.formatted[0])
+      message = JSON.parse(formatted[0])
+      assert_equal(message, {"c" => "baz"})
+      attributes = formatted[1]
+      assert_equal(attributes, {"a" => "foo", "b" => "bar"})
+    end
+
+    test 'with attribute_key_values' do
+      d = create_driver(%[
+        project project-test
+        topic topic-test
+        key key-test
+        attribute_key_values {"key1": "value1", "key2": "value2"}
+      ])
+
+      @publisher.publish.once
+      d.run(default_tag: "test") do
+        d.feed(@time, {"a" => "foo", "b" => "bar"})
+      end
+
+      formatted = MessagePack.unpack(d.formatted[0])
+      message = JSON.parse(formatted[0])
+      assert_equal(message, {"a" => "foo", "b" => "bar"})
+      attributes = formatted[1]
+      assert_equal(attributes, {"key1" => "value1", "key2" => "value2"})
+    end
+
+    test 'with attribute_keys and attribute_key_values' do
+      d = create_driver(%[
+        project project-test
+        topic topic-test
+        key key-test
+        attribute_keys a
+        attribute_key_values {"key": "value"}
+      ])
+
+      @publisher.publish.once
+      d.run(default_tag: "test") do
+        d.feed(@time, {"a" => "foo", "b" => "bar"})
+      end
+
+      formatted = MessagePack.unpack(d.formatted[0])
+      message = JSON.parse(formatted[0])
+      assert_equal(message, {"b" => "bar"})
+      attributes = formatted[1]
+      assert_equal(attributes, {"a" => "foo", "key" => "value"})
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-gcloud-pubsub-custom
 version: !ruby/object:Gem::Version
-  version: 1.5.0
+  version: 1.6.0
 platform: ruby
 authors:
 - Yoshihiro MIYAI
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-01-26 00:00:00.000000000 Z
+date: 2021-06-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd