fluent-plugin-gcloud-pubsub-custom 1.2.0 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 76c67cd4a1690b17057f8961cc06136afe5ae3d56adfc6dc37c49190c845368b
-  data.tar.gz: f0cf742945f94fcfc05e2364ad4881dce89c1bc6dfc723143236390ae8a90a53
+  metadata.gz: 374946914ce3cfc2d8118fc8f0b4f98d9b3b9a588f2a44b05f540cf39f79b416
+  data.tar.gz: b593cb71d7b4305d3bb230829a7a7653038dbd33f180ba0f1d1903429203dddf
 SHA512:
-  metadata.gz: 671c29ebbef19b4155100aa6bedfdb686ee2bb79310e8c5ccb553806fde095cbd6c3ea5ff225eecfc29f6607ffc270205490873e564d3b2dfc713ce190cb26c0
-  data.tar.gz: 158b3b2aa6f5f59e5fe0d8e8cfe6e1dbb2cf1e49541a6d0b8d9e21e15f0d4f8a536ea120bc76ef255a8b66c250ca1c92351bfa087c82eac00851a2f2b033307d
+  metadata.gz: a926442360078e78350dae24fef50412bbd9fbca6bbfd5a0ede662c47c9e2b5eef815ad5da6b23e5ffe76014b22138f530768add2dbdb39d0dface069f1315f9
+  data.tar.gz: 7aa23e9902f6720b8e755031bcba63a49c69fb43d1f6600fb69d681197dbe8b25fcdbbaba48bac844ab331254c01307da5ea4c999f7cb13ca49ff65682e53ef7

data/.github/workflows/test.yml

@@ -0,0 +1,25 @@
+name: Test
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby-version: ['2.4', '2.5', '2.6', '2.7']
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+        bundler-cache: true
+    - name: Run tests
+      run: bundle exec rake test

data/CHANGELOG.md

@@ -1,5 +1,32 @@
 ## ChangeLog
 
+### Release 1.5.0 - 2021/01/26
+
+- Bump up google-cloud-pubsub to v2.3.x
+- Add `endpoint` parameter
+- Add gzip compression/decompression feature
+
+### Release 1.4.0 - 2020/08/11
+
+- Bump up google-cloud-pubsub to v0.39.x
+  - In preparation for updating to v1.x
+- Output plugin
+  - Add `dest_project` parameter
+
+### Release 1.3.2 - 2019/08/16
+
+- Input plugin
+  - Make `topic` parameter optional
+
+### Release 1.3.1 - 2018/06/08
+
+- Input plugin
+  - Dumps raw string to logs for parsing errors
+
+### Release 1.3.0 - 2018/05/18
+
+- Make attributes available 
+
 ### Release 1.2.0 - 2018/04/24
 
 - Output plugin

data/README.md

@@ -1,6 +1,6 @@
 # fluent-plugin-gcloud-pubsub-custom
 
-[![Build Status](https://travis-ci.org/mia-0032/fluent-plugin-gcloud-pubsub-custom.svg?branch=master)](https://travis-ci.org/mia-0032/fluent-plugin-gcloud-pubsub-custom)
+![Test](https://github.com/mia-0032/fluent-plugin-gcloud-pubsub-custom/workflows/Test/badge.svg)
 [![Gem Version](https://badge.fury.io/rb/fluent-plugin-gcloud-pubsub-custom.svg)](http://badge.fury.io/rb/fluent-plugin-gcloud-pubsub-custom)
 
 This plugin is forked from https://github.com/mdoi/fluent-plugin-gcloud-pubsub
@@ -60,6 +60,7 @@ Use `gcloud_pubsub` output plugin.
   max_messages 1000
   max_total_size 9800000
   max_message_size 4000000
+  endpoint <Endpoint URL>
   <buffer>
     @type memory
     flush_interval 1s
@@ -81,6 +82,8 @@ Use `gcloud_pubsub` output plugin.
 - `topic` (required)
   - Set topic name to publish.
   - You can use placeholder in this param. See: https://docs.fluentd.org/v1.0/articles/buffer-section
+- `dest_project` (optional, default: `nil`)
+  - Set your destination GCP project if you publish messages cross project.
 - `autocreate_topic` (optional, default: `false`)
   - If set to `true`, specified topic will be created when it doesn't exist.
 - `max_messages` (optional, default: `1000`)
@@ -91,6 +94,12 @@ Use `gcloud_pubsub` output plugin.
     - See https://cloud.google.com/pubsub/quotas#other_limits
 - `max_message_size` (optional, default: `4000000` = `4MB`)
   - Messages exceeding `max_message_size` are not published because Pub/Sub clients cannot receive it.
+- `attribute_keys` (optional, default: `[]`)
+  - Publishing the set fields as attributes.
+- `endpoint`(optional)
+  - Set Pub/Sub service endpoint. For more information, see [Service Endpoints](https://cloud.google.com/pubsub/docs/reference/service_apis_overview#service_endpoints)
+- `compression` (optional, default: `nil`)
+  - If set to `gzip`, messages will be compressed with gzip.
 
 ### Pull messages
 
@@ -131,8 +140,8 @@ Use `gcloud_pubsub` input plugin.
   - Set your credential file path.
   - Running fluentd on GCP, you can use scope instead of specifying this.
   - You can also use environment variable such as `GCLOUD_KEYFILE`.
-- `topic` (required)
-  - Set topic name to pull.
+- `topic` (optional)
+  - Set topic name that the subscription belongs to.
 - `subscription` (required)
   - Set subscription name to pull.
 - `max_messages` (optional, default: `100`)
@@ -144,6 +153,8 @@ Use `gcloud_pubsub` input plugin.
   - Pulling messages by intervals of specified seconds.
 - `pull_threads` (optional, default: `1`)
   - Set number of threads to pull messages.
+- `attribute_keys` (optional, default: `[]`)
+  - Specify the key of the attribute to be emitted as the field of record.
 - `parse_error_action` (optional, default: `exception`)
   - Set error type when parsing messages fails.
     - `exception`: Raise exception. Messages are not acknowledged.
@@ -154,6 +165,8 @@ Use `gcloud_pubsub` input plugin.
   - Bind IP address for HTTP RPC.
 - `rpc_port` (optional, default: `24680`)
   - Port for HTTP RPC.
+- `decompression` (optional, default: `nil`)
+  - If set to `gzip`, messages will be decompressed with gzip.
 
 ## Contributing
 

data/Rakefile

@@ -7,6 +7,7 @@ Rake::TestTask.new(:test) do |test|
   test.libs << 'lib' << 'test'
   test.test_files = FileList['test/plugin/test_*.rb']
   test.verbose = true
+  test.warning = false
 end
 
 task :default => [:build]

data/fluent-plugin-gcloud-pubsub-custom.gemspec

@@ -7,17 +7,16 @@ Gem::Specification.new do |gem|
   gem.license     = "MIT"
   gem.homepage    = "https://github.com/mia-0032/fluent-plugin-gcloud-pubsub-custom"
   gem.summary     = gem.description
-  gem.version     = "1.2.0"
+  gem.version     = "1.5.0"
   gem.authors     = ["Yoshihiro MIYAI"]
   gem.email       = "msparrow17@gmail.com"
-  gem.has_rdoc    = false
   gem.files       = `git ls-files`.split("\n")
   gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
   gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   gem.require_paths = ['lib']
 
   gem.add_runtime_dependency "fluentd", [">= 0.14.15", "< 2"]
-  gem.add_runtime_dependency "google-cloud-pubsub", "~> 0.30.0"
+  gem.add_runtime_dependency "google-cloud-pubsub", "~> 2.3.1"
 
   gem.add_development_dependency "bundler"
   gem.add_development_dependency "rake"

data/lib/fluent/plugin/gcloud_pubsub/client.rb

@@ -7,19 +7,40 @@ module Fluent
     class RetryableError < Error
     end
 
+    class Message
+      attr_reader :message, :attributes
+      def initialize(message, attributes={})
+        @message = message
+        @attributes = attributes
+      end
+
+      def bytesize()
+        attr_size = 0
+        @attributes.each do |key, val|
+          attr_size += key.bytesize + val.bytesize
+        end
+        @message.bytesize + attr_size
+      end
+    end
+
     class Publisher
-      def initialize(project, key, autocreate_topic)
-        @pubsub = Google::Cloud::Pubsub.new project_id: project, credentials: key
+      def initialize(project, key, autocreate_topic, dest_project, endpoint)
+        @pubsub = Google::Cloud::Pubsub.new project_id: project, credentials: key, endpoint: endpoint
         @autocreate_topic = autocreate_topic
+        @dest_project = dest_project
         @topics = {}
       end
 
       def topic(topic_name)
         return @topics[topic_name] if @topics.has_key? topic_name
 
-        client = @pubsub.topic topic_name
-        if client.nil? && @autocreate_topic
-          client = @pubsub.create_topic topic_name
+        if @dest_project.nil?
+          client = @pubsub.topic topic_name
+          if client.nil? && @autocreate_topic
+            client = @pubsub.create_topic topic_name
+          end
+        else
+          client = @pubsub.topic topic_name, project: @dest_project
         end
         if client.nil?
           raise Error.new "topic:#{topic_name} does not exist."
@@ -32,7 +53,7 @@ module Fluent
       def publish(topic_name, messages)
         topic(topic_name).publish do |batch|
           messages.each do |m|
-            batch.publish m
+            batch.publish m.message, m.attributes
           end
         end
       rescue Google::Cloud::UnavailableError, Google::Cloud::DeadlineExceededError, Google::Cloud::InternalError => ex
@@ -43,8 +64,12 @@ module Fluent
     class Subscriber
       def initialize(project, key, topic_name, subscription_name)
         pubsub = Google::Cloud::Pubsub.new project_id: project, credentials: key
-        topic = pubsub.topic topic_name
-        @client = topic.subscription subscription_name
+        if topic_name.nil?
+          @client = pubsub.subscription subscription_name
+        else
+          topic = pubsub.topic topic_name
+          @client = topic.subscription subscription_name
+        end
         raise Error.new "subscription:#{subscription_name} does not exist." if @client.nil?
       end
 

data/lib/fluent/plugin/in_gcloud_pubsub.rb

@@ -1,6 +1,7 @@
 require 'json'
 require 'webrick'
 
+require 'fluent/plugin/compressable'
 require 'fluent/plugin/input'
 require 'fluent/plugin/parser'
 
@@ -8,6 +9,8 @@ require 'fluent/plugin/gcloud_pubsub/client'
 
 module Fluent::Plugin
   class GcloudPubSubInput < Input
+    include Fluent::Plugin::Compressable
+
     Fluent::Plugin.register_input('gcloud_pubsub', self)
 
     helpers :compat_parameters, :parser, :thread
@@ -26,7 +29,7 @@ module Fluent::Plugin
     desc 'Set your credential file path.'
     config_param :key,                :string,  default: nil
     desc 'Set topic name to pull.'
-    config_param :topic,              :string
+    config_param :topic,              :string,  default: nil
     desc 'Set subscription name to pull.'
     config_param :subscription,       :string
     desc 'Pulling messages by intervals of specified seconds.'
@@ -37,6 +40,8 @@ module Fluent::Plugin
     config_param :return_immediately, :bool,    default: true
     desc 'Set number of threads to pull messages.'
     config_param :pull_threads,       :integer, default: 1
+    desc 'Specify the key of the attribute to be acquired as a record'
+    config_param :attribute_keys,     :array,   default: []
     desc 'Set error type when parsing messages fails.'
     config_param :parse_error_action, :enum,    default: :exception, list: [:exception, :warning]
     # for HTTP RPC
@@ -46,6 +51,8 @@ module Fluent::Plugin
     config_param :rpc_bind,           :string,  default: '0.0.0.0'
     desc 'Port for HTTP RPC.'
     config_param :rpc_port,           :integer, default: 24680
+    desc 'Decompress messages'
+    config_param :decompression,      :string,  default: nil
 
     config_section :parse do
       config_set_default :@type, DEFAULT_PARSER_TYPE
@@ -112,6 +119,11 @@ module Fluent::Plugin
                      end
 
       @parser = parser_create
+      @decompress = if @decompression == 'gzip'
+                      method(:gzip_decompress)
+                    else
+                      method(:no_decompress)
+                    end
     end
 
     def start
@@ -166,6 +178,14 @@ module Fluent::Plugin
       record.delete(@tag_key) || @tag
     end
 
+    def gzip_decompress(message)
+      decompress message
+    end
+
+    def no_decompress(message)
+      message
+    end
+
     def start_rpc
       log.info "listening http rpc server on http://#{@rpc_bind}:#{@rpc_port}/"
       @rpc_srv = WEBrick::HTTPServer.new(
@@ -219,16 +239,21 @@ module Fluent::Plugin
       end
 
       messages.each do |m|
-        line = m.message.data.chomp
+        line = @decompress.call(m.message.data).chomp
+        attributes = m.attributes
         @parser.parse(line) do |time, record|
           if time && record
+            @attribute_keys.each do |key|
+              record[key] = attributes[key]
+            end
+
             event_streams[@extract_tag.call(record)].add(time, record)
           else
             case @parse_error_action
             when :exception
-              raise FailedParseError.new "pattern not match: #{line.inspect}"
+              raise FailedParseError.new "pattern not match: #{line}"
             else
-              log.warn 'pattern not match', record: line.inspect
+              log.warn 'pattern not match', record: line
             end
           end
         end

data/lib/fluent/plugin/out_gcloud_pubsub.rb

@@ -1,9 +1,11 @@
+require 'fluent/plugin/compressable'
 require 'fluent/plugin/output'
 require 'fluent/plugin/gcloud_pubsub/client'
 require 'fluent/plugin_helper/inject'
 
 module Fluent::Plugin
   class GcloudPubSubOutput < Output
+    include Fluent::Plugin::Compressable
     include Fluent::PluginHelper::Inject
 
     Fluent::Plugin.register_output('gcloud_pubsub', self)
@@ -19,6 +21,8 @@ module Fluent::Plugin
     config_param :key,                :string,  :default => nil
     desc 'Set topic name to publish.'
     config_param :topic,              :string
+    desc "Set your dest GCP project if publishing cross project"
+    config_param :dest_project,       :string,  :default => nil
     desc "If set to `true`, specified topic will be created when it doesn't exist."
     config_param :autocreate_topic,   :bool,    :default => false
     desc 'Publishing messages count per request to Cloud Pub/Sub.'
@@ -27,6 +31,12 @@ module Fluent::Plugin
     config_param :max_total_size,     :integer, :default => 9800000  # 9.8MB
     desc 'Limit bytesize per message.'
     config_param :max_message_size,   :integer, :default => 4000000  # 4MB
+    desc 'Publishing the set field as an attribute'
+    config_param :attribute_keys,     :array,   :default => []
+    desc 'Set service endpoint'
+    config_param :endpoint, :string, :default => nil
+    desc 'Compress messages'
+    config_param :compression, :string, :default => nil
 
     config_section :buffer do
       config_set_default :@type, DEFAULT_BUFFER_TYPE
@@ -41,16 +51,25 @@ module Fluent::Plugin
       super
       placeholder_validate!(:topic, @topic)
       @formatter = formatter_create
+      @compress = if @compression == 'gzip'
+                    method(:gzip_compress)
+                  else
+                    method(:no_compress)
+                  end
     end
 
     def start
       super
-      @publisher = Fluent::GcloudPubSub::Publisher.new @project, @key, @autocreate_topic
+      @publisher = Fluent::GcloudPubSub::Publisher.new @project, @key, @autocreate_topic, @dest_project, @endpoint
     end
 
     def format(tag, time, record)
       record = inject_values_to_record(tag, time, record)
-      @formatter.format(tag, time, record).to_msgpack
+      attributes = {}
+      @attribute_keys.each do |key|
+        attributes[key] = record.delete(key)
+      end
+      [@compress.call(@formatter.format(tag, time, record)), attributes].to_msgpack
     end
 
     def formatted_to_msgpack_binary?
@@ -67,7 +86,8 @@ module Fluent::Plugin
       messages = []
       size = 0
 
-      chunk.msgpack_each do |msg|
+      chunk.msgpack_each do |msg, attr|
+        msg = Fluent::GcloudPubSub::Message.new(msg, attr)
         if msg.bytesize > @max_message_size
           log.warn 'Drop a message because its size exceeds `max_message_size`', size: msg.bytesize
           next
@@ -99,5 +119,13 @@ module Fluent::Plugin
       log.debug "send message topic:#{topic} length:#{messages.length} size:#{messages.map(&:bytesize).inject(:+)}"
       @publisher.publish(topic, messages)
     end
+
+    def gzip_compress(message)
+      compress message
+    end
+
+    def no_compress(message)
+      message
+    end
   end
 end

data/test/{test_helper.rb → helper.rb}

@@ -10,22 +10,14 @@ end
 require 'test/unit'
 require "test/unit/rr"
 
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
 
-require 'fluent/test'
-unless ENV.has_key?('VERBOSE')
-  nulllogger = Object.new
-  nulllogger.instance_eval {|obj|
-    def method_missing(method, *args)
-      # pass
-    end
-  }
-  $log = nulllogger
-end
+require "fluent/test"
+require "fluent/test/driver/output"
+require "fluent/test/helpers"
 
 require 'fluent/plugin/in_gcloud_pubsub'
 require 'fluent/plugin/out_gcloud_pubsub'
 
-class Test::Unit::TestCase
-end
+Test::Unit::TestCase.include(Fluent::Test::Helpers)
+Test::Unit::TestCase.extend(Fluent::Test::Helpers)

data/test/plugin/test_in_gcloud_pubsub.rb

@@ -1,7 +1,8 @@
 require 'net/http'
 require 'json'
 
-require_relative "../test_helper"
+require_relative "../helper"
+require 'fluent/plugin/compressable'
 require "fluent/test/driver/input"
 
 class GcloudPubSubInputTest < Test::Unit::TestCase
@@ -18,13 +19,16 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
 
   class DummyInvalidMsgData
     def data
-      return 'foo:bar'
+      'foo:bar'
     end
   end
   class DummyInvalidMessage
     def message
       DummyInvalidMsgData.new
     end
+    def attributes
+      {"attr_1" => "a", "attr_2" => "b"}
+    end
   end
 
   def create_driver(conf=CONFIG)
@@ -53,6 +57,7 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
         return_immediately true
         pull_interval 2
         pull_threads 3
+        attribute_keys attr-test
         enable_rpc true
         rpc_bind 127.0.0.1
         rpc_port 24681
@@ -67,6 +72,7 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
       assert_equal(1000, d.instance.max_messages)
       assert_equal(true, d.instance.return_immediately)
       assert_equal(3, d.instance.pull_threads)
+      assert_equal(['attr-test'], d.instance.attribute_keys)
       assert_equal(true, d.instance.enable_rpc)
       assert_equal('127.0.0.1', d.instance.rpc_bind)
       assert_equal(24681, d.instance.rpc_port)
@@ -78,6 +84,7 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
       assert_equal(100, d.instance.max_messages)
       assert_equal(true, d.instance.return_immediately)
       assert_equal(1, d.instance.pull_threads)
+      assert_equal([], d.instance.attribute_keys)
       assert_equal(false, d.instance.enable_rpc)
       assert_equal('0.0.0.0', d.instance.rpc_bind)
       assert_equal(24680, d.instance.rpc_port)
@@ -126,13 +133,16 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
   sub_test_case 'emit' do
     class DummyMsgData
       def data
-        return '{"foo": "bar"}'
+        '{"foo": "bar"}'
       end
     end
     class DummyMessage
       def message
         DummyMsgData.new
       end
+      def attributes
+        {"attr_1" => "a", "attr_2" => "b"}
+      end
     end
 
     class DummyMsgDataWithTagKey
@@ -140,7 +150,7 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
         @tag = tag
       end
       def data
-        return '{"foo": "bar", "test_tag_key": "' + @tag + '"}'
+        '{"foo": "bar", "test_tag_key": "' + @tag + '"}'
       end
     end
     class DummyMessageWithTagKey
@@ -150,6 +160,25 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
       def message
         DummyMsgDataWithTagKey.new @tag
       end
+      def attributes
+        {"attr_1" => "a", "attr_2" => "b"}
+      end
+    end
+
+    class DummyMsgDataGzipped
+      include Fluent::Plugin::Compressable
+
+      def data
+        compress '{"foo": "bar"}'
+      end
+    end
+    class DummyMessageGzipped
+      def message
+        DummyMsgDataGzipped.new
+      end
+      def attributes
+        {"attr_1" => "a", "attr_2" => "b"}
+      end
     end
 
     setup do
@@ -235,6 +264,38 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
       assert_true d.events.empty?
     end
 
+    test 'with attributes' do
+      messages = Array.new(1, DummyMessage.new)
+      @subscriber.pull(immediate: true, max: 100).at_least(1) { messages }
+      @subscriber.acknowledge(messages).at_least(1)
+
+      d = create_driver("#{CONFIG}\nattribute_keys attr_1")
+      d.run(expect_emits: 1, timeout: 3)
+      emits = d.events
+
+      assert(1 <= emits.length)
+      emits.each do |tag, time, record|
+        assert_equal("test", tag)
+        assert_equal({"foo" => "bar", "attr_1" => "a"}, record)
+      end
+    end
+
+    test 'with gzip decompression' do
+      messages = Array.new(1, DummyMessageGzipped.new)
+      @subscriber.pull(immediate: true, max: 100).at_least(1) { messages }
+      @subscriber.acknowledge(messages).at_least(1)
+
+      d = create_driver("#{CONFIG}\ndecompression gzip")
+      d.run(expect_emits: 1, timeout: 3)
+      emits = d.events
+
+      assert_equal(1, emits.length)
+      emits.each do |tag, time, record|
+        assert_equal("test", tag)
+        assert_equal({"foo" => "bar"}, record)
+      end
+    end
+
     test 'invalid messages with parse_error_action warning' do
       messages = Array.new(1, DummyInvalidMessage.new)
       @subscriber.pull(immediate: true, max: 100).at_least(1) { messages }

data/test/plugin/test_out_gcloud_pubsub.rb

@@ -1,10 +1,12 @@
 # coding: utf-8
-require_relative "../test_helper"
+require_relative "../helper"
+require 'fluent/plugin/compressable'
 require "fluent/test/driver/output"
 require "fluent/test/helpers"
 
 class GcloudPubSubOutputTest < Test::Unit::TestCase
   include Fluent::Test::Helpers
+  include Fluent::Plugin::Compressable
 
   CONFIG = %[
     project project-test
@@ -62,6 +64,17 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
 
       assert_equal(true, d.instance.autocreate_topic)
     end
+
+    test '"dest_project" can be specified' do
+      d = create_driver(%[
+        project project-test
+        topic topic-test
+        key key-test
+        dest_project dest-project-test
+      ])
+
+      assert_equal("dest-project-test", d.instance.dest_project)
+    end
   end
 
   sub_test_case 'topic' do
@@ -69,6 +82,13 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
       @publisher = mock!
       @pubsub_mock = mock!
       stub(Google::Cloud::Pubsub).new { @pubsub_mock }
+
+      @old_report_on_exception = Thread.report_on_exception
+      Thread.report_on_exception = false
+    end
+
+    teardown do
+      Thread.report_on_exception = @old_report_on_exception
     end
 
     test '"autocreate_topic" is enabled' do
@@ -87,6 +107,21 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
       end
     end
 
+    test '"dest_project" is set' do
+      d = create_driver(%[
+        project project-test
+        topic topic-test
+        key key-test
+        dest_project dest-project-test
+      ])
+
+      @publisher.publish.once
+      @pubsub_mock.topic("topic-test", {:project=>"dest-project-test"}).once { @publisher }
+      d.run(default_tag: "test") do
+        d.feed(@time, {"a" => "b"})
+      end
+    end
+
     test '40x error occurred on connecting to Pub/Sub' do
       d = create_driver
 
@@ -147,6 +182,13 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
       @publisher = mock!
       @pubsub_mock = mock!.topic(anything) { @publisher }
       stub(Google::Cloud::Pubsub).new { @pubsub_mock }
+
+      @old_report_on_exception = Thread.report_on_exception
+      Thread.report_on_exception = false
+    end
+
+    teardown do
+      Thread.report_on_exception = @old_report_on_exception
     end
 
     test 'messages are divided into "max_messages"' do
@@ -220,7 +262,24 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
       d.run(default_tag: 'test') do
         d.feed({"foo" => "bar"})
       end
-      assert_equal({"tag" => 'test', "foo" => "bar"}, JSON.parse(MessagePack.unpack(d.formatted.first)))
+      assert_equal({"tag" => 'test', "foo" => "bar"}, JSON.parse(MessagePack.unpack(d.formatted.first)[0]))
+    end
+
+    test 'with gzip compression' do
+      d = create_driver(%[
+        project project-test
+        topic topic-test
+        key key-test
+        compression gzip
+      ])
+      test_msg = {"a" => "abc"}
+
+      @publisher.publish.once
+      d.run(default_tag: "test") do
+        d.feed(test_msg)
+      end
+
+      assert_equal(test_msg, JSON.parse(decompress(MessagePack.unpack(d.formatted.first)[0])))
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-gcloud-pubsub-custom
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.5.0
 platform: ruby
 authors:
 - Yoshihiro MIYAI
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-04-24 00:00:00.000000000 Z
+date: 2021-01-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -36,14 +36,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.30.0
+        version: 2.3.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.30.0
+        version: 2.3.1
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -106,8 +106,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/test.yml"
 - ".gitignore"
-- ".travis.yml"
 - CHANGELOG.md
 - Gemfile
 - LICENSE
@@ -117,14 +117,14 @@ files:
 - lib/fluent/plugin/gcloud_pubsub/client.rb
 - lib/fluent/plugin/in_gcloud_pubsub.rb
 - lib/fluent/plugin/out_gcloud_pubsub.rb
+- test/helper.rb
 - test/plugin/test_in_gcloud_pubsub.rb
 - test/plugin/test_out_gcloud_pubsub.rb
-- test/test_helper.rb
 homepage: https://github.com/mia-0032/fluent-plugin-gcloud-pubsub-custom
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -139,12 +139,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.3
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: Google Cloud Pub/Sub input/output plugin for Fluentd event collector
 test_files:
+- test/helper.rb
 - test/plugin/test_in_gcloud_pubsub.rb
 - test/plugin/test_out_gcloud_pubsub.rb
-- test/test_helper.rb

data/.travis.yml

@@ -1,25 +0,0 @@
-language: ruby
-
-rvm:
-  - 2.1
-  - 2.2
-  - 2.3.6
-  - 2.4.3
-  - 2.5.0
-  - ruby-head
-
-gemfile:
- - Gemfile
-
-branches:
-  only:
-    - master
-
-before_install: gem update bundler
-script: bundle exec rake test
-
-sudo: false
-
-matrix:
-  allow_failures:
-    - rvm: ruby-head