fluent-plugin-gcloud-pubsub-custom 0.4.5 → 0.4.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 629ea3f3f5f1f6be876e055dc8725630318002eb
-  data.tar.gz: 86a7b557b7c2a209bf3cacebf3450034d34626a4
+  metadata.gz: 9db89e553b8ed2af172483421345f062fb457bdb
+  data.tar.gz: 6a48713f2def51581fcd5745b513a8b5d2f17970
 SHA512:
-  metadata.gz: d145243e20fd085994ca66f77b904290d11b489ccda739db2d4355659d833e6f3a78420c6f2dafa44f558946d38a9e49cc551ce1ebefc77457d10d038e349425
-  data.tar.gz: 18338d5d011d45db17ced62e9e9f168dd68cef67dcd4c153922107750725112343ff1986207c5f099b01102c800906c2c36ff3a13bbeac2c5d88b904d06a6d83
+  metadata.gz: d57966e03358dd7d660f3448ac9106a36496643f9f7109fa75f87f0294bd596775c4ccb7703f689e03de844d7826dc0376858c910b9fa4d02d86af7b4a58f645
+  data.tar.gz: 757ee54ba5d8ccb9872535819cb27242642541cb0b2cb18f7802b8e2f95037ee191cfbaf10616ec7e7a22913c1ba7fb9fa7afb442b92d09c42e9ebced80ac1e8

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 ## ChangeLog
 
+### Release 0.4.6 - 2017/05/14
+
+- Output plugin
+  - Make messages exceeding configured size not be published because Pub/Sub clients cannot receive it
+
 ### Release 0.4.5 - 2017/04/02
 
 - Bump up google-cloud-pubsub to v0.24.x

data/README.md

@@ -48,6 +48,7 @@ Use `gcloud_pubsub` output plugin.
   autocreate_topic false
   max_messages 1000
   max_total_size 9800000
+  max_message_size 4000000
   buffer_type file
   buffer_path /path/to/your/buffer
   flush_interval 1s
@@ -74,6 +75,8 @@ Use `gcloud_pubsub` output plugin.
 - `max_total_size` (optional, default: `9800000` = `9.8MB`)
   - Publishing messages bytesize per request to Cloud Pub/Sub. This parameter affects only message size. You should specify a little smaller value than quota.
     - See https://cloud.google.com/pubsub/quotas#other_limits
+- `max_message_size` (optional, default: `4000000` = `4MB`)
+  - Messages exceeding `max_message_size` are not published because Pub/Sub clients cannot receive it.
 - `buffer_type`, `buffer_path`, `flush_interval`, `try_flush_interval`
   - These are fluentd buffer configuration. See http://docs.fluentd.org/articles/buffer-plugin-overview
 - `format` (optional, default: `json`)

data/fluent-plugin-gcloud-pubsub-custom.gemspec

@@ -7,7 +7,7 @@ Gem::Specification.new do |gem|
   gem.license     = "MIT"
   gem.homepage    = "https://github.com/mia-0032/fluent-plugin-gcloud-pubsub-custom"
   gem.summary     = gem.description
-  gem.version     = "0.4.5"
+  gem.version     = "0.4.6"
   gem.authors     = ["Yoshihiro MIYAI"]
   gem.email       = "msparrow17@gmail.com"
   gem.has_rdoc    = false

data/lib/fluent/plugin/out_gcloud_pubsub.rb

@@ -25,6 +25,8 @@ module Fluent
     config_param :max_messages,       :integer, :default => 1000
     desc 'Publishing messages bytesize per request to Cloud Pub/Sub.'
     config_param :max_total_size,     :integer, :default => 9800000  # 9.8MB
+    desc 'Limit bytesize per message.'
+    config_param :max_message_size,   :integer, :default => 4000000  # 4MB
     desc 'Set output format.'
     config_param :format,             :string,  :default => 'json'
 
@@ -57,6 +59,10 @@ module Fluent
       size = 0
 
       chunk.msgpack_each do |msg|
+        if msg.bytesize > @max_message_size
+          log.warn 'Drop a message because its size exceeds `max_message_size`', size: msg.bytesize
+          next
+        end
         if messages.length + 1 > @max_messages || size + msg.bytesize > @max_total_size
           publish messages
           messages = []

data/test/plugin/test_out_gcloud_pubsub.rb

@@ -31,6 +31,7 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
       assert_equal(false, d.instance.autocreate_topic)
       assert_equal(1000, d.instance.max_messages)
       assert_equal(9800000, d.instance.max_total_size)
+      assert_equal(4000000, d.instance.max_message_size)
     end
 
     test '"topic" must be specified' do
@@ -146,6 +147,19 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
       d.run
     end
 
+    test 'messages exceeding "max_message_size" are not published' do
+      d = create_driver(%[
+        project project-test
+        topic topic-test
+        key key-test
+        max_message_size 1000
+      ])
+
+      @publisher.publish.times(0)
+      d.emit({"a" => "a" * 1000}, @time)
+      d.run
+    end
+
     test 'accept "ASCII-8BIT" encoded multibyte strings' do
       # on fluentd v0.14, all strings treated as "ASCII-8BIT" except specified encoding.
       d = create_driver

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-gcloud-pubsub-custom
 version: !ruby/object:Gem::Version
-  version: 0.4.5
+  version: 0.4.6
 platform: ruby
 authors:
 - Yoshihiro MIYAI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-04-02 00:00:00.000000000 Z
+date: 2017-05-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd