google-cloud-pubsub 0.30.2 → 0.31.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8bc2cbfb8b149d600fbdf725934ccef18019853c3c371d34d964a47532640b33
-  data.tar.gz: 836fae2f33841cd070137fab258d208b456018214a25335f6e7848b1c9e0270b
+  metadata.gz: 3817e592071ccc0badbc1c511c848341de65d0e5fb4b9feba8c528074cd71724
+  data.tar.gz: 9e789ffcc98b3672040ff67f5e198f6f214cde83a205d6893982ca87b5e99e2b
 SHA512:
-  metadata.gz: 55cf18d0b63d2f68f28db33f85dd21ded29e960e3cd871278b9abfedfa86829b49e3ea6ba96a7ad1fad15f96da65a2d75fb3f9c2dcd59025f1aba753878a79ca
-  data.tar.gz: ce9772ea6d63bc664739c58ee56c32f84c44f9662c6ba9800a9aacd1a7e09eea9757649ca3290f8db894a07a4401ec4fd9f9573d5c4c7230b8e91ac0141c5fb1
+  metadata.gz: 52e115e0f6bf8968976a853064e30088374ea1836410785e3042fe00324f065e085d54e7b1502a34cd8fde8dc4395b901a95293856846f8b5b6f0af421c11586
+  data.tar.gz: 2cf2bf6b75e926fcb0fda9ba22cae4a49a75b0fc7d408faf61ecc90fbd5a424e496369780eadee1b3fa90a448b37052933d70ee6b790dbed81ecbaed18380bae

data/README.md

@@ -50,6 +50,28 @@ subscriber.start
 subscriber.stop.wait!
 ```
 
+## Enabling Logging
+
+To enable logging for this library, set the logger for the underlying [gRPC](https://github.com/grpc/grpc/tree/master/src/ruby) library. The logger that you set may be a Ruby stdlib [`Logger`](https://ruby-doc.org/stdlib-2.5.0/libdoc/logger/rdoc/Logger.html) as shown below, or a [`Google::Cloud::Logging::Logger`](https://googlecloudplatform.github.io/google-cloud-ruby/#/docs/google-cloud-logging/latest/google/cloud/logging/logger) that will write logs to [Stackdriver Logging](https://cloud.google.com/logging/). See [grpc/logconfig.rb](https://github.com/grpc/grpc/blob/master/src/ruby/lib/grpc/logconfig.rb) and the gRPC [spec_helper.rb](https://github.com/grpc/grpc/blob/master/src/ruby/spec/spec_helper.rb) for additional information.
+
+Configuring a Ruby stdlib logger:
+
+```ruby
+require "logger"
+
+module MyLogger
+  LOGGER = Logger.new $stderr, level: Logger::WARN
+  def logger
+    LOGGER
+  end
+end
+
+# Define a gRPC module-level logger method before grpc/logconfig.rb loads.
+module GRPC
+  extend MyLogger
+end
+```
+
 ## Supported Ruby Versions
 
 This library is supported on Ruby 2.0+.

data/lib/google/cloud/pubsub.rb

@@ -51,6 +51,39 @@ module Google
     # To learn more about Pub/Sub, read the [Google Cloud Pub/Sub Overview
     # ](https://cloud.google.com/pubsub/overview).
     #
+    # ## Enabling Logging
+    #
+    # To enable logging for this library, set the logger for the underlying
+    # [gRPC](https://github.com/grpc/grpc/tree/master/src/ruby) library. The
+    # logger that you set may be a Ruby stdlib
+    # [`Logger`](https://ruby-doc.org/stdlib-2.5.0/libdoc/logger/rdoc/Logger.html)
+    # as shown below, or a
+    # [`Google::Cloud::Logging::Logger`](https://googlecloudplatform.github.io/google-cloud-ruby/#/docs/google-cloud-logging/latest/google/cloud/logging/logger)
+    # that will write logs to [Stackdriver
+    # Logging](https://cloud.google.com/logging/). See
+    # [grpc/logconfig.rb](https://github.com/grpc/grpc/blob/master/src/ruby/lib/grpc/logconfig.rb)
+    # and the gRPC
+    # [spec_helper.rb](https://github.com/grpc/grpc/blob/master/src/ruby/spec/spec_helper.rb)
+    # for additional information.
+    #
+    # Configuring a Ruby stdlib logger:
+    #
+    # ```ruby
+    # require "logger"
+    #
+    # module MyLogger
+    #   LOGGER = Logger.new $stderr, level: Logger::WARN
+    #   def logger
+    #     LOGGER
+    #   end
+    # end
+    #
+    # # Define a gRPC module-level logger method before grpc/logconfig.rb loads.
+    # module GRPC
+    #   extend MyLogger
+    # end
+    # ```
+    #
     # ## Retrieving Topics
     #
     # A Topic is a named resource to which messages are sent by publishers.

data/lib/google/cloud/pubsub/async_publisher.rb

@@ -267,7 +267,8 @@ module Google
             data = data.read
           end
           # Convert data to encoded byte array to match the protobuf defn
-          data_bytes = String(data).dup.force_encoding("ASCII-8BIT").freeze
+          data_bytes = \
+            String(data).dup.force_encoding(Encoding::ASCII_8BIT).freeze
 
           # Convert attributes to strings to match the protobuf definition
           attributes = Hash[attributes.map { |k, v| [String(k), String(v)] }]

data/lib/google/cloud/pubsub/batch_publisher.rb

@@ -84,7 +84,8 @@ module Google
             data = data.read
           end
           # Convert data to encoded byte array to match the protobuf defn
-          data_bytes = String(data).dup.force_encoding("ASCII-8BIT").freeze
+          data_bytes = \
+            String(data).dup.force_encoding(Encoding::ASCII_8BIT).freeze
 
           # Convert attributes to strings to match the protobuf definition
           attributes = Hash[attributes.map { |k, v| [String(k), String(v)] }]

data/lib/google/cloud/pubsub/message.rb

@@ -65,7 +65,7 @@ module Google
           attributes = Hash[attributes.map { |k, v| [String(k), String(v)] }]
 
           @grpc = Google::Pubsub::V1::PubsubMessage.new(
-            data: String(data).encode("ASCII-8BIT"),
+            data: String(data).dup.force_encoding(Encoding::ASCII_8BIT),
             attributes: attributes
           )
         end
@@ -104,7 +104,7 @@ module Google
         # @private New Message from a Google::Pubsub::V1::PubsubMessage object.
         def self.from_grpc grpc
           new.tap do |m|
-            m.instance_variable_set "@grpc", grpc
+            m.instance_variable_set :@grpc, grpc
           end
         end
       end

data/lib/google/cloud/pubsub/snapshot/list.rb

@@ -150,10 +150,10 @@ module Google
               Snapshot.from_grpc grpc, service
             end)
             token = grpc_list.next_page_token
-            token = nil if token == ""
-            subs.instance_variable_set "@token",   token
-            subs.instance_variable_set "@service", service
-            subs.instance_variable_set "@max",     max
+            token = nil if token == "".freeze
+            subs.instance_variable_set :@token,   token
+            subs.instance_variable_set :@service, service
+            subs.instance_variable_set :@max,     max
             subs
           end
 

data/lib/google/cloud/pubsub/subscriber/{async_pusher.rb → async_stream_pusher.rb}

@@ -22,9 +22,9 @@ module Google
       class Subscriber
         ##
         # @private
-        # # AsyncPusher
+        # # AsyncStreamPusher
         #
-        class AsyncPusher
+        class AsyncStreamPusher
           include MonitorMixin
 
           attr_reader :batch
@@ -108,9 +108,7 @@ module Google
               @background_thread.kill if @background_thread
             end
 
-            return nil if @batch.nil?
-
-            @batch.request
+            push_batch_request!
           end
 
           def started?
@@ -131,14 +129,14 @@ module Google
                   next
                 end
 
-                time_since_first_publish = Time.now - @batch_created_at
-                if time_since_first_publish > @interval
+                time_since_batch_creation = Time.now - @batch_created_at
+                if time_since_batch_creation > @interval
                   # interval met, publish the batch...
                   push_batch_request!
                   @cond.wait
                 else
                   # still waiting for the interval to publish the batch...
-                  @cond.wait(@interval - time_since_first_publish)
+                  @cond.wait(@interval - time_since_batch_creation)
                 end
               end
             end
@@ -147,10 +145,7 @@ module Google
           def push_batch_request!
             return unless @batch
 
-            request = @batch.request
-            Concurrent::Future.new(executor: @stream.push_thread_pool) do
-              @stream.push request
-            end.execute
+            @stream.push @batch.request
 
             @batch = nil
             @batch_created_at = nil

data/lib/google/cloud/pubsub/subscriber/async_unary_pusher.rb

@@ -0,0 +1,251 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "monitor"
+require "concurrent"
+
+module Google
+  module Cloud
+    module Pubsub
+      class Subscriber
+        ##
+        # @private
+        # # AsyncUnaryPusher
+        #
+        class AsyncUnaryPusher
+          include MonitorMixin
+
+          attr_reader :batch
+          attr_reader :max_bytes, :interval
+
+          def initialize stream, max_bytes: 10000000, interval: 1.0
+            @stream = stream
+
+            @max_bytes = max_bytes
+            @interval = interval
+
+            @cond = new_cond
+
+            # init MonitorMixin
+            super()
+          end
+
+          def acknowledge ack_ids
+            return true if ack_ids.empty?
+
+            synchronize do
+              ack_ids.each do |ack_id|
+                if @batch.nil?
+                  @batch = Batch.new max_bytes: @max_bytes
+                  @batch.ack ack_id
+                else
+                  unless @batch.try_ack ack_id
+                    push_batch_request!
+
+                    @batch = Batch.new max_bytes: @max_bytes
+                    @batch.ack ack_id
+                  end
+                end
+
+                @batch_created_at ||= Time.now
+                @background_thread ||= Thread.new { run_background }
+
+                push_batch_request! if @batch.ready?
+              end
+
+              @cond.signal
+            end
+
+            nil
+          end
+
+          def delay deadline, ack_ids
+            return true if ack_ids.empty?
+
+            synchronize do
+              ack_ids.each do |ack_id|
+                if @batch.nil?
+                  @batch = Batch.new max_bytes: @max_bytes
+                  @batch.delay deadline, ack_id
+                else
+                  unless @batch.try_delay deadline, ack_id
+                    push_batch_request!
+
+                    @batch = Batch.new max_bytes: @max_bytes
+                    @batch.delay deadline, ack_id
+                  end
+                end
+
+                @batch_created_at ||= Time.now
+                @background_thread ||= Thread.new { run_background }
+
+                push_batch_request! if @batch.ready?
+              end
+
+              @cond.signal
+            end
+
+            nil
+          end
+
+          def stop
+            synchronize do
+              @stopped = true
+
+              # Stop any background activity, clean up happens in wait!
+              @background_thread.kill if @background_thread
+            end
+
+            push_batch_request!
+          end
+
+          def started?
+            !stopped?
+          end
+
+          def stopped?
+            synchronize { @stopped }
+          end
+
+          protected
+
+          def run_background
+            synchronize do
+              until @stopped
+                if @batch.nil?
+                  @cond.wait
+                  next
+                end
+
+                time_since_batch_creation = Time.now - @batch_created_at
+                if time_since_batch_creation > @interval
+                  # interval met, publish the batch...
+                  push_batch_request!
+                  @cond.wait
+                else
+                  # still waiting for the interval to publish the batch...
+                  @cond.wait(@interval - time_since_batch_creation)
+                end
+              end
+            end
+          end
+
+          def push_batch_request!
+            return unless @batch
+
+            service = @stream.subscriber.service
+            name = @stream.subscriber.subscription_name
+
+            if @batch.ack?
+              ack_ids = @batch.ack_ids
+              Concurrent::Future.new(executor: @stream.push_thread_pool) do
+                service.acknowledge name, *ack_ids
+              end.execute
+            end
+            if @batch.delay?
+              @batch.modify_deadline_hash.each do |delay_seconds, delay_ack_ids|
+                Concurrent::Future.new(executor: @stream.push_thread_pool) do
+                  service.modify_ack_deadline name, delay_ack_ids, delay_seconds
+                end.execute
+              end
+            end
+
+            @batch = nil
+            @batch_created_at = nil
+          end
+
+          class Batch
+            attr_reader :max_bytes, :request
+
+            def initialize max_bytes: 10000000
+              @max_bytes = max_bytes
+              @request = Google::Pubsub::V1::StreamingPullRequest.new
+              @total_message_bytes = 0
+            end
+
+            def ack ack_id
+              @request.ack_ids << ack_id
+              @total_message_bytes += addl_ack_bytes ack_id
+            end
+
+            def try_ack ack_id
+              addl_bytes = addl_ack_bytes ack_id
+              return false if total_message_bytes + addl_bytes >= @max_bytes
+
+              ack ack_id
+              true
+            end
+
+            def addl_ack_bytes ack_id
+              ack_id.bytesize + 2
+            end
+
+            def delay deadline, ack_id
+              @request.modify_deadline_seconds << deadline
+              @request.modify_deadline_ack_ids << ack_id
+              @total_message_bytes += addl_delay_bytes deadline, ack_id
+            end
+
+            def try_delay deadline, ack_id
+              addl_bytes = addl_delay_bytes deadline, ack_id
+              return false if total_message_bytes + addl_bytes >= @max_bytes
+
+              delay deadline, ack_id
+              true
+            end
+
+            def addl_delay_bytes deadline, ack_id
+              bytes_for_int(deadline) + ack_id.bytesize + 4
+            end
+
+            def bytes_for_int num
+              # Ruby 2.0 does not have Integer#bit_length
+              return [num].pack("s").bytesize unless num.respond_to? :bit_length
+
+              (num.bit_length / 8.0).ceil
+            end
+
+            def ready?
+              total_message_bytes >= @max_bytes
+            end
+
+            def ack?
+              @request.ack_ids.any?
+            end
+
+            def delay?
+              @request.modify_deadline_ack_ids.any?
+            end
+
+            def ack_ids
+              @request.ack_ids
+            end
+
+            def modify_deadline_hash
+              grouped_hash = @request.modify_deadline_ack_ids.zip(
+                @request.modify_deadline_seconds
+              ).group_by { |_ack_id, seconds| seconds }
+              Hash[grouped_hash.map { |k, v| [k, v.map(&:first)] }]
+            end
+
+            def total_message_bytes
+              @total_message_bytes
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/google/cloud/pubsub/subscriber/stream.rb

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 
-require "google/cloud/pubsub/subscriber/async_pusher"
+require "google/cloud/pubsub/subscriber/async_unary_pusher"
 require "google/cloud/pubsub/subscriber/enumerator_queue"
 require "google/cloud/pubsub/service"
 require "google/cloud/errors"
@@ -53,6 +53,15 @@ module Google
             @push_thread_pool = Concurrent::FixedThreadPool.new \
               subscriber.push_threads
 
+            @stream_keepalive_task = Concurrent::TimerTask.new(
+              execution_interval: 30
+            ) do
+              # push empty request every 30 seconds to keep stream alive
+              unless inventory.empty?
+                push Google::Pubsub::V1::StreamingPullRequest.new
+              end
+            end.execute
+
             super() # to init MonitorMixin
           end
 
@@ -98,14 +107,7 @@ module Google
 
               # Once all the callbacks are complete, we can stop the publisher
               # and send the final request to the steeam.
-              if @async_pusher
-                request = @async_pusher.stop
-                if request
-                  Concurrent::Future.new(executor: @push_thread_pool) do
-                    @request_queue.push request
-                  end.execute
-                end
-              end
+              @async_pusher.stop if @async_pusher # will push current batch
 
               # Close the push thread pool now that the pusher is closed.
               @push_thread_pool.shutdown
@@ -125,7 +127,7 @@ module Google
             return true if ack_ids.empty?
 
             synchronize do
-              @async_pusher ||= AsyncPusher.new self
+              @async_pusher ||= AsyncUnaryPusher.new self
               @async_pusher.acknowledge ack_ids
               @inventory.remove ack_ids
               unpause_streaming!
@@ -141,7 +143,7 @@ module Google
             return true if mod_ack_ids.empty?
 
             synchronize do
-              @async_pusher ||= AsyncPusher.new self
+              @async_pusher ||= AsyncUnaryPusher.new self
               @async_pusher.delay deadline, mod_ack_ids
               @inventory.remove mod_ack_ids
               unpause_streaming!
@@ -168,7 +170,7 @@ module Google
             synchronize do
               return true if @inventory.empty?
 
-              @async_pusher ||= AsyncPusher.new self
+              @async_pusher ||= AsyncUnaryPusher.new self
               @async_pusher.delay subscriber.deadline, @inventory.ack_ids
             end
 
@@ -211,7 +213,7 @@ module Google
 
                 synchronize do
                   # Create receipt of received messages reception
-                  @async_pusher ||= AsyncPusher.new self
+                  @async_pusher ||= AsyncUnaryPusher.new self
                   @async_pusher.delay subscriber.deadline, received_ack_ids
 
                   # Add received messages to inventory

data/lib/google/cloud/pubsub/subscription/list.rb

@@ -155,10 +155,10 @@ module Google
               Subscription.from_grpc grpc, service
             end)
             token = grpc_list.next_page_token
-            token = nil if token == ""
-            subs.instance_variable_set "@token",   token
-            subs.instance_variable_set "@service", service
-            subs.instance_variable_set "@max",     max
+            token = nil if token == "".freeze
+            subs.instance_variable_set :@token,   token
+            subs.instance_variable_set :@service, service
+            subs.instance_variable_set :@max,     max
             subs
           end
 
@@ -170,11 +170,11 @@ module Google
               Subscription.new_lazy grpc, service
             end)
             token = grpc_list.next_page_token
-            token = nil if token == ""
-            subs.instance_variable_set "@token",   token
-            subs.instance_variable_set "@service", service
-            subs.instance_variable_set "@topic",   topic
-            subs.instance_variable_set "@max",     max
+            token = nil if token == "".freeze
+            subs.instance_variable_set :@token,   token
+            subs.instance_variable_set :@service, service
+            subs.instance_variable_set :@topic,   topic
+            subs.instance_variable_set :@max,     max
             subs
           end
 

data/lib/google/cloud/pubsub/topic/list.rb

@@ -149,10 +149,10 @@ module Google
               Topic.from_grpc grpc, service
             end)
             token = grpc_list.next_page_token
-            token = nil if token == ""
-            topics.instance_variable_set "@token",   token
-            topics.instance_variable_set "@service", service
-            topics.instance_variable_set "@max",     max
+            token = nil if token == "".freeze
+            topics.instance_variable_set :@token,   token
+            topics.instance_variable_set :@service, service
+            topics.instance_variable_set :@max,     max
             topics
           end
 

data/lib/google/cloud/pubsub/v1/doc/overview.rb

@@ -17,7 +17,7 @@ module Google
     # rubocop:disable LineLength
 
     ##
-    # # Ruby Client for Google Cloud Pub/Sub API ([Alpha](https://github.com/GoogleCloudPlatform/google-cloud-ruby#versioning))
+    # # Ruby Client for Google Cloud Pub/Sub API ([Beta](https://github.com/GoogleCloudPlatform/google-cloud-ruby#versioning))
     #
     # [Google Cloud Pub/Sub API][Product Documentation]:
     # Provides reliable, many-to-many, asynchronous messaging between applications.
@@ -72,4 +72,4 @@ module Google
       end
     end
   end
-end
+end

data/lib/google/cloud/pubsub/version.rb

@@ -16,7 +16,7 @@
 module Google
   module Cloud
     module Pubsub
-      VERSION = "0.30.2".freeze
+      VERSION = "0.31.1".freeze
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: google-cloud-pubsub
 version: !ruby/object:Gem::Version
-  version: 0.30.2
+  version: 0.31.1
 platform: ruby
 authors:
 - Mike Moore
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-04-02 00:00:00.000000000 Z
+date: 2018-08-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: google-cloud-core
@@ -219,7 +219,8 @@ files:
 - lib/google/cloud/pubsub/snapshot.rb
 - lib/google/cloud/pubsub/snapshot/list.rb
 - lib/google/cloud/pubsub/subscriber.rb
-- lib/google/cloud/pubsub/subscriber/async_pusher.rb
+- lib/google/cloud/pubsub/subscriber/async_stream_pusher.rb
+- lib/google/cloud/pubsub/subscriber/async_unary_pusher.rb
 - lib/google/cloud/pubsub/subscriber/enumerator_queue.rb
 - lib/google/cloud/pubsub/subscriber/stream.rb
 - lib/google/cloud/pubsub/subscription.rb
@@ -259,7 +260,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 2.7.7
 signing_key: 
 specification_version: 4
 summary: API Client library for Google Cloud Pub/Sub