gcpc 0.0.7 → 0.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5fdb3019e64a9e547b7fb58a1a5ebf45b82020029f047dc2f7fcaa237d720093
-  data.tar.gz: 02a5b5b94f4bed91f1d49bd8c8bf0f97fa568b02bd05d543205b482c9fa47570
+  metadata.gz: d89748f1a7066c40f0b4c6c4477bab098ab45c206ca7f019d5f29f13be62c875
+  data.tar.gz: 76261a92198cd9e53629455aba3859117e649004838da3752ead8dbbe8445f14
 SHA512:
-  metadata.gz: ff644c9d6d8257aa128e940547a31db1a9190e6a8165bf6ad0e776c3cc6aef00ed22f68f98812fa938186ea0cba069fb2d53c29e478ea5ec92a6de7b7bac5d5d
-  data.tar.gz: c50b8e7e95f3c1651c0ecfdb6ef85e16ba4503eced5a74331eb6173ad2ebe87f5475287097b5ab3912ae20d0989e5462ab4578b0e866f4211843cb1b70f69bdc
+  metadata.gz: fdf792b95470615cc0e9967dab7efaccde16aa62a60dbde2c0c0728df37e13d51cc3f046585152fde3a528032ebe39b59fee8a05e0f86680ffafb37bfd4b5350
+  data.tar.gz: 77b9b3d6f3a61bfeef13b4094363aa98c1a91679dd532712753061abc761d4a13e31696b7036b9d84c9b78a9b6cd3e055b896923624859d8a82cf4693af2762b

data/.github/workflows/ci.yml

@@ -0,0 +1,24 @@
+name: test
+
+on:
+  push:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby-version: ['3.2', '3.1', '3.0']
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Ruby ${{ matrix.ruby-version }}
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+          bundler-cache: true
+      - name: Run rspec
+        run: bundle exec rspec
+      - name: run pubsub emulator
+        run: docker run -d -p 8085:8085 -it gcr.io/google.com/cloudsdktool/google-cloud-cli:latest gcloud beta emulators pubsub start --host-port=0.0.0.0:8085
+      - name: Run rspec with emulator
+        run: bundle exec rspec --tag emulator

data/CHANGELOG.md

@@ -4,16 +4,20 @@
 <!-- All changes (implementation, or doc changes) are worth noting. -->
 <!-- The changes listed above will go below when a new release is being prepared. -->
 
+## 0.0.8
+
+- Add an API for heartbeat https://github.com/wantedly/gcpc/pull/25
+
 ## 0.0.7
 
-- Add explicit loading of forwardable https://github.com/south37/gcpc/pull/10
-- Add ruby 2.7 as a test target https://github.com/south37/gcpc/pull/12
+- Add explicit loading of forwardable https://github.com/wantedly/gcpc/pull/10
+- Add ruby 2.7 as a test target https://github.com/wantedly/gcpc/pull/12
 
 ## 0.0.6
 
-- Add test of interceptors https://github.com/south37/gcpc/pull/7
-- Add publish_batch https://github.com/south37/gcpc/pull/6
+- Add test of interceptors https://github.com/wantedly/gcpc/pull/7
+- Add publish_batch https://github.com/wantedly/gcpc/pull/6
 
 ## 0.0.5
 
-- Add publish_async https://github.com/south37/gcpc/pull/5
+- Add publish_async https://github.com/wantedly/gcpc/pull/5

data/Gemfile.lock

@@ -1,74 +1,82 @@
 PATH
   remote: .
   specs:
-    gcpc (0.0.7)
+    gcpc (0.0.8)
       google-cloud-pubsub
 
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.7.0)
-      public_suffix (>= 2.0.2, < 5.0)
+    addressable (2.8.5)
+      public_suffix (>= 2.0.2, < 6.0)
     coderay (1.1.3)
-    concurrent-ruby (1.1.7)
+    concurrent-ruby (1.2.2)
     diff-lcs (1.3)
-    faraday (1.3.0)
-      faraday-net_http (~> 1.0)
-      multipart-post (>= 1.2, < 3)
-      ruby2_keywords
-    faraday-net_http (1.0.0)
-    gapic-common (0.3.4)
-      google-protobuf (~> 3.12, >= 3.12.2)
-      googleapis-common-protos (>= 1.3.9, < 2.0)
-      googleapis-common-protos-types (>= 1.0.4, < 2.0)
-      googleauth (~> 0.9)
-      grpc (~> 1.25)
-    google-cloud-core (1.5.0)
+    faraday (2.7.10)
+      faraday-net_http (>= 2.0, < 3.1)
+      ruby2_keywords (>= 0.0.4)
+    faraday-net_http (3.0.2)
+    faraday-retry (2.2.0)
+      faraday (~> 2.0)
+    gapic-common (0.19.1)
+      faraday (>= 1.9, < 3.a)
+      faraday-retry (>= 1.0, < 3.a)
+      google-protobuf (~> 3.14)
+      googleapis-common-protos (>= 1.3.12, < 2.a)
+      googleapis-common-protos-types (>= 1.3.1, < 2.a)
+      googleauth (~> 1.0)
+      grpc (~> 1.36)
+    google-cloud-core (1.6.0)
       google-cloud-env (~> 1.0)
       google-cloud-errors (~> 1.0)
-    google-cloud-env (1.4.0)
-      faraday (>= 0.17.3, < 2.0)
-    google-cloud-errors (1.0.1)
-    google-cloud-pubsub (2.3.0)
+    google-cloud-env (1.6.0)
+      faraday (>= 0.17.3, < 3.0)
+    google-cloud-errors (1.3.1)
+    google-cloud-pubsub (2.15.4)
       concurrent-ruby (~> 1.1)
       google-cloud-core (~> 1.5)
-      google-cloud-pubsub-v1 (~> 0.0)
-    google-cloud-pubsub-v1 (0.2.0)
-      gapic-common (~> 0.3)
+      google-cloud-pubsub-v1 (~> 0.8)
+      retriable (~> 3.1)
+    google-cloud-pubsub-v1 (0.17.3)
+      gapic-common (>= 0.19.1, < 2.a)
       google-cloud-errors (~> 1.0)
-      grpc-google-iam-v1 (>= 0.6.10, < 2.0)
-    google-protobuf (3.14.0)
-    googleapis-common-protos (1.3.10)
-      google-protobuf (~> 3.11)
-      googleapis-common-protos-types (>= 1.0.5, < 2.0)
+      google-iam-v1 (>= 0.4, < 2.a)
+    google-iam-v1 (0.5.2)
+      gapic-common (>= 0.19.1, < 2.a)
+      google-cloud-errors (~> 1.0)
+      grpc-google-iam-v1 (~> 1.1)
+    google-protobuf (3.23.4)
+    googleapis-common-protos (1.4.0)
+      google-protobuf (~> 3.14)
+      googleapis-common-protos-types (~> 1.2)
       grpc (~> 1.27)
-    googleapis-common-protos-types (1.0.5)
-      google-protobuf (~> 3.11)
-    googleauth (0.14.0)
-      faraday (>= 0.17.3, < 2.0)
+    googleapis-common-protos-types (1.8.0)
+      google-protobuf (~> 3.18)
+    googleauth (1.7.0)
+      faraday (>= 0.17.3, < 3.a)
       jwt (>= 1.4, < 3.0)
       memoist (~> 0.16)
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
-      signet (~> 0.14)
-    grpc (1.34.0)
-      google-protobuf (~> 3.13)
+      signet (>= 0.16, < 2.a)
+    grpc (1.56.2)
+      google-protobuf (~> 3.23)
       googleapis-common-protos-types (~> 1.0)
-    grpc-google-iam-v1 (0.6.10)
-      google-protobuf (~> 3.11)
-      googleapis-common-protos (>= 1.3.10, < 2.0)
-      grpc (~> 1.27)
-    jwt (2.2.2)
+    grpc-google-iam-v1 (1.3.0)
+      google-protobuf (~> 3.18)
+      googleapis-common-protos (~> 1.4)
+      grpc (~> 1.41)
+    jwt (2.7.1)
     memoist (0.16.2)
     method_source (1.0.0)
     multi_json (1.15.0)
-    multipart-post (2.1.1)
-    os (1.1.1)
+    os (1.1.4)
     pry (0.13.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    public_suffix (4.0.6)
+    public_suffix (5.0.3)
     rake (13.0.1)
+    retriable (3.1.2)
     rspec (3.8.0)
       rspec-core (~> 3.8.0)
       rspec-expectations (~> 3.8.0)
@@ -82,10 +90,10 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
     rspec-support (3.8.0)
-    ruby2_keywords (0.0.2)
-    signet (0.14.0)
-      addressable (~> 2.3)
-      faraday (>= 0.17.3, < 2.0)
+    ruby2_keywords (0.0.5)
+    signet (0.17.0)
+      addressable (~> 2.8)
+      faraday (>= 0.17.5, < 3.a)
       jwt (>= 1.5, < 3.0)
       multi_json (~> 1.10)
 

data/README.md

@@ -154,7 +154,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/south37/gcpc.
+Bug reports and pull requests are welcome on GitHub at https://github.com/wantedly/gcpc.
 
 ## License
 

data/gcpc.gemspec

@@ -11,7 +11,7 @@ Gem::Specification.new do |spec|
 
   spec.summary       = %q{Google Cloud Pub/Sub Client}
   spec.description   = %q{Google Cloud Pub/Sub Client}
-  spec.homepage      = "https://github.com/south37/gcpc"
+  spec.homepage      = "https://github.com/wantedly/gcpc"
   spec.license       = "MIT"
 
   # Specify which files should be added to the gem when it is released.

data/lib/gcpc/config.rb

@@ -0,0 +1,18 @@
+require "singleton"
+
+module Gcpc
+  class Config
+    include Singleton
+    attr_reader :beat
+
+    def initialize
+      @beat = []
+    end
+
+    def on(event, &block)
+      raise ArgumentError, "Invalid event name: #{event}" if event != :beat
+
+      @beat << block
+    end
+  end
+end

data/lib/gcpc/subscriber/engine.rb

@@ -4,6 +4,9 @@ module Gcpc
   class Subscriber
     class Engine
       WAIT_INTERVAL = 1
+      WORKER_DEAD_THRESHOLD = 30 # second
+      BEAT_INTERVAL = 10
+      HEART_BEAT_WORKER_NAME = 'heartbeat-worker'
 
       # @param [Google::Cloud::Pubsub::Subscription] subscription
       # @param [<#handle, #on_error>] interceptors
@@ -20,6 +23,9 @@ module Gcpc
         @interceptors    = interceptors
         @ack_immediately = ack_immediately
         @logger          = logger
+        @subscriber_thread_status = {}
+        @subscriber_thread_status_mutex = Mutex.new
+        @heartbeat_worker_thread = nil
 
         @subscriber      = nil  # @subscriber is created by calling `#run`
         @handler         = nil  # @handler must be registered by `#handle`
@@ -47,6 +53,8 @@ module Gcpc
 
         @logger.info("Started")
 
+        run_heartbeat_worker
+
         loop_until_receiving_signals(signals)
       end
 
@@ -64,6 +72,16 @@ module Gcpc
         @logger.info('Stopping, will wait for background threads to exit')
 
         @subscriber.stop
+
+        begin
+          @heartbeat_worker_thread&.wakeup
+        # ThreadError exeption will be raised when the thread already dead
+        rescue ThreadError => e
+          @logger.error(e.message)
+        end
+
+        @heartbeat_worker_thread&.join
+
         @subscriber.wait!
 
         @logger.info('Stopped, background threads are shutdown')
@@ -92,12 +110,51 @@ module Gcpc
         stop unless stopped?
       end
 
+      def run_heartbeat_worker
+        @heartbeat_worker_thread = Thread.new do
+          @logger.info("Starting heartbeat worker...")
+          begin
+            loop do
+              break if stopped?
+
+              next unless alive?
+
+              Gcpc::Config.instance.beat.each(&:call)
+
+              sleep BEAT_INTERVAL
+            end
+          ensure
+            @logger.info("heartbeat worker stopped")
+          end
+        end
+
+        @heartbeat_worker_thread.name = HEART_BEAT_WORKER_NAME
+      end
+
+      def alive?
+        # ・When processing a message, write the thread_id and timestamp at the start time into @subscriber_thread_status,
+        #   and remove that information from @subscriber_thread_status when the processing within that thread is finished.
+        #   @subscriber_thread_status = {#<Thread:0x000000010302cd40 run>=>1690757417}
+        # ・If the processing of the message gets stuck, the key, value will not be removed from @subscriber_thread_status.
+        # ・Since the application holds as many callback_threads as @subscriber.callback_threads with Subscription,
+        #   if the number of threads that have gotten stuck exceeds that callback_threads, it is considered that the worker unable to process Subscription queue.
+        return false unless @subscriber && @subscriber.started?
+        return false if @subscriber.stopped?
+
+        number_of_dead_threads = @subscriber_thread_status.count { |_, v| v < Time.now.to_i - WORKER_DEAD_THRESHOLD }
+
+        return @subscriber.callback_threads > number_of_dead_threads
+      end
+
+
       # @param [Google::Cloud::Pubsub::ReceivedMessage] message
       def handle_message(message)
+        write_heartbeat_to_subscriber_thread_status
+
         ack(message) if @ack_immediately
 
         begin
-          worker_info("Started hanlding message")
+          worker_info("Started handling message")
           @handler.handle(message)
           worker_info("Finished hanlding message successfully")
         rescue => e
@@ -110,11 +167,13 @@ module Gcpc
 
       def ack(message)
         message.ack!
+        cleanup_subscriber_thread_status
         worker_info("Acked message")
       end
 
       def nack(message)
         message.nack!
+        cleanup_subscriber_thread_status
         worker_info("Nacked message")
       end
 
@@ -139,6 +198,27 @@ module Gcpc
       def stopped?
         @stopped_mutex.synchronize { @stopped }
       end
+      
+      def write_heartbeat_to_subscriber_thread_status
+        begin
+          @subscriber_thread_status_mutex.synchronize do
+            @subscriber_thread_status[Thread.current] = Time.now.to_i
+          end
+        rescue ThreadError => e
+          @logger.info("Falied to write subscriber_thread_status. thread_id: #{Thread.current.object_id}, subscriber_thread_status: #{@subscriber_thread_status}, error: #{e.message}")
+        end
+      end
+
+      def cleanup_subscriber_thread_status
+        begin
+          @subscriber_thread_status_mutex.synchronize do
+            # cleanup to avoid memory leak
+            @subscriber_thread_status.delete(Thread.current)
+          end
+        rescue ThreadError => e
+          @logger.info("Falied to cleanup subscriber_thread_status. thread_id: #{Thread.current.object_id}, subscriber_thread_status: #{@subscriber_thread_status}, error: #{e.message}")
+        end
+      end
     end
   end
 end

data/lib/gcpc/version.rb

@@ -1,3 +1,3 @@
 module Gcpc
-  VERSION = "0.0.7"
+  VERSION = "0.0.8"
 end

data/lib/gcpc.rb

@@ -1,6 +1,7 @@
 require "gcpc/version"
 require "gcpc/publisher"
 require "gcpc/subscriber"
+require "gcpc/config"
 
 module Gcpc
 end

data/renovate.json

@@ -0,0 +1,5 @@
+{
+  "extends": [
+    "config:base"
+  ]
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gcpc
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.8
 platform: ruby
 authors:
 - Nao Minami
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-01-12 00:00:00.000000000 Z
+date: 2023-08-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -87,9 +87,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rspec"
-- ".travis.yml"
 - CHANGELOG.md
 - Gemfile
 - Gemfile.lock
@@ -104,6 +104,7 @@ files:
 - examples/subscriber-example/subscriber-example.rb
 - gcpc.gemspec
 - lib/gcpc.rb
+- lib/gcpc/config.rb
 - lib/gcpc/publisher.rb
 - lib/gcpc/publisher/base_interceptor.rb
 - lib/gcpc/publisher/engine.rb
@@ -118,7 +119,8 @@ files:
 - lib/gcpc/subscriber/handle_engine.rb
 - lib/gcpc/subscriber/subscription_client.rb
 - lib/gcpc/version.rb
-homepage: https://github.com/south37/gcpc
+- renovate.json
+homepage: https://github.com/wantedly/gcpc
 licenses:
 - MIT
 metadata: {}
@@ -137,7 +139,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.4.10
 signing_key:
 specification_version: 4
 summary: Google Cloud Pub/Sub Client

data/.travis.yml

@@ -1,21 +0,0 @@
----
-sudo: false
-language: ruby
-cache: bundler
-rvm:
-  - 2.6.6
-  - 2.7.3
-before_install:
-  - gem install bundler
-jobs:
-  include:
-    - name: 'test'
-      script:
-        - bundle exec rspec
-    - name: 'test with a emulator'
-      before_install:
-        # Start a Cloud Pub/Sub emulator. See https://cloud.google.com/pubsub/docs/emulator
-        # We use `google/cloud-sdk:321.0.0` because we can't start pubsub with the latest.
-        - docker run -d -p 8085:8085 -it google/cloud-sdk:321.0.0 gcloud beta emulators pubsub start --host-port=0.0.0.0:8085
-      script:
-        - bundle exec rspec --tag emulator