gcpc 0.0.6 → 0.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d4dfcbba14815f166e194aa5d106bf86c1adb86d4feb7bd1b406808974a2137d
-  data.tar.gz: ed718afdbaeeb0b415cd9f5923b30f69dd9a17066712f4a932f2d55102da27c1
+  metadata.gz: d89748f1a7066c40f0b4c6c4477bab098ab45c206ca7f019d5f29f13be62c875
+  data.tar.gz: 76261a92198cd9e53629455aba3859117e649004838da3752ead8dbbe8445f14
 SHA512:
-  metadata.gz: 21c978898799abeb533747aca3bcb52d152d9f92c6260db56ca5b0da74b08985a521974b9707a3d36a3772ebe1017edab5a2ee2d948841a81e414594647fa39a
-  data.tar.gz: 1d99113e0c48bad884963d12c43878efb55681fe47268ce1fbc2bb975ffb9a6d83e426f7dc654810ea6a1b4227542c3b21c6a8c2acd6c2185c6e2a030addc52d
+  metadata.gz: fdf792b95470615cc0e9967dab7efaccde16aa62a60dbde2c0c0728df37e13d51cc3f046585152fde3a528032ebe39b59fee8a05e0f86680ffafb37bfd4b5350
+  data.tar.gz: 77b9b3d6f3a61bfeef13b4094363aa98c1a91679dd532712753061abc761d4a13e31696b7036b9d84c9b78a9b6cd3e055b896923624859d8a82cf4693af2762b

data/.github/workflows/ci.yml

@@ -0,0 +1,24 @@
+name: test
+
+on:
+  push:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby-version: ['3.2', '3.1', '3.0']
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Ruby ${{ matrix.ruby-version }}
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+          bundler-cache: true
+      - name: Run rspec
+        run: bundle exec rspec
+      - name: run pubsub emulator
+        run: docker run -d -p 8085:8085 -it gcr.io/google.com/cloudsdktool/google-cloud-cli:latest gcloud beta emulators pubsub start --host-port=0.0.0.0:8085
+      - name: Run rspec with emulator
+        run: bundle exec rspec --tag emulator

data/CHANGELOG.md

@@ -0,0 +1,23 @@
+## Unreleased
+
+<!-- When you commit changes, describe the changes above (in the ## Unreleased section). -->
+<!-- All changes (implementation, or doc changes) are worth noting. -->
+<!-- The changes listed above will go below when a new release is being prepared. -->
+
+## 0.0.8
+
+- Add an API for heartbeat https://github.com/wantedly/gcpc/pull/25
+
+## 0.0.7
+
+- Add explicit loading of forwardable https://github.com/wantedly/gcpc/pull/10
+- Add ruby 2.7 as a test target https://github.com/wantedly/gcpc/pull/12
+
+## 0.0.6
+
+- Add test of interceptors https://github.com/wantedly/gcpc/pull/7
+- Add publish_batch https://github.com/wantedly/gcpc/pull/6
+
+## 0.0.5
+
+- Add publish_async https://github.com/wantedly/gcpc/pull/5

data/Gemfile.lock

@@ -1,67 +1,82 @@
 PATH
   remote: .
   specs:
-    gcpc (0.0.6)
+    gcpc (0.0.8)
       google-cloud-pubsub
 
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.6.0)
-      public_suffix (>= 2.0.2, < 4.0)
-    coderay (1.1.2)
-    concurrent-ruby (1.1.5)
+    addressable (2.8.5)
+      public_suffix (>= 2.0.2, < 6.0)
+    coderay (1.1.3)
+    concurrent-ruby (1.2.2)
     diff-lcs (1.3)
-    faraday (0.15.4)
-      multipart-post (>= 1.2, < 3)
-    google-cloud-core (1.3.0)
+    faraday (2.7.10)
+      faraday-net_http (>= 2.0, < 3.1)
+      ruby2_keywords (>= 0.0.4)
+    faraday-net_http (3.0.2)
+    faraday-retry (2.2.0)
+      faraday (~> 2.0)
+    gapic-common (0.19.1)
+      faraday (>= 1.9, < 3.a)
+      faraday-retry (>= 1.0, < 3.a)
+      google-protobuf (~> 3.14)
+      googleapis-common-protos (>= 1.3.12, < 2.a)
+      googleapis-common-protos-types (>= 1.3.1, < 2.a)
+      googleauth (~> 1.0)
+      grpc (~> 1.36)
+    google-cloud-core (1.6.0)
       google-cloud-env (~> 1.0)
-    google-cloud-env (1.2.0)
-      faraday (~> 0.11)
-    google-cloud-pubsub (0.38.0)
+      google-cloud-errors (~> 1.0)
+    google-cloud-env (1.6.0)
+      faraday (>= 0.17.3, < 3.0)
+    google-cloud-errors (1.3.1)
+    google-cloud-pubsub (2.15.4)
       concurrent-ruby (~> 1.1)
-      google-cloud-core (~> 1.2)
-      google-gax (~> 1.7)
-      googleapis-common-protos (>= 1.3.9, < 2.0)
-      grpc-google-iam-v1 (~> 0.6.9)
-    google-gax (1.7.0)
-      google-protobuf (~> 3.2)
-      googleapis-common-protos (>= 1.3.5, < 2.0)
-      googleauth (>= 0.6.2, < 0.10.0)
-      grpc (>= 1.7.2, < 2.0)
-      rly (~> 0.2.3)
-    google-protobuf (3.9.0-universal-darwin)
-    googleapis-common-protos (1.3.9)
-      google-protobuf (~> 3.0)
-      googleapis-common-protos-types (~> 1.0)
-      grpc (~> 1.0)
-    googleapis-common-protos-types (1.0.4)
-      google-protobuf (~> 3.0)
-    googleauth (0.8.1)
-      faraday (~> 0.12)
+      google-cloud-core (~> 1.5)
+      google-cloud-pubsub-v1 (~> 0.8)
+      retriable (~> 3.1)
+    google-cloud-pubsub-v1 (0.17.3)
+      gapic-common (>= 0.19.1, < 2.a)
+      google-cloud-errors (~> 1.0)
+      google-iam-v1 (>= 0.4, < 2.a)
+    google-iam-v1 (0.5.2)
+      gapic-common (>= 0.19.1, < 2.a)
+      google-cloud-errors (~> 1.0)
+      grpc-google-iam-v1 (~> 1.1)
+    google-protobuf (3.23.4)
+    googleapis-common-protos (1.4.0)
+      google-protobuf (~> 3.14)
+      googleapis-common-protos-types (~> 1.2)
+      grpc (~> 1.27)
+    googleapis-common-protos-types (1.8.0)
+      google-protobuf (~> 3.18)
+    googleauth (1.7.0)
+      faraday (>= 0.17.3, < 3.a)
       jwt (>= 1.4, < 3.0)
       memoist (~> 0.16)
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
-      signet (~> 0.7)
-    grpc (1.22.0-universal-darwin)
-      google-protobuf (~> 3.8)
+      signet (>= 0.16, < 2.a)
+    grpc (1.56.2)
+      google-protobuf (~> 3.23)
       googleapis-common-protos-types (~> 1.0)
-    grpc-google-iam-v1 (0.6.9)
-      googleapis-common-protos (>= 1.3.1, < 2.0)
-      grpc (~> 1.0)
-    jwt (2.2.1)
-    memoist (0.16.0)
-    method_source (0.9.2)
-    multi_json (1.13.1)
-    multipart-post (2.1.1)
-    os (1.0.1)
-    pry (0.12.2)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
-    public_suffix (3.1.1)
-    rake (10.5.0)
-    rly (0.2.3)
+    grpc-google-iam-v1 (1.3.0)
+      google-protobuf (~> 3.18)
+      googleapis-common-protos (~> 1.4)
+      grpc (~> 1.41)
+    jwt (2.7.1)
+    memoist (0.16.2)
+    method_source (1.0.0)
+    multi_json (1.15.0)
+    os (1.1.4)
+    pry (0.13.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    public_suffix (5.0.3)
+    rake (13.0.1)
+    retriable (3.1.2)
     rspec (3.8.0)
       rspec-core (~> 3.8.0)
       rspec-expectations (~> 3.8.0)
@@ -75,9 +90,10 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
     rspec-support (3.8.0)
-    signet (0.11.0)
-      addressable (~> 2.3)
-      faraday (~> 0.9)
+    ruby2_keywords (0.0.5)
+    signet (0.17.0)
+      addressable (~> 2.8)
+      faraday (>= 0.17.5, < 3.a)
       jwt (>= 1.5, < 3.0)
       multi_json (~> 1.10)
 
@@ -85,11 +101,11 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 1.17)
+  bundler
   gcpc!
   pry
-  rake (~> 10.0)
+  rake (~> 13.0)
   rspec (~> 3.2)
 
 BUNDLED WITH
-   1.17.2
+   2.2.5

data/README.md

@@ -154,7 +154,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/south37/gcpc.
+Bug reports and pull requests are welcome on GitHub at https://github.com/wantedly/gcpc.
 
 ## License
 

data/gcpc.gemspec

@@ -11,7 +11,7 @@ Gem::Specification.new do |spec|
 
   spec.summary       = %q{Google Cloud Pub/Sub Client}
   spec.description   = %q{Google Cloud Pub/Sub Client}
-  spec.homepage      = "https://github.com/south37/gcpc"
+  spec.homepage      = "https://github.com/wantedly/gcpc"
   spec.license       = "MIT"
 
   # Specify which files should be added to the gem when it is released.
@@ -23,8 +23,8 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.17"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.2"
   spec.add_development_dependency "pry"
   spec.add_runtime_dependency "google-cloud-pubsub"

data/lib/gcpc/config.rb

@@ -0,0 +1,18 @@
+require "singleton"
+
+module Gcpc
+  class Config
+    include Singleton
+    attr_reader :beat
+
+    def initialize
+      @beat = []
+    end
+
+    def on(event, &block)
+      raise ArgumentError, "Invalid event name: #{event}" if event != :beat
+
+      @beat << block
+    end
+  end
+end

data/lib/gcpc/publisher.rb

@@ -1,3 +1,4 @@
+require "forwardable"
 require "gcpc/publisher/base_interceptor"
 require "gcpc/publisher/engine"
 require "gcpc/publisher/topic_client"

data/lib/gcpc/subscriber/engine.rb

@@ -4,6 +4,9 @@ module Gcpc
   class Subscriber
     class Engine
       WAIT_INTERVAL = 1
+      WORKER_DEAD_THRESHOLD = 30 # second
+      BEAT_INTERVAL = 10
+      HEART_BEAT_WORKER_NAME = 'heartbeat-worker'
 
       # @param [Google::Cloud::Pubsub::Subscription] subscription
       # @param [<#handle, #on_error>] interceptors
@@ -20,6 +23,9 @@ module Gcpc
         @interceptors    = interceptors
         @ack_immediately = ack_immediately
         @logger          = logger
+        @subscriber_thread_status = {}
+        @subscriber_thread_status_mutex = Mutex.new
+        @heartbeat_worker_thread = nil
 
         @subscriber      = nil  # @subscriber is created by calling `#run`
         @handler         = nil  # @handler must be registered by `#handle`
@@ -47,6 +53,8 @@ module Gcpc
 
         @logger.info("Started")
 
+        run_heartbeat_worker
+
         loop_until_receiving_signals(signals)
       end
 
@@ -64,6 +72,16 @@ module Gcpc
         @logger.info('Stopping, will wait for background threads to exit')
 
         @subscriber.stop
+
+        begin
+          @heartbeat_worker_thread&.wakeup
+        # ThreadError exeption will be raised when the thread already dead
+        rescue ThreadError => e
+          @logger.error(e.message)
+        end
+
+        @heartbeat_worker_thread&.join
+
         @subscriber.wait!
 
         @logger.info('Stopped, background threads are shutdown')
@@ -92,12 +110,51 @@ module Gcpc
         stop unless stopped?
       end
 
+      def run_heartbeat_worker
+        @heartbeat_worker_thread = Thread.new do
+          @logger.info("Starting heartbeat worker...")
+          begin
+            loop do
+              break if stopped?
+
+              next unless alive?
+
+              Gcpc::Config.instance.beat.each(&:call)
+
+              sleep BEAT_INTERVAL
+            end
+          ensure
+            @logger.info("heartbeat worker stopped")
+          end
+        end
+
+        @heartbeat_worker_thread.name = HEART_BEAT_WORKER_NAME
+      end
+
+      def alive?
+        # ・When processing a message, write the thread_id and timestamp at the start time into @subscriber_thread_status,
+        #   and remove that information from @subscriber_thread_status when the processing within that thread is finished.
+        #   @subscriber_thread_status = {#<Thread:0x000000010302cd40 run>=>1690757417}
+        # ・If the processing of the message gets stuck, the key, value will not be removed from @subscriber_thread_status.
+        # ・Since the application holds as many callback_threads as @subscriber.callback_threads with Subscription,
+        #   if the number of threads that have gotten stuck exceeds that callback_threads, it is considered that the worker unable to process Subscription queue.
+        return false unless @subscriber && @subscriber.started?
+        return false if @subscriber.stopped?
+
+        number_of_dead_threads = @subscriber_thread_status.count { |_, v| v < Time.now.to_i - WORKER_DEAD_THRESHOLD }
+
+        return @subscriber.callback_threads > number_of_dead_threads
+      end
+
+
       # @param [Google::Cloud::Pubsub::ReceivedMessage] message
       def handle_message(message)
+        write_heartbeat_to_subscriber_thread_status
+
         ack(message) if @ack_immediately
 
         begin
-          worker_info("Started hanlding message")
+          worker_info("Started handling message")
           @handler.handle(message)
           worker_info("Finished hanlding message successfully")
         rescue => e
@@ -110,11 +167,13 @@ module Gcpc
 
       def ack(message)
         message.ack!
+        cleanup_subscriber_thread_status
         worker_info("Acked message")
       end
 
       def nack(message)
         message.nack!
+        cleanup_subscriber_thread_status
         worker_info("Nacked message")
       end
 
@@ -139,6 +198,27 @@ module Gcpc
       def stopped?
         @stopped_mutex.synchronize { @stopped }
       end
+      
+      def write_heartbeat_to_subscriber_thread_status
+        begin
+          @subscriber_thread_status_mutex.synchronize do
+            @subscriber_thread_status[Thread.current] = Time.now.to_i
+          end
+        rescue ThreadError => e
+          @logger.info("Falied to write subscriber_thread_status. thread_id: #{Thread.current.object_id}, subscriber_thread_status: #{@subscriber_thread_status}, error: #{e.message}")
+        end
+      end
+
+      def cleanup_subscriber_thread_status
+        begin
+          @subscriber_thread_status_mutex.synchronize do
+            # cleanup to avoid memory leak
+            @subscriber_thread_status.delete(Thread.current)
+          end
+        rescue ThreadError => e
+          @logger.info("Falied to cleanup subscriber_thread_status. thread_id: #{Thread.current.object_id}, subscriber_thread_status: #{@subscriber_thread_status}, error: #{e.message}")
+        end
+      end
     end
   end
 end

data/lib/gcpc/subscriber.rb

@@ -1,3 +1,4 @@
+require "forwardable"
 require "gcpc/subscriber/base_handler"
 require "gcpc/subscriber/base_interceptor"
 require "gcpc/subscriber/default_logger"

data/lib/gcpc/version.rb

@@ -1,3 +1,3 @@
 module Gcpc
-  VERSION = "0.0.6"
+  VERSION = "0.0.8"
 end

data/lib/gcpc.rb

@@ -1,6 +1,7 @@
 require "gcpc/version"
 require "gcpc/publisher"
 require "gcpc/subscriber"
+require "gcpc/config"
 
 module Gcpc
 end

data/renovate.json

@@ -0,0 +1,5 @@
+{
+  "extends": [
+    "config:base"
+  ]
+}

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: gcpc
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.8
 platform: ruby
 authors:
 - Nao Minami
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2019-08-05 00:00:00.000000000 Z
+date: 2023-08-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.17'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.17'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -87,9 +87,10 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rspec"
-- ".travis.yml"
+- CHANGELOG.md
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
@@ -103,6 +104,7 @@ files:
 - examples/subscriber-example/subscriber-example.rb
 - gcpc.gemspec
 - lib/gcpc.rb
+- lib/gcpc/config.rb
 - lib/gcpc/publisher.rb
 - lib/gcpc/publisher/base_interceptor.rb
 - lib/gcpc/publisher/engine.rb
@@ -117,11 +119,12 @@ files:
 - lib/gcpc/subscriber/handle_engine.rb
 - lib/gcpc/subscriber/subscription_client.rb
 - lib/gcpc/version.rb
-homepage: https://github.com/south37/gcpc
+- renovate.json
+homepage: https://github.com/wantedly/gcpc
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -136,8 +139,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.4.10
+signing_key:
 specification_version: 4
 summary: Google Cloud Pub/Sub Client
 test_files: []

data/.travis.yml

@@ -1,19 +0,0 @@
----
-sudo: false
-language: ruby
-cache: bundler
-rvm:
-  - 2.6.2
-before_install:
-  - gem install bundler -v 1.17.2
-jobs:
-  include:
-    - name: 'test'
-      script:
-        - bundle exec rspec
-    - name: 'test with a emulator'
-      before_install:
-        # Start a Cloud Pub/Sub emulator. See https://cloud.google.com/pubsub/docs/emulator
-        - docker run -d -p 8085:8085 -it google/cloud-sdk:latest gcloud beta emulators pubsub start --host-port=0.0.0.0:8085
-      script:
-        - bundle exec rspec --tag emulator