cloudenvoy 0.5.0 → 0.6.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 797f52c1101a9e1727d691794494be30e2d8d40a196d64316563596fb071d91b
-  data.tar.gz: e4975868db644a18733020bd78b216aa3ad2a1fb6293c554dc88f39405c39483
+  metadata.gz: cb4f27d4f01e971d8542f5768166caa9d797ad9799372bf167ddfe5e970ecb51
+  data.tar.gz: 5f2ed34bde0799fc94d574203f4589ede66b90da39a4f8e9bf4b6a8b41fac483
 SHA512:
-  metadata.gz: 056d422ee16380228e6e7e8b774669f1935fec652cfd7fad840f776096fa566e5cc6a6eecb0d4cd9839323e9a65f9bb4866d38d927d86ac0475c58af98e868c0
-  data.tar.gz: 1e43db1aa3f40ea7727c2bd588133bac7187e7d98a04624c6c855c29ee0f48e6375594e5c791c8f1c62b0cd47775071383fb6183fb7a960afa3c638e3b1832a7
+  metadata.gz: 6a71d84e06873d76352df8ccef0ee32f95a439e62447bf90e2a57694816753337186b6f57ad9864911d8bfec270eff31b8ccd0183c36f894ce030cd1b5c6ccb8
+  data.tar.gz: ac0be67ec3bc95c2b9cc32745dcbd72c87fcd06cf329dbf007cde92139b8c70a712545afc7e69bb44bd61397641989e1af7f8ff41bdeb82efe5400ca84993a58

data/.gitignore

@@ -5,6 +5,7 @@
 /doc/
 /examples/rails/log/*.log
 /examples/rails/tmp/
+/gemfiles/*.gemfile.lock
 /pkg/
 /spec/reports/
 /tmp/

data/.rubocop.yml

@@ -1,14 +1,13 @@
 require: rubocop-rspec
 
 AllCops:
+  NewCops: enable
   Exclude:
     - 'gemfiles/**/*'
     - 'vendor/**/*'
 
-# Ruby 3.0: curly braces around last argument has meaning
-# See: https://github.com/rubocop/rubocop/issues/7641
-Style/BracesAroundHashParameters:
-  Enabled: false
+Layout/LineLength:
+  Max: 160
 
 Metrics/ClassLength:
   Max: 150
@@ -17,14 +16,20 @@ Metrics/ModuleLength:
   Max: 150
 
 Metrics/AbcSize:
-  Max: 20
-
-Metrics/LineLength:
-  Max: 120
+  Max: 30
 
 Metrics/MethodLength:
   Max: 20
 
+Metrics/BlockLength:
+  Exclude:
+    - cloudenvoy.gemspec
+    - lib/tasks/**/*
+    - 'spec/**/*'
+
+Metrics/ParameterLists:
+  CountKeywordArgs: false
+
 RSpec/ExpectInHook:
   Enabled: false
 
@@ -34,19 +39,10 @@ RSpec/EmptyLineAfterHook:
 RSpec/ScatteredSetup:
   Enabled: false
 
-Metrics/BlockLength:
-  Exclude:
-    - cloudenvoy.gemspec
-    - lib/tasks/**/*
-    - 'spec/**/*'
+RSpec/MessageSpies:
+  Enabled: false
 
 Style/Documentation:
   Exclude:
     - 'examples/**/*'
-    - 'spec/**/*'
-
-Metrics/ParameterLists:
-  CountKeywordArgs: false
-
-RSpec/MessageSpies:
-  Enabled: false
+    - 'spec/**/*'

data/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## [v0.6.rc1](https://github.com/keypup-io/cloudenvoy/tree/v0.6.rc1) (2022-04-11)
+
+[Full Changelog](https://github.com/keypup-io/cloudenvoy/compare/v0.5.0...v0.6.rc1)
+
+**Improvements:**
+- Pub/sub: Add `publish_all` method to publish multiple messages in one batch
+- Ruby: Drop support for Ruby `2.3`
+- Style: Upgrade rubocop to `0.93.0`
+
 ## [v0.5.0](https://github.com/keypup-io/cloudenvoy/tree/v0.5.0) (2022-03-11)
 
 [Full Changelog](https://github.com/keypup-io/cloudenvoy/compare/v0.4.2...v0.5.0)

data/README.md

@@ -35,9 +35,6 @@ Add this line to your application's Gemfile:
 
 ```ruby
 gem 'cloudenvoy'
-
-# For Ruby 3 support, please use v0.5.rc1
-gem 'cloudenvoy, '~> 0.5.rc1'
 ```
 
 And then execute:
@@ -136,7 +133,12 @@ rails s -p 3000
 
 Open a Rails console and send a message
 ```ruby
+  # One message at a time
   DummyPublisher.publish('Hello pub/sub')
+
+  # Publish multiple messages in one batch
+  # Only available since v0.6.rc1
+  DummyPublisher.publish_all(['Hello pub/sub', 'Hello again!', 'Hello again and again!'])
 ```
 
 Your Rails logs should display the following:
@@ -252,10 +254,21 @@ DummySubscriber.setup
 ## Publishing messages
 
 ### Sending messages
+**Note**: The `publish_all` method is only available since `v0.6.rc1`
 
 Cloudenvoy provides a helper method to publish arbitrary messages to any topic.
 ```ruby
+# Publish a single message
 Cloudenvoy.publish('my-topic', { 'some' => 'payload' }, { 'optional' => 'message attribute' })
+
+# Publish multiple messages in one batch
+# Only available since v0.6.rc1
+Cloudenvoy.publish_all('my-topic', [
+  # Message 1
+  [{ 'some' => 'msg1 payload' }, { 'optional' => 'msg1 attribute' }],
+  # Message 2
+  [{ 'some' => 'msg2 payload' }, { 'optional' => 'msg2 attribute' }]
+])
 ```
 
 This helper is useful for sending basic messages however it is not the preferred way of sending messages as you will quickly clutter your application with message formatting logic over time.
@@ -292,8 +305,21 @@ Then in your user model you can do the following:
 class User < ApplicationRecord
   after_create :publish_user
 
+  # Example: publish multiple messages in one batch
+  # Only available since v0.6.rc1
+  # 
+  #
+  # Set user status to 'active' without involving callbacks
+  # then manually publish the new state of users in one batch
+  def self.enable_all_users
+    User.update_all(status: 'active', updated_at: Time.current)
+    UserPublisher.publish_all(User.all)
+  end
+
   private
 
+  # Example: publish one message at a time
+  #
   # Publish users after they have been created
   def publish_user
     UserPublisher.publish(self)
@@ -604,9 +630,17 @@ end
 
 ## Development
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+After checking out the repo, run `bin/setup` to install dependencies.
+
+For tests, run `rake` to run the tests. Note that Rails is not in context by default, which means Rails-specific test will not run.
+For tests including Rails-specific tests, run `bundle exec appraisal rails-7.0 rake`
+For all context-specific tests (incl. Rails), run the [appraisal tests](Appraisals) using `bundle exec appraisal rake`.
+
+You can run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`.
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
 ## Contributing
 

data/cloudenvoy.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |spec|
   spec.description   = 'Cross-application messaging using GCP Pub/Sub (alpha)'
   spec.homepage      = 'https://github.com/keypup-io/cloudenvoy'
   spec.license       = 'MIT'
-  spec.required_ruby_version = Gem::Requirement.new('>= 2.3.0')
+  spec.required_ruby_version = Gem::Requirement.new('>= 2.4.0')
 
   spec.metadata['homepage_uri'] = spec.homepage
   spec.metadata['source_code_uri'] = 'https://github.com/keypup-io/cloudenvoy'
@@ -35,7 +35,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'appraisal'
   spec.add_development_dependency 'rake', '>= 12.3.3'
   spec.add_development_dependency 'rspec', '~> 3.0'
-  spec.add_development_dependency 'rubocop', '0.76.0'
+  spec.add_development_dependency 'rubocop', '0.93.0'
   spec.add_development_dependency 'rubocop-rspec', '1.37.0'
   spec.add_development_dependency 'semantic_logger'
   spec.add_development_dependency 'timecop'

data/examples/rails/Gemfile

@@ -9,7 +9,7 @@ ruby '3.0.0'
 gem 'rails', '~> 6.0.2'
 
 # Server
-gem 'puma', '~> 5.6.2'
+gem 'puma', '~> 5.6.4'
 
 # Messaging via GCP Pub/Sub
 gem 'cloudenvoy', path: '../../'

data/examples/rails/Gemfile.lock

@@ -160,7 +160,7 @@ GEM
       racc (~> 1.4)
     os (1.1.4)
     public_suffix (4.0.6)
-    puma (5.6.2)
+    puma (5.6.4)
       nio4r (~> 2.0)
     racc (1.6.0)
     rack (2.2.3)
@@ -222,7 +222,7 @@ PLATFORMS
 
 DEPENDENCIES
   cloudenvoy!
-  puma (~> 5.6.2)
+  puma (~> 5.6.4)
   rails (~> 6.0.2)
   sqlite3
 

data/examples/rails/README.md

@@ -12,6 +12,10 @@
 HelloPublisher.publish('Some message')
 ```
 7. Tail the logs to see how message get processed by `HelloSubscriber`
+8. Try sending multiple messages at once:
+```ruby
+HelloPublisher.publish_all(['message1', 'message2', 'message3'])
+```
 
 ## Run using GCP Pub/Sub
 

data/examples/rails/app/subscribers/hello_subscriber.rb

@@ -11,6 +11,6 @@ class HelloSubscriber
   # @param [Cloudenvoy::Message] message The message to process.
   #
   def process(message)
-    logger.info("Received message #{message.payload.dig('content')}")
+    logger.info("Received message #{message.payload['content']}")
   end
 end

data/examples/rails/config/environments/production.rb

@@ -83,7 +83,7 @@ Rails.application.configure do
   # config.logger = ActiveSupport::TaggedLogging.new(Syslog::Logger.new 'app-name')
 
   if ENV['RAILS_LOG_TO_STDOUT'].present?
-    logger           = ActiveSupport::Logger.new(STDOUT)
+    logger           = ActiveSupport::Logger.new($stdout)
     logger.formatter = config.log_formatter
     config.logger    = ActiveSupport::TaggedLogging.new(logger)
   end

data/examples/rails/config/puma.rb

@@ -6,17 +6,17 @@
 # the maximum value specified for Puma. Default is set to 5 threads for minimum
 # and maximum; this matches the default thread size of Active Record.
 #
-max_threads_count = ENV.fetch('RAILS_MAX_THREADS') { 5 }
+max_threads_count = ENV.fetch('RAILS_MAX_THREADS', 5)
 min_threads_count = ENV.fetch('RAILS_MIN_THREADS') { max_threads_count }
 threads min_threads_count, max_threads_count
 
 # Specifies the `port` that Puma will listen on to receive requests; default is 3000.
 #
-port        ENV.fetch('PORT') { 3000 }
+port        ENV.fetch('PORT', 3000)
 
 # Specifies the `environment` that Puma will run in.
 #
-environment ENV.fetch('RAILS_ENV') { 'development' }
+environment ENV.fetch('RAILS_ENV', 'development')
 
 # Specifies the number of `workers` to boot in clustered mode.
 # Workers are forked web server processes. If using threads and workers together

data/examples/sinatra/Gemfile

@@ -9,7 +9,7 @@ ruby '3.0.0'
 gem 'sinatra'
 
 # Server
-gem 'puma', '~> 5.6.2'
+gem 'puma', '~> 5.6.4'
 
 # Support rake tasks
 gem 'rake'

data/examples/sinatra/Gemfile.lock

@@ -99,7 +99,7 @@ GEM
     nio4r (2.5.8)
     os (1.1.4)
     public_suffix (4.0.6)
-    puma (5.6.2)
+    puma (5.6.4)
       nio4r (~> 2.0)
     rack (2.2.3)
     rack-protection (2.2.0)
@@ -128,7 +128,7 @@ PLATFORMS
 
 DEPENDENCIES
   cloudenvoy!
-  puma (~> 5.6.2)
+  puma (~> 5.6.4)
   rake
   sinatra
 

data/examples/sinatra/README.md

@@ -17,6 +17,10 @@ Cloudenvoy.setup_subscribers
 HelloPublisher.publish('Some message')
 ```
 8. Tail the logs to see how message get processed by `HelloSubscriber`
+9. Try sending multiple messages at once:
+```ruby
+HelloPublisher.publish_all(['message1', 'message2', 'message3'])
+```
 
 ## Run using GCP Pub/Sub
 

data/examples/sinatra/app/subscribers/hello_subscriber.rb

@@ -11,6 +11,6 @@ class HelloSubscriber
   # @param [Cloudenvoy::Message] message The message to process.
   #
   def process(message)
-    logger.info("Received message #{message.payload.dig('content')}")
+    logger.info("Received message #{message.payload['content']}")
   end
 end

data/examples/sinatra/app.rb

@@ -1,13 +1,13 @@
 # frozen_string_literal: true
 
 # Force logging to be realtime
-STDOUT.sync = true
+$stdout.sync = true
 
 # Require project files
 require 'sinatra'
-Dir.glob('./config/initializers/*.rb').each { |file| require file }
-Dir.glob('./app/publishers/*.rb').each { |file| require file }
-Dir.glob('./app/subscribers/*.rb').each { |file| require file }
+Dir.glob('./config/initializers/*.rb').sort.each { |file| require file }
+Dir.glob('./app/publishers/*.rb').sort.each { |file| require file }
+Dir.glob('./app/subscribers/*.rb').sort.each { |file| require file }
 
 #---------------------------------------------------
 # Routes

data/lib/cloudenvoy/backend/google_pub_sub.rb

@@ -54,7 +54,7 @@ module Cloudenvoy
       # @param [Hash, String] payload The message content.
       # @param [Hash] metadata The message attributes.
       #
-      # @return [Cloudenvoy::Message] The created message.
+      # @return [Cloudenvoy::Message] The published message.
       #
       def publish(topic, payload, metadata = {})
         # Retrieve the topic
@@ -72,6 +72,38 @@ module Cloudenvoy
         )
       end
 
+      #
+      # Publish multiple messages to a topic.
+      #
+      # @param [String] topic The name of the topic
+      # @param [Array<Array<[Hash, String]>>] msg_args A list of message [payload, metadata].
+      #
+      # @return [Array<Cloudenvoy::Message>] The published messages.
+      #
+      def publish_all(topic, msg_args)
+        # Retrieve the topic
+        ps_topic = backend.topic(topic, skip_lookup: true)
+
+        # Publish the message
+        ps_msgs = ps_topic.publish do |batch|
+          msg_args.each do |(payload, metadata)|
+            batch.publish(payload.to_json, metadata.to_h)
+          end
+        end
+
+        # Return the formatted messages
+        ps_msgs.each_with_index.map do |ps_msg, index|
+          payload, metadata = msg_args[index]
+
+          Message.new(
+            id: ps_msg.message_id,
+            payload: payload,
+            metadata: metadata,
+            topic: topic
+          )
+        end
+      end
+
       #
       # Create or update a subscription for a specific topic.
       #
@@ -134,8 +166,8 @@ module Cloudenvoy
       def upsert_topic(topic)
         ps_topic = begin
           backend.create_topic(topic)
-                   rescue Google::Cloud::AlreadyExistsError
-                     backend.topic(topic)
+        rescue Google::Cloud::AlreadyExistsError
+          backend.topic(topic)
         end
 
         # Return formatted subscription

data/lib/cloudenvoy/backend/memory_pub_sub.rb

@@ -47,7 +47,7 @@ module Cloudenvoy
       # @param [Hash, String] payload The message content.
       # @param [Hash] attrs The message attributes.
       #
-      # @return [Cloudenvoy::Message] The created message.
+      # @return [Cloudenvoy::Message] The published message.
       #
       def publish(topic, payload, metadata = {})
         msg = Message.new(
@@ -61,6 +61,30 @@ module Cloudenvoy
         msg
       end
 
+      #
+      # Publish multiple messages to a topic.
+      #
+      # @param [String] topic The name of the topic
+      # @param [Array<Array<[Hash, String]>>] msg_args A list of message [payload, metadata].
+      #
+      # @return [Array<Cloudenvoy::Message>] The published messages.
+      #
+      def publish_all(topic, msg_args)
+        # Build the messages
+        msgs = msg_args.map do |(payload, metadata)|
+          Message.new(
+            id: SecureRandom.uuid,
+            payload: payload,
+            metadata: metadata,
+            topic: topic
+          )
+        end
+
+        # Push all the messages and return them
+        queue(topic).push(*msgs)
+        msgs
+      end
+
       #
       # Create or update a subscription for a specific topic.
       #

data/lib/cloudenvoy/config.rb

@@ -57,7 +57,7 @@ module Cloudenvoy
     # @return [Logger, any] The cloudenvoy logger.
     #
     def logger
-      @logger ||= defined?(Rails) ? Rails.logger : ::Logger.new(STDOUT)
+      @logger ||= defined?(Rails) ? Rails.logger : ::Logger.new($stdout)
     end
 
     #

data/lib/cloudenvoy/message.rb

@@ -75,8 +75,8 @@ module Cloudenvoy
     def to_h
       {
         id: id,
-        payload: payload&.dup,
-        metadata: metadata&.dup,
+        payload: payload.dup,
+        metadata: metadata.dup,
         topic: topic,
         sub_uri: sub_uri
       }.compact

data/lib/cloudenvoy/pub_sub_client.rb

@@ -30,12 +30,24 @@ module Cloudenvoy
     # @param [Hash, String] payload The message content.
     # @param [Hash] attrs The message attributes.
     #
-    # @return [Cloudenvoy::Message] The created message.
+    # @return [Cloudenvoy::Message] The published message.
     #
     def self.publish(topic, payload, attrs = {})
       backend.publish(topic, payload, attrs)
     end
 
+    #
+    # Publish multiple messages to a topic.
+    #
+    # @param [String] topic The name of the topic
+    # @param [Array<Array<[Hash, String]>>] msg_args A list of message [payload, metadata].
+    #
+    # @return [Array<Cloudenvoy::Message>] The published messages.
+    #
+    def self.publish_all(topic, msg_args)
+      backend.publish_all(topic, msg_args)
+    end
+
     #
     # Create or update a subscription for a specific topic.
     #

data/lib/cloudenvoy/publisher.rb

@@ -89,6 +89,40 @@ module Cloudenvoy
 
         PubSubClient.upsert_topic(default_topic)
       end
+
+      #
+      # Publish all messages in one batch.
+      #
+      # @param [Array<Any>] The list of publisher arguments.
+      #
+      # @return [Array<Cloudenvoy::Message>] The published messages.
+      #
+      def publish_all(arg_list)
+        # Build the list of publishers
+        publishers = arg_list.map { |e| new(msg_args: [e].flatten(1)) }
+
+        # Batches are topic specific. We must send one batch of messages per topic.
+        publishers.group_by { |e| e.topic(*e.msg_args) }.map do |topic, topic_publishers|
+          # Chain publish calls. The very last call (when the chain becomes empty)
+          # is the actual batch publishing of messages to pub/sub
+          chain = topic_publishers.dup
+          traverse_chain = lambda do
+            if chain.empty?
+              # Send the messages in one batch and retrospectively attach them
+              # to each publisher
+              arg_list = topic_publishers.map { |e| [e.payload(*e.msg_args), e.metadata(*e.msg_args)] }
+              PubSubClient.publish_all(topic, arg_list).each_with_index do |msg, pub_index|
+                topic_publishers[pub_index].message = msg
+              end
+            else
+              # Defer message publishing to the next element
+              # in the chain until the chain is empty
+              chain.shift.publish(&traverse_chain)
+            end
+          end
+          traverse_chain.call
+        end.flatten
+      end
     end
 
     #
@@ -155,9 +189,9 @@ module Cloudenvoy
     #
     # @return [Cloudenvoy::Message] The created message.
     #
-    def publish
+    def publish(&block)
       # Format and publish message
-      resp = execute_middleware_chain
+      resp = execute_middleware_chain(&block)
 
       # Log job completion and return result
       logger.info("Published message in #{publishing_duration}s") { { duration: publishing_duration } }
@@ -191,14 +225,14 @@ module Cloudenvoy
     #
     # Execute the subscriber process method through the middleware chain.
     #
-    # @return [Any] The result of the perform method.
+    # @return [Cloudenvoy::Message] The published message.
     #
-    def execute_middleware_chain
+    def execute_middleware_chain(&block)
       self.publishing_started_at = Time.now
 
       Cloudenvoy.config.publisher_middleware.invoke(self) do
         begin
-          publish_message
+          block_given? ? block.call : publish_message
         rescue StandardError => e
           logger.error([e, e.backtrace.join("\n")].join("\n"))
           try(:on_error, e)

data/lib/cloudenvoy/publisher_logger.rb

@@ -9,7 +9,7 @@ module Cloudenvoy
     # @return [Proc] The context processor proc.
     #
     def self.default_context_processor
-      @default_context_processor ||= ->(loggable) { loggable.message&.to_h&.slice(:id, :metadata, :topic) || {} }
+      @default_context_processor ||= ->(loggable) { loggable.message.to_h&.slice(:id, :metadata, :topic) || {} }
     end
 
     #

data/lib/cloudenvoy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cloudenvoy
-  VERSION = '0.5.0'
+  VERSION = '0.6.rc1'
 end

data/lib/cloudenvoy.rb

@@ -63,6 +63,18 @@ module Cloudenvoy
     PubSubClient.publish(topic, payload, attrs)
   end
 
+  #
+  # Publish multiple messages to a topic. Shorthand method to Cloudenvoy::PubSubClient#publish_all.
+  #
+  # @param [String] topic The name of the topic
+  # @param [Array<Array<[Hash, String]>>] msg_args A list of message [payload, metadata].
+  #
+  # @return [Array<Cloudenvoy::Message>] The published messages.
+  #
+  def self.publish_all(topic, msg_args)
+    PubSubClient.publish_all(topic, msg_args)
+  end
+
   #
   # Return the list of registered publishers.
   #