cloudenvoy 0.5.rc1 → 0.6.0

data/examples/sinatra/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ../..
   specs:
-    cloudenvoy (0.4.2)
+    cloudenvoy (0.5.0)
       activesupport
       google-cloud-pubsub (~> 2.0)
       jwt
@@ -10,7 +10,7 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (7.0.0)
+    activesupport (7.0.2.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -18,26 +18,30 @@ GEM
     addressable (2.8.0)
       public_suffix (>= 2.0.2, < 5.0)
     concurrent-ruby (1.1.9)
-    faraday (1.8.0)
+    faraday (1.10.0)
       faraday-em_http (~> 1.0)
       faraday-em_synchrony (~> 1.0)
       faraday-excon (~> 1.1)
-      faraday-httpclient (~> 1.0.1)
+      faraday-httpclient (~> 1.0)
+      faraday-multipart (~> 1.0)
       faraday-net_http (~> 1.0)
-      faraday-net_http_persistent (~> 1.1)
+      faraday-net_http_persistent (~> 1.0)
       faraday-patron (~> 1.0)
       faraday-rack (~> 1.0)
-      multipart-post (>= 1.2, < 3)
+      faraday-retry (~> 1.0)
       ruby2_keywords (>= 0.0.4)
     faraday-em_http (1.0.0)
     faraday-em_synchrony (1.0.0)
     faraday-excon (1.1.0)
     faraday-httpclient (1.0.1)
+    faraday-multipart (1.0.3)
+      multipart-post (>= 1.2, < 3)
     faraday-net_http (1.0.1)
     faraday-net_http_persistent (1.2.0)
     faraday-patron (1.0.0)
     faraday-rack (1.0.0)
-    gapic-common (0.7.0)
+    faraday-retry (1.0.3)
+    gapic-common (0.8.0)
       faraday (~> 1.3)
       google-protobuf (~> 3.14)
       googleapis-common-protos (>= 1.3.11, < 2.a)
@@ -50,36 +54,40 @@ GEM
     google-cloud-env (1.5.0)
       faraday (>= 0.17.3, < 2.0)
     google-cloud-errors (1.2.0)
-    google-cloud-pubsub (2.9.0)
+    google-cloud-pubsub (2.9.1)
       concurrent-ruby (~> 1.1)
       google-cloud-core (~> 1.5)
       google-cloud-pubsub-v1 (~> 0.0)
-    google-cloud-pubsub-v1 (0.6.1)
+    google-cloud-pubsub-v1 (0.7.1)
       gapic-common (>= 0.7, < 2.a)
       google-cloud-errors (~> 1.0)
       grpc-google-iam-v1 (>= 0.6.10, < 2.a)
-    google-protobuf (3.19.1-x86_64-darwin)
+    google-protobuf (3.19.4-x86_64-darwin)
+    google-protobuf (3.19.4-x86_64-linux)
     googleapis-common-protos (1.3.12)
       google-protobuf (~> 3.14)
       googleapis-common-protos-types (~> 1.2)
       grpc (~> 1.27)
     googleapis-common-protos-types (1.3.0)
       google-protobuf (~> 3.14)
-    googleauth (1.1.0)
-      faraday (>= 0.17.3, < 2.0)
+    googleauth (1.1.2)
+      faraday (>= 0.17.3, < 3.a)
       jwt (>= 1.4, < 3.0)
       memoist (~> 0.16)
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
       signet (>= 0.16, < 2.a)
-    grpc (1.40.0-universal-darwin)
-      google-protobuf (~> 3.15)
+    grpc (1.43.1-universal-darwin)
+      google-protobuf (~> 3.18)
+      googleapis-common-protos-types (~> 1.0)
+    grpc (1.43.1-x86_64-linux)
+      google-protobuf (~> 3.18)
       googleapis-common-protos-types (~> 1.0)
     grpc-google-iam-v1 (1.0.0)
       google-protobuf (~> 3.14)
       googleapis-common-protos (>= 1.3.12, < 2.0)
       grpc (~> 1.27)
-    i18n (1.8.11)
+    i18n (1.10.0)
       concurrent-ruby (~> 1.0)
     jwt (2.3.0)
     memoist (0.16.2)
@@ -91,23 +99,23 @@ GEM
     nio4r (2.5.8)
     os (1.1.4)
     public_suffix (4.0.6)
-    puma (5.5.2)
+    puma (5.6.4)
       nio4r (~> 2.0)
-    rack (2.2.3)
-    rack-protection (2.1.0)
+    rack (2.2.4)
+    rack-protection (2.2.0)
       rack
     rake (13.0.6)
     retriable (3.1.2)
     ruby2_keywords (0.0.5)
-    signet (0.16.0)
+    signet (0.16.1)
       addressable (~> 2.8)
-      faraday (>= 0.17.3, < 2.0)
+      faraday (>= 0.17.5, < 3.0)
       jwt (>= 1.5, < 3.0)
       multi_json (~> 1.10)
-    sinatra (2.1.0)
+    sinatra (2.2.0)
       mustermann (~> 1.0)
       rack (~> 2.2)
-      rack-protection (= 2.1.0)
+      rack-protection (= 2.2.0)
       tilt (~> 2.0)
     tilt (2.0.10)
     tzinfo (2.0.4)
@@ -116,11 +124,11 @@ GEM
 PLATFORMS
   x86_64-darwin-17
   x86_64-darwin-19
+  x86_64-linux
 
 DEPENDENCIES
   cloudenvoy!
-  grpc (~> 1.40.0)
-  puma (~> 5.5.2)
+  puma (~> 5.6.4)
   rake
   sinatra
 

data/examples/sinatra/README.md

@@ -17,6 +17,10 @@ Cloudenvoy.setup_subscribers
 HelloPublisher.publish('Some message')
 ```
 8. Tail the logs to see how message get processed by `HelloSubscriber`
+9. Try sending multiple messages at once:
+```ruby
+HelloPublisher.publish_all(['message1', 'message2', 'message3'])
+```
 
 ## Run using GCP Pub/Sub
 

data/examples/sinatra/app/subscribers/hello_subscriber.rb

@@ -11,6 +11,6 @@ class HelloSubscriber
   # @param [Cloudenvoy::Message] message The message to process.
   #
   def process(message)
-    logger.info("Received message #{message.payload.dig('content')}")
+    logger.info("Received message #{message.payload['content']}")
   end
 end

data/examples/sinatra/app.rb

@@ -1,13 +1,13 @@
 # frozen_string_literal: true
 
 # Force logging to be realtime
-STDOUT.sync = true
+$stdout.sync = true
 
 # Require project files
 require 'sinatra'
-Dir.glob('./config/initializers/*.rb').each { |file| require file }
-Dir.glob('./app/publishers/*.rb').each { |file| require file }
-Dir.glob('./app/subscribers/*.rb').each { |file| require file }
+Dir.glob('./config/initializers/*.rb').sort.each { |file| require file }
+Dir.glob('./app/publishers/*.rb').sort.each { |file| require file }
+Dir.glob('./app/subscribers/*.rb').sort.each { |file| require file }
 
 #---------------------------------------------------
 # Routes

data/lib/cloudenvoy/backend/google_pub_sub.rb

@@ -54,7 +54,7 @@ module Cloudenvoy
       # @param [Hash, String] payload The message content.
       # @param [Hash] metadata The message attributes.
       #
-      # @return [Cloudenvoy::Message] The created message.
+      # @return [Cloudenvoy::Message] The published message.
       #
       def publish(topic, payload, metadata = {})
         # Retrieve the topic
@@ -72,6 +72,38 @@ module Cloudenvoy
         )
       end
 
+      #
+      # Publish multiple messages to a topic.
+      #
+      # @param [String] topic The name of the topic
+      # @param [Array<Array<[Hash, String]>>] msg_args A list of message [payload, metadata].
+      #
+      # @return [Array<Cloudenvoy::Message>] The published messages.
+      #
+      def publish_all(topic, msg_args)
+        # Retrieve the topic
+        ps_topic = backend.topic(topic, skip_lookup: true)
+
+        # Publish the message
+        ps_msgs = ps_topic.publish do |batch|
+          msg_args.each do |(payload, metadata)|
+            batch.publish(payload.to_json, metadata.to_h)
+          end
+        end
+
+        # Return the formatted messages
+        ps_msgs.each_with_index.map do |ps_msg, index|
+          payload, metadata = msg_args[index]
+
+          Message.new(
+            id: ps_msg.message_id,
+            payload: payload,
+            metadata: metadata,
+            topic: topic
+          )
+        end
+      end
+
       #
       # Create or update a subscription for a specific topic.
       #
@@ -134,8 +166,8 @@ module Cloudenvoy
       def upsert_topic(topic)
         ps_topic = begin
           backend.create_topic(topic)
-                   rescue Google::Cloud::AlreadyExistsError
-                     backend.topic(topic)
+        rescue Google::Cloud::AlreadyExistsError
+          backend.topic(topic)
         end
 
         # Return formatted subscription

data/lib/cloudenvoy/backend/memory_pub_sub.rb

@@ -47,7 +47,7 @@ module Cloudenvoy
       # @param [Hash, String] payload The message content.
       # @param [Hash] attrs The message attributes.
       #
-      # @return [Cloudenvoy::Message] The created message.
+      # @return [Cloudenvoy::Message] The published message.
       #
       def publish(topic, payload, metadata = {})
         msg = Message.new(
@@ -61,6 +61,30 @@ module Cloudenvoy
         msg
       end
 
+      #
+      # Publish multiple messages to a topic.
+      #
+      # @param [String] topic The name of the topic
+      # @param [Array<Array<[Hash, String]>>] msg_args A list of message [payload, metadata].
+      #
+      # @return [Array<Cloudenvoy::Message>] The published messages.
+      #
+      def publish_all(topic, msg_args)
+        # Build the messages
+        msgs = msg_args.map do |(payload, metadata)|
+          Message.new(
+            id: SecureRandom.uuid,
+            payload: payload,
+            metadata: metadata,
+            topic: topic
+          )
+        end
+
+        # Push all the messages and return them
+        queue(topic).push(*msgs)
+        msgs
+      end
+
       #
       # Create or update a subscription for a specific topic.
       #

data/lib/cloudenvoy/config.rb

@@ -57,7 +57,7 @@ module Cloudenvoy
     # @return [Logger, any] The cloudenvoy logger.
     #
     def logger
-      @logger ||= defined?(Rails) ? Rails.logger : ::Logger.new(STDOUT)
+      @logger ||= defined?(Rails) ? Rails.logger : ::Logger.new($stdout)
     end
 
     #

data/lib/cloudenvoy/message.rb

@@ -75,8 +75,8 @@ module Cloudenvoy
     def to_h
       {
         id: id,
-        payload: payload&.dup,
-        metadata: metadata&.dup,
+        payload: payload.dup,
+        metadata: metadata.dup,
         topic: topic,
         sub_uri: sub_uri
       }.compact

data/lib/cloudenvoy/pub_sub_client.rb

@@ -30,12 +30,24 @@ module Cloudenvoy
     # @param [Hash, String] payload The message content.
     # @param [Hash] attrs The message attributes.
     #
-    # @return [Cloudenvoy::Message] The created message.
+    # @return [Cloudenvoy::Message] The published message.
     #
     def self.publish(topic, payload, attrs = {})
       backend.publish(topic, payload, attrs)
     end
 
+    #
+    # Publish multiple messages to a topic.
+    #
+    # @param [String] topic The name of the topic
+    # @param [Array<Array<[Hash, String]>>] msg_args A list of message [payload, metadata].
+    #
+    # @return [Array<Cloudenvoy::Message>] The published messages.
+    #
+    def self.publish_all(topic, msg_args)
+      backend.publish_all(topic, msg_args)
+    end
+
     #
     # Create or update a subscription for a specific topic.
     #

data/lib/cloudenvoy/publisher.rb

@@ -89,6 +89,40 @@ module Cloudenvoy
 
         PubSubClient.upsert_topic(default_topic)
       end
+
+      #
+      # Publish all messages in one batch.
+      #
+      # @param [Array<Any>] The list of publisher arguments.
+      #
+      # @return [Array<Cloudenvoy::Message>] The published messages.
+      #
+      def publish_all(arg_list)
+        # Build the list of publishers
+        publishers = arg_list.map { |e| new(msg_args: [e].flatten(1)) }
+
+        # Batches are topic specific. We must send one batch of messages per topic.
+        publishers.group_by { |e| e.topic(*e.msg_args) }.map do |topic, topic_publishers|
+          # Chain publish calls. The very last call (when the chain becomes empty)
+          # is the actual batch publishing of messages to pub/sub
+          chain = topic_publishers.dup
+          traverse_chain = lambda do
+            if chain.empty?
+              # Send the messages in one batch and retrospectively attach them
+              # to each publisher
+              arg_list = topic_publishers.map { |e| [e.payload(*e.msg_args), e.metadata(*e.msg_args)] }
+              PubSubClient.publish_all(topic, arg_list).each_with_index do |msg, pub_index|
+                topic_publishers[pub_index].message = msg
+              end
+            else
+              # Defer message publishing to the next element
+              # in the chain until the chain is empty
+              chain.shift.publish(&traverse_chain)
+            end
+          end
+          traverse_chain.call
+        end.flatten
+      end
     end
 
     #
@@ -155,9 +189,9 @@ module Cloudenvoy
     #
     # @return [Cloudenvoy::Message] The created message.
     #
-    def publish
+    def publish(&block)
       # Format and publish message
-      resp = execute_middleware_chain
+      resp = execute_middleware_chain(&block)
 
       # Log job completion and return result
       logger.info("Published message in #{publishing_duration}s") { { duration: publishing_duration } }
@@ -191,14 +225,14 @@ module Cloudenvoy
     #
     # Execute the subscriber process method through the middleware chain.
     #
-    # @return [Any] The result of the perform method.
+    # @return [Cloudenvoy::Message] The published message.
     #
-    def execute_middleware_chain
+    def execute_middleware_chain(&block)
       self.publishing_started_at = Time.now
 
       Cloudenvoy.config.publisher_middleware.invoke(self) do
         begin
-          publish_message
+          block_given? ? block.call : publish_message
         rescue StandardError => e
           logger.error([e, e.backtrace.join("\n")].join("\n"))
           try(:on_error, e)

data/lib/cloudenvoy/publisher_logger.rb

@@ -9,7 +9,7 @@ module Cloudenvoy
     # @return [Proc] The context processor proc.
     #
     def self.default_context_processor
-      @default_context_processor ||= ->(loggable) { loggable.message&.to_h&.slice(:id, :metadata, :topic) || {} }
+      @default_context_processor ||= ->(loggable) { loggable.message.to_h&.slice(:id, :metadata, :topic) || {} }
     end
 
     #

data/lib/cloudenvoy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cloudenvoy
-  VERSION = '0.5.rc1'
+  VERSION = '0.6.0'
 end

data/lib/cloudenvoy.rb

@@ -63,6 +63,18 @@ module Cloudenvoy
     PubSubClient.publish(topic, payload, attrs)
   end
 
+  #
+  # Publish multiple messages to a topic. Shorthand method to Cloudenvoy::PubSubClient#publish_all.
+  #
+  # @param [String] topic The name of the topic
+  # @param [Array<Array<[Hash, String]>>] msg_args A list of message [payload, metadata].
+  #
+  # @return [Array<Cloudenvoy::Message>] The published messages.
+  #
+  def self.publish_all(topic, msg_args)
+    PubSubClient.publish_all(topic, msg_args)
+  end
+
   #
   # Return the list of registered publishers.
   #

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cloudenvoy
 version: !ruby/object:Gem::Version
-  version: 0.5.rc1
+  version: 0.6.0
 platform: ruby
 authors:
 - Arnaud Lachaume
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-01-05 00:00:00.000000000 Z
+date: 2022-07-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -114,14 +114,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.76.0
+        version: 0.93.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.76.0
+        version: 0.93.0
 - !ruby/object:Gem::Dependency
   name: rubocop-rspec
   requirement: !ruby/object:Gem::Requirement
@@ -281,19 +281,13 @@ files:
 - examples/sinatra/config/initializers/cloudenvoy.rb
 - gemfiles/.bundle/config
 - gemfiles/rails_5.2.gemfile
-- gemfiles/rails_5.2.gemfile.lock
 - gemfiles/rails_6.0.gemfile
-- gemfiles/rails_6.0.gemfile.lock
 - gemfiles/rails_6.1.gemfile
 - gemfiles/rails_7.0.gemfile
 - gemfiles/semantic_logger_3.4.gemfile
-- gemfiles/semantic_logger_3.4.gemfile.lock
 - gemfiles/semantic_logger_4.6.gemfile
-- gemfiles/semantic_logger_4.6.gemfile.lock
 - gemfiles/semantic_logger_4.7.0.gemfile
-- gemfiles/semantic_logger_4.7.0.gemfile.lock
 - gemfiles/semantic_logger_4.7.2.gemfile
-- gemfiles/semantic_logger_4.7.2.gemfile.lock
 - lib/cloudenvoy.rb
 - lib/cloudenvoy/authentication_error.rb
 - lib/cloudenvoy/authenticator.rb
@@ -336,14 +330,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.3.0
+      version: 2.4.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubygems_version: 3.2.3
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Cross-application messaging using GCP Pub/Sub (alpha)