cloudenvoy 0.4.2 → 0.6.rc1

data/lib/cloudenvoy/backend/google_pub_sub.rb

@@ -32,7 +32,7 @@ module Cloudenvoy
       # @return [Google::Cloud::Pub] The low level client instance.
       #
       def backend
-        @backend ||= Google::Cloud::PubSub.new({
+        @backend ||= Google::Cloud::PubSub.new(**{
           project_id: config.gcp_project_id,
           emulator_host: development? ? Cloudenvoy::Config::EMULATOR_HOST : nil
         }.compact)
@@ -54,7 +54,7 @@ module Cloudenvoy
       # @param [Hash, String] payload The message content.
       # @param [Hash] metadata The message attributes.
       #
-      # @return [Cloudenvoy::Message] The created message.
+      # @return [Cloudenvoy::Message] The published message.
       #
       def publish(topic, payload, metadata = {})
         # Retrieve the topic
@@ -72,6 +72,38 @@ module Cloudenvoy
         )
       end
 
+      #
+      # Publish multiple messages to a topic.
+      #
+      # @param [String] topic The name of the topic
+      # @param [Array<Array<[Hash, String]>>] msg_args A list of message [payload, metadata].
+      #
+      # @return [Array<Cloudenvoy::Message>] The published messages.
+      #
+      def publish_all(topic, msg_args)
+        # Retrieve the topic
+        ps_topic = backend.topic(topic, skip_lookup: true)
+
+        # Publish the message
+        ps_msgs = ps_topic.publish do |batch|
+          msg_args.each do |(payload, metadata)|
+            batch.publish(payload.to_json, metadata.to_h)
+          end
+        end
+
+        # Return the formatted messages
+        ps_msgs.each_with_index.map do |ps_msg, index|
+          payload, metadata = msg_args[index]
+
+          Message.new(
+            id: ps_msg.message_id,
+            payload: payload,
+            metadata: metadata,
+            topic: topic
+          )
+        end
+      end
+
       #
       # Create or update a subscription for a specific topic.
       #
@@ -109,7 +141,7 @@ module Cloudenvoy
             ps_topic = backend.topic(topic, skip_lookup: true)
 
             # Attempt to create the subscription
-            ps_topic.subscribe(name, sub_config)
+            ps_topic.subscribe(name, **sub_config)
           rescue Google::Cloud::AlreadyExistsError
             # Update endpoint on subscription
             # Topic is not updated as it is name-dependent
@@ -134,8 +166,8 @@ module Cloudenvoy
       def upsert_topic(topic)
         ps_topic = begin
           backend.create_topic(topic)
-                   rescue Google::Cloud::AlreadyExistsError
-                     backend.topic(topic)
+        rescue Google::Cloud::AlreadyExistsError
+          backend.topic(topic)
         end
 
         # Return formatted subscription

data/lib/cloudenvoy/backend/memory_pub_sub.rb

@@ -47,7 +47,7 @@ module Cloudenvoy
       # @param [Hash, String] payload The message content.
       # @param [Hash] attrs The message attributes.
       #
-      # @return [Cloudenvoy::Message] The created message.
+      # @return [Cloudenvoy::Message] The published message.
       #
       def publish(topic, payload, metadata = {})
         msg = Message.new(
@@ -61,6 +61,30 @@ module Cloudenvoy
         msg
       end
 
+      #
+      # Publish multiple messages to a topic.
+      #
+      # @param [String] topic The name of the topic
+      # @param [Array<Array<[Hash, String]>>] msg_args A list of message [payload, metadata].
+      #
+      # @return [Array<Cloudenvoy::Message>] The published messages.
+      #
+      def publish_all(topic, msg_args)
+        # Build the messages
+        msgs = msg_args.map do |(payload, metadata)|
+          Message.new(
+            id: SecureRandom.uuid,
+            payload: payload,
+            metadata: metadata,
+            topic: topic
+          )
+        end
+
+        # Push all the messages and return them
+        queue(topic).push(*msgs)
+        msgs
+      end
+
       #
       # Create or update a subscription for a specific topic.
       #

data/lib/cloudenvoy/config.rb

@@ -57,7 +57,7 @@ module Cloudenvoy
     # @return [Logger, any] The cloudenvoy logger.
     #
     def logger
-      @logger ||= defined?(Rails) ? Rails.logger : ::Logger.new(STDOUT)
+      @logger ||= defined?(Rails) ? Rails.logger : ::Logger.new($stdout)
     end
 
     #

data/lib/cloudenvoy/message.rb

@@ -75,8 +75,8 @@ module Cloudenvoy
     def to_h
       {
         id: id,
-        payload: payload&.dup,
-        metadata: metadata&.dup,
+        payload: payload.dup,
+        metadata: metadata.dup,
         topic: topic,
         sub_uri: sub_uri
       }.compact

data/lib/cloudenvoy/pub_sub_client.rb

@@ -30,12 +30,24 @@ module Cloudenvoy
     # @param [Hash, String] payload The message content.
     # @param [Hash] attrs The message attributes.
     #
-    # @return [Cloudenvoy::Message] The created message.
+    # @return [Cloudenvoy::Message] The published message.
     #
     def self.publish(topic, payload, attrs = {})
       backend.publish(topic, payload, attrs)
     end
 
+    #
+    # Publish multiple messages to a topic.
+    #
+    # @param [String] topic The name of the topic
+    # @param [Array<Array<[Hash, String]>>] msg_args A list of message [payload, metadata].
+    #
+    # @return [Array<Cloudenvoy::Message>] The published messages.
+    #
+    def self.publish_all(topic, msg_args)
+      backend.publish_all(topic, msg_args)
+    end
+
     #
     # Create or update a subscription for a specific topic.
     #

data/lib/cloudenvoy/publisher.rb

@@ -89,6 +89,40 @@ module Cloudenvoy
 
         PubSubClient.upsert_topic(default_topic)
       end
+
+      #
+      # Publish all messages in one batch.
+      #
+      # @param [Array<Any>] The list of publisher arguments.
+      #
+      # @return [Array<Cloudenvoy::Message>] The published messages.
+      #
+      def publish_all(arg_list)
+        # Build the list of publishers
+        publishers = arg_list.map { |e| new(msg_args: [e].flatten(1)) }
+
+        # Batches are topic specific. We must send one batch of messages per topic.
+        publishers.group_by { |e| e.topic(*e.msg_args) }.map do |topic, topic_publishers|
+          # Chain publish calls. The very last call (when the chain becomes empty)
+          # is the actual batch publishing of messages to pub/sub
+          chain = topic_publishers.dup
+          traverse_chain = lambda do
+            if chain.empty?
+              # Send the messages in one batch and retrospectively attach them
+              # to each publisher
+              arg_list = topic_publishers.map { |e| [e.payload(*e.msg_args), e.metadata(*e.msg_args)] }
+              PubSubClient.publish_all(topic, arg_list).each_with_index do |msg, pub_index|
+                topic_publishers[pub_index].message = msg
+              end
+            else
+              # Defer message publishing to the next element
+              # in the chain until the chain is empty
+              chain.shift.publish(&traverse_chain)
+            end
+          end
+          traverse_chain.call
+        end.flatten
+      end
     end
 
     #
@@ -155,9 +189,9 @@ module Cloudenvoy
     #
     # @return [Cloudenvoy::Message] The created message.
     #
-    def publish
+    def publish(&block)
       # Format and publish message
-      resp = execute_middleware_chain
+      resp = execute_middleware_chain(&block)
 
       # Log job completion and return result
       logger.info("Published message in #{publishing_duration}s") { { duration: publishing_duration } }
@@ -191,14 +225,14 @@ module Cloudenvoy
     #
     # Execute the subscriber process method through the middleware chain.
     #
-    # @return [Any] The result of the perform method.
+    # @return [Cloudenvoy::Message] The published message.
     #
-    def execute_middleware_chain
+    def execute_middleware_chain(&block)
       self.publishing_started_at = Time.now
 
       Cloudenvoy.config.publisher_middleware.invoke(self) do
         begin
-          publish_message
+          block_given? ? block.call : publish_message
         rescue StandardError => e
           logger.error([e, e.backtrace.join("\n")].join("\n"))
           try(:on_error, e)

data/lib/cloudenvoy/publisher_logger.rb

@@ -9,7 +9,7 @@ module Cloudenvoy
     # @return [Proc] The context processor proc.
     #
     def self.default_context_processor
-      @default_context_processor ||= ->(loggable) { loggable.message&.to_h&.slice(:id, :metadata, :topic) || {} }
+      @default_context_processor ||= ->(loggable) { loggable.message.to_h&.slice(:id, :metadata, :topic) || {} }
     end
 
     #

data/lib/cloudenvoy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cloudenvoy
-  VERSION = '0.4.2'
+  VERSION = '0.6.rc1'
 end

data/lib/cloudenvoy.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'active_support/core_ext/string/inflections'
+require 'active_support/core_ext/object/try'
 
 require 'cloudenvoy/version'
 require 'cloudenvoy/config'
@@ -62,6 +63,18 @@ module Cloudenvoy
     PubSubClient.publish(topic, payload, attrs)
   end
 
+  #
+  # Publish multiple messages to a topic. Shorthand method to Cloudenvoy::PubSubClient#publish_all.
+  #
+  # @param [String] topic The name of the topic
+  # @param [Array<Array<[Hash, String]>>] msg_args A list of message [payload, metadata].
+  #
+  # @return [Array<Cloudenvoy::Message>] The published messages.
+  #
+  def self.publish_all(topic, msg_args)
+    PubSubClient.publish_all(topic, msg_args)
+  end
+
   #
   # Return the list of registered publishers.
   #

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cloudenvoy
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.6.rc1
 platform: ruby
 authors:
 - Arnaud Lachaume
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-10-25 00:00:00.000000000 Z
+date: 2022-04-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -114,14 +114,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.76.0
+        version: 0.93.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.76.0
+        version: 0.93.0
 - !ruby/object:Gem::Dependency
   name: rubocop-rspec
   requirement: !ruby/object:Gem::Requirement
@@ -178,48 +178,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: rails
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rspec-rails
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: sqlite3
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: Cross-application messaging using GCP Pub/Sub (alpha)
 email:
 - arnaud.lachaume@keypup.io
@@ -227,7 +185,10 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".github/workflows/test.yml"
+- ".github/workflows/lint_rubocop.yml"
+- ".github/workflows/test_ruby_2.5_2.6.yml"
+- ".github/workflows/test_ruby_2.7.yml"
+- ".github/workflows/test_ruby_3.x.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
@@ -318,18 +279,15 @@ files:
 - examples/sinatra/app/subscribers/hello_subscriber.rb
 - examples/sinatra/bin/console
 - examples/sinatra/config/initializers/cloudenvoy.rb
+- gemfiles/.bundle/config
 - gemfiles/rails_5.2.gemfile
-- gemfiles/rails_5.2.gemfile.lock
 - gemfiles/rails_6.0.gemfile
-- gemfiles/rails_6.0.gemfile.lock
+- gemfiles/rails_6.1.gemfile
+- gemfiles/rails_7.0.gemfile
 - gemfiles/semantic_logger_3.4.gemfile
-- gemfiles/semantic_logger_3.4.gemfile.lock
 - gemfiles/semantic_logger_4.6.gemfile
-- gemfiles/semantic_logger_4.6.gemfile.lock
 - gemfiles/semantic_logger_4.7.0.gemfile
-- gemfiles/semantic_logger_4.7.0.gemfile.lock
 - gemfiles/semantic_logger_4.7.2.gemfile
-- gemfiles/semantic_logger_4.7.2.gemfile.lock
 - lib/cloudenvoy.rb
 - lib/cloudenvoy/authentication_error.rb
 - lib/cloudenvoy/authenticator.rb
@@ -372,14 +330,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.3.0
+      version: 2.4.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubygems_version: 3.0.0
+rubygems_version: 3.2.3
 signing_key:
 specification_version: 4
 summary: Cross-application messaging using GCP Pub/Sub (alpha)

data/gemfiles/rails_5.2.gemfile.lock

@@ -1,264 +0,0 @@
-PATH
-  remote: ..
-  specs:
-    cloudenvoy (0.4.1)
-      activesupport
-      google-cloud-pubsub (~> 2.0)
-      jwt
-      retriable
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    actioncable (5.2.0)
-      actionpack (= 5.2.0)
-      nio4r (~> 2.0)
-      websocket-driver (>= 0.6.1)
-    actionmailer (5.2.0)
-      actionpack (= 5.2.0)
-      actionview (= 5.2.0)
-      activejob (= 5.2.0)
-      mail (~> 2.5, >= 2.5.4)
-      rails-dom-testing (~> 2.0)
-    actionpack (5.2.0)
-      actionview (= 5.2.0)
-      activesupport (= 5.2.0)
-      rack (~> 2.0)
-      rack-test (>= 0.6.3)
-      rails-dom-testing (~> 2.0)
-      rails-html-sanitizer (~> 1.0, >= 1.0.2)
-    actionview (5.2.0)
-      activesupport (= 5.2.0)
-      builder (~> 3.1)
-      erubi (~> 1.4)
-      rails-dom-testing (~> 2.0)
-      rails-html-sanitizer (~> 1.0, >= 1.0.3)
-    activejob (5.2.0)
-      activesupport (= 5.2.0)
-      globalid (>= 0.3.6)
-    activemodel (5.2.0)
-      activesupport (= 5.2.0)
-    activerecord (5.2.0)
-      activemodel (= 5.2.0)
-      activesupport (= 5.2.0)
-      arel (>= 9.0)
-    activestorage (5.2.0)
-      actionpack (= 5.2.0)
-      activerecord (= 5.2.0)
-      marcel (~> 0.3.1)
-    activesupport (5.2.0)
-      concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (>= 0.7, < 2)
-      minitest (~> 5.1)
-      tzinfo (~> 1.1)
-    addressable (2.7.0)
-      public_suffix (>= 2.0.2, < 5.0)
-    appraisal (2.4.0)
-      bundler
-      rake
-      thor (>= 0.14.0)
-    arel (9.0.0)
-    ast (2.4.2)
-    builder (3.2.4)
-    concurrent-ruby (1.1.8)
-    crack (0.4.5)
-      rexml
-    crass (1.0.6)
-    diff-lcs (1.4.4)
-    erubi (1.10.0)
-    faraday (1.4.1)
-      faraday-excon (~> 1.1)
-      faraday-net_http (~> 1.0)
-      faraday-net_http_persistent (~> 1.1)
-      multipart-post (>= 1.2, < 3)
-      ruby2_keywords (>= 0.0.4)
-    faraday-excon (1.1.0)
-    faraday-net_http (1.0.1)
-    faraday-net_http_persistent (1.1.0)
-    gapic-common (0.4.1)
-      faraday (~> 1.3)
-      google-protobuf (~> 3.15, >= 3.15.2)
-      googleapis-common-protos (>= 1.3.11, < 2.0)
-      googleapis-common-protos-types (>= 1.0.6, < 2.0)
-      googleauth (~> 0.15, >= 0.15.1)
-      grpc (~> 1.36)
-    globalid (0.4.2)
-      activesupport (>= 4.2.0)
-    google-cloud-core (1.6.0)
-      google-cloud-env (~> 1.0)
-      google-cloud-errors (~> 1.0)
-    google-cloud-env (1.5.0)
-      faraday (>= 0.17.3, < 2.0)
-    google-cloud-errors (1.1.0)
-    google-cloud-pubsub (2.5.0)
-      concurrent-ruby (~> 1.1)
-      google-cloud-core (~> 1.5)
-      google-cloud-pubsub-v1 (~> 0.0)
-    google-cloud-pubsub-v1 (0.4.0)
-      gapic-common (~> 0.3)
-      google-cloud-errors (~> 1.0)
-      grpc-google-iam-v1 (>= 0.6.10, < 2.0)
-    google-protobuf (3.15.8)
-    googleapis-common-protos (1.3.11)
-      google-protobuf (~> 3.14)
-      googleapis-common-protos-types (>= 1.0.6, < 2.0)
-      grpc (~> 1.27)
-    googleapis-common-protos-types (1.0.6)
-      google-protobuf (~> 3.14)
-    googleauth (0.16.1)
-      faraday (>= 0.17.3, < 2.0)
-      jwt (>= 1.4, < 3.0)
-      memoist (~> 0.16)
-      multi_json (~> 1.11)
-      os (>= 0.9, < 2.0)
-      signet (~> 0.14)
-    grpc (1.37.0)
-      google-protobuf (~> 3.15)
-      googleapis-common-protos-types (~> 1.0)
-    grpc-google-iam-v1 (0.6.11)
-      google-protobuf (~> 3.14)
-      googleapis-common-protos (>= 1.3.11, < 2.0)
-      grpc (~> 1.27)
-    hashdiff (1.0.1)
-    i18n (1.8.10)
-      concurrent-ruby (~> 1.0)
-    jaro_winkler (1.5.4)
-    jwt (2.2.2)
-    loofah (2.9.1)
-      crass (~> 1.0.2)
-      nokogiri (>= 1.5.9)
-    mail (2.7.1)
-      mini_mime (>= 0.1.1)
-    marcel (0.3.3)
-      mimemagic (~> 0.3.2)
-    memoist (0.16.2)
-    method_source (1.0.0)
-    mimemagic (0.3.10)
-      nokogiri (~> 1)
-      rake
-    mini_mime (1.1.0)
-    mini_portile2 (2.5.0)
-    minitest (5.14.4)
-    multi_json (1.15.0)
-    multipart-post (2.1.1)
-    nio4r (2.5.7)
-    nokogiri (1.11.3)
-      mini_portile2 (~> 2.5.0)
-      racc (~> 1.4)
-    os (1.1.1)
-    parallel (1.20.1)
-    parser (3.0.1.0)
-      ast (~> 2.4.1)
-    public_suffix (4.0.6)
-    racc (1.5.2)
-    rack (2.2.3)
-    rack-test (1.1.0)
-      rack (>= 1.0, < 3)
-    rails (5.2.0)
-      actioncable (= 5.2.0)
-      actionmailer (= 5.2.0)
-      actionpack (= 5.2.0)
-      actionview (= 5.2.0)
-      activejob (= 5.2.0)
-      activemodel (= 5.2.0)
-      activerecord (= 5.2.0)
-      activestorage (= 5.2.0)
-      activesupport (= 5.2.0)
-      bundler (>= 1.3.0)
-      railties (= 5.2.0)
-      sprockets-rails (>= 2.0.0)
-    rails-dom-testing (2.0.3)
-      activesupport (>= 4.2.0)
-      nokogiri (>= 1.6)
-    rails-html-sanitizer (1.3.0)
-      loofah (~> 2.3)
-    railties (5.2.0)
-      actionpack (= 5.2.0)
-      activesupport (= 5.2.0)
-      method_source
-      rake (>= 0.8.7)
-      thor (>= 0.18.1, < 2.0)
-    rainbow (3.0.0)
-    rake (13.0.3)
-    retriable (3.1.2)
-    rexml (3.2.5)
-    rspec (3.10.0)
-      rspec-core (~> 3.10.0)
-      rspec-expectations (~> 3.10.0)
-      rspec-mocks (~> 3.10.0)
-    rspec-core (3.10.1)
-      rspec-support (~> 3.10.0)
-    rspec-expectations (3.10.1)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-mocks (3.10.2)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-rails (5.0.1)
-      actionpack (>= 5.2)
-      activesupport (>= 5.2)
-      railties (>= 5.2)
-      rspec-core (~> 3.10)
-      rspec-expectations (~> 3.10)
-      rspec-mocks (~> 3.10)
-      rspec-support (~> 3.10)
-    rspec-support (3.10.2)
-    rubocop (0.76.0)
-      jaro_winkler (~> 1.5.1)
-      parallel (~> 1.10)
-      parser (>= 2.6)
-      rainbow (>= 2.2.2, < 4.0)
-      ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 1.7)
-    rubocop-rspec (1.37.0)
-      rubocop (>= 0.68.1)
-    ruby-progressbar (1.11.0)
-    ruby2_keywords (0.0.4)
-    semantic_logger (4.7.4)
-      concurrent-ruby (~> 1.0)
-    signet (0.15.0)
-      addressable (~> 2.3)
-      faraday (>= 0.17.3, < 2.0)
-      jwt (>= 1.5, < 3.0)
-      multi_json (~> 1.10)
-    sprockets (4.0.2)
-      concurrent-ruby (~> 1.0)
-      rack (> 1, < 3)
-    sprockets-rails (3.2.2)
-      actionpack (>= 4.0)
-      activesupport (>= 4.0)
-      sprockets (>= 3.0.0)
-    sqlite3 (1.4.2)
-    thor (1.1.0)
-    thread_safe (0.3.6)
-    timecop (0.9.4)
-    tzinfo (1.2.9)
-      thread_safe (~> 0.1)
-    unicode-display_width (1.6.1)
-    webmock (3.12.2)
-      addressable (>= 2.3.6)
-      crack (>= 0.3.2)
-      hashdiff (>= 0.4.0, < 2.0.0)
-    websocket-driver (0.7.3)
-      websocket-extensions (>= 0.1.0)
-    websocket-extensions (0.1.5)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  appraisal
-  cloudenvoy!
-  rails (= 5.2)
-  rake (>= 12.3.3)
-  rspec (~> 3.0)
-  rspec-rails
-  rubocop (= 0.76.0)
-  rubocop-rspec (= 1.37.0)
-  semantic_logger
-  sqlite3
-  timecop
-  webmock
-
-BUNDLED WITH
-   2.2.9