anway 0.5.1 → 0.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 414e59f65eafee592adb08dbe1e5a28c8655e8c13b10d5fbc88597d1df552eb1
-  data.tar.gz: 59ce7fa161329b4abbafff3a18fc57d55f829b77a806ef39ec19843e3ceb40b3
+  metadata.gz: 879b6b41434558306382e3c24155f9fd38bf307af3782a7540fadb7047e6874f
+  data.tar.gz: c1daefd209281acc66457927bd76cd5914c1ddabb881e58de993fcb6cdc9652a
 SHA512:
-  metadata.gz: 46aa7b2a3c725a6c0d6bdbf3929de6f0c0f038c30bf156dabb6eab1bf462173f0e42b77138f929111fcbeba05129116e04a152f14aae9588c192d870086df4b9
-  data.tar.gz: b5a597b6a917520cacf61bf2fc12b3f3c267b6032e796b23c5c406f1dc19822c481e87cf0a8e2d8cca77463f3dc8be433a826c0b37a8a8a4dc5d087dae205459
+  metadata.gz: e255b3d87fc235bad83961e227598b577588aa8940fb3acce0e499bb8e262a5fb6d070c392013c6e3ca314a747dd0f44e3bf21aa62284bbcadd7c35181a71834
+  data.tar.gz: fea336b393f25fee2227dcd9cb6a326df8c847b531e979740978f0001d5d32b0fce2448a9b36455c10775a532b1eb44a4037ea6ce018b0d24e8eec7d6f78cbb6

data/README.md

@@ -211,6 +211,20 @@ client.subscribe('chat.message', queue: 'realtime', event: 'BroadcastMessage', r
 client.unsubscribe('chat.message', queue: 'realtime', event: 'BroadcastMessage')
 ```
 
+### Bulk publish
+
+`Anway.publish_bulk` sends many events with one `PUBLISHB` for each 1,000 events. The server saves all copies of the events of one `PUBLISHB` in one write, so an import, an outbox relay, or a replay is much faster than one `publish` for each event. Use `publish` for one event, such as in a request.
+
+```ruby
+results = Anway.publish_bulk(orders.map { |order| { topic: 'order.paid', args: [order.id], id: "order-#{order.id}" } })
+
+results.reject { |result| result.error.nil? }.each { |result| Rails.logger.warn("event #{result.event}: #{result.error}") }
+```
+
+- Each event is a hash with `topic:` and `args:`, and the options of `publish`: `id:`, `unique_for:`, and `custom:`.
+- The results are `Anway::Client::PublishResult` values (`event`, `jobs`, `duplicate`, `error`), in the order of the events. A refused event, such as a bad topic, has `error` and does not stop the others.
+- `client.publish_bulk(events)` sends one `PUBLISHB`. It takes at most 1,000 events, and more raise `ArgumentError` before the send. A bulk can make at most 10,000 jobs over all subscriptions.
+
 [docs/topics.md](docs/topics.md) explains the chat example with three apps, the options, the tests, and problems.
 
 ## Rails
@@ -327,7 +341,7 @@ Anway::Testing.events.last['topic']
 Anway::Testing.inline!
 ```
 
-In inline mode, `Anway.publish` runs one copy in each job class of the app that subscribes to the topic.
+In inline mode, `Anway.publish` runs one copy in each job class of the app that subscribes to the topic. `Anway.publish_bulk` works as one `Anway.publish` for each event.
 
 `perform_bulk` and `Anway.push_bulk` work as single pushes do: fake mode adds each job to `Anway::Testing.jobs`, and inline mode runs each job. The result is `{}`.
 

data/lib/anway/client.rb

@@ -10,7 +10,8 @@ module Anway
     DEFAULT_PORT = 7419
     FETCH_WAIT = 2
 
-    # The most jobs that one PUSHB takes.
+    # The most jobs that one PUSHB takes, and the most events that one
+    # PUBLISHB takes.
     MAX_BULK = 1000
 
     Endpoint = Data.define(:host, :port, :password, :tls)
@@ -23,6 +24,10 @@ module Anway
     # server already had an event with this id.
     Published = Data.define(:event, :jobs, :duplicate)
 
+    # One result of a PUBLISHB: as Published, and the reason in error when the
+    # server refused the event.
+    PublishResult = Data.define(:event, :jobs, :duplicate, :error)
+
     # A blank URL uses ANWAY_URL, then the local default.
     def initialize(url = nil, pool_size: 8, timeout: 10, tls: nil, wid: nil, labels: [], identity: {})
       @endpoint = parse_url(url || ENV.fetch('ANWAY_URL', nil) || DEFAULT_URL)
@@ -135,14 +140,33 @@ module Anway
     # publish of the same id within unique_for seconds makes no job, and the
     # reply has duplicate true. Returns a Published.
     def publish(topic, args = [], id: nil, unique_for: nil, custom: nil)
-      Job.validate_args!(args)
-
-      event = { topic: topic, args: args, id: id, unique_for: unique_for, custom: custom&.transform_keys(&:to_s) }
-      data = JSON.parse(call('PUBLISH', event.compact))
+      data = JSON.parse(call('PUBLISH', event_payload(topic, args, id: id, unique_for: unique_for, custom: custom)))
 
       Published.new(event: data['event'], jobs: data['jobs'], duplicate: data['duplicate'] == true)
     end
 
+    # Sends up to MAX_BULK events with one PUBLISHB, and the server saves them
+    # in one write. Each event is a hash with topic: and args:, and the
+    # options of publish (id:, unique_for:, custom:). Returns one
+    # PublishResult for each event, in the same order. A refused event, such
+    # as a bad topic, has error, and does not stop the others. More than
+    # MAX_BULK events raise ArgumentError before the send.
+    def publish_bulk(events)
+      events = events.to_a
+
+      raise ArgumentError, "a bulk publish has at most #{MAX_BULK} events, not #{events.size}" if events.size > MAX_BULK
+
+      return [] if events.empty?
+
+      payload = events.map do |event|
+        event_payload(event.fetch(:topic), event.fetch(:args, []), **event.slice(:id, :unique_for, :custom))
+      end
+
+      JSON.parse(call('PUBLISHB', payload)).map do |data|
+        PublishResult.new(event: data['event'], jobs: data['jobs'], duplicate: data['duplicate'] == true, error: data['error'])
+      end
+    end
+
     # Each event of the topic makes one job in queue, with event as its job
     # type. A second call with the same topic, queue, and event sets new retry
     # and reserve_for.
@@ -199,6 +223,13 @@ module Anway
 
     private
 
+    # The argument of PUBLISH, and of each event of PUBLISHB.
+    def event_payload(topic, args, id: nil, unique_for: nil, custom: nil)
+      Job.validate_args!(args)
+
+      { topic: topic, args: args, id: id, unique_for: unique_for, custom: custom&.transform_keys(&:to_s) }.compact
+    end
+
     def queue_command(action, queues)
       call('QUEUE', "#{action} #{queues.join(' ')}")
 

data/lib/anway/testing.rb

@@ -67,6 +67,15 @@ module Anway
         Client::Published.new(event: event_id, jobs: subscriptions.size, duplicate: false)
       end
 
+      # Publishes each event as a single publish does. No event is refused.
+      def publish_bulk(events)
+        events.map do |event|
+          published = publish(event.fetch(:topic), event.fetch(:args, []), **event.slice(:id, :unique_for, :custom))
+
+          Client::PublishResult.new(event: published.event, jobs: published.jobs, duplicate: false, error: nil)
+        end
+      end
+
       # Runs and removes all fake jobs. Jobs that they push also run.
       def drain
         Executor.call(@jobs.shift) until @jobs.empty?

data/lib/anway/version.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 module Anway
-  VERSION = '0.5.1'
+  VERSION = '0.5.2'
 
   # The anway server that the platform gems carry, and that CI tests against.
-  SERVER_VERSION = '0.5.1'
+  SERVER_VERSION = '0.5.2'
 end

data/lib/anway.rb

@@ -93,6 +93,26 @@ module Anway
       published
     end
 
+    # Sends the events with one PUBLISHB for each Client::MAX_BULK events.
+    # Each event is a hash with topic: and args:, and the options of publish.
+    # Returns one Client::PublishResult for each event, in the same order.
+    # Inside a database transaction, the publish waits for the commit and
+    # returns nil.
+    def publish_bulk(events)
+      events = events.to_a
+      testing = const_defined?(:Testing) ? Testing : nil
+
+      return testing.publish_bulk(events) if testing&.mode
+
+      results = nil
+
+      config.push_wrapper.call do
+        results = events.each_slice(Client::MAX_BULK).flat_map { |slice| client.publish_bulk(slice) }
+      end
+
+      results
+    end
+
     def reset!
       @lock.synchronize do
         @client&.close

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: anway
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.5.2
 platform: ruby
 authors:
 - Thadeu