sbmt-outbox 6.6.0 → 6.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a86388ca6b13cdaef9411bd8baf1ea045fcc1b941721cdb8d4fb15bda4db29be
-  data.tar.gz: 6fbc5a6e5d62df9aaf1cf2f0afdc3cf49e8bf5ccdc0a49d86a0b8feb8e51f200
+  metadata.gz: 799e491b65b103541a47db1bb723e75a5a42ce04a3bbc3e20589310bf45c0601
+  data.tar.gz: 354049f421f106505186307c73c233d8e11f1f52d8ffd617ce7b4d65ba4ca2c7
 SHA512:
-  metadata.gz: bca4330c17aa8d6d8f6563ebc52dae02d9029fb01580b9294ffd5d20822768f95f5c41a0c5bdd733d9f6481632f44a754ce67079cf843e9cf3c58ada9f9fbb39
-  data.tar.gz: 3c24bf5fa9af05b1ef48f38ff20519466c5f63a47a1acbb5bf788d7e197c8af17759581c52443b8650c8ff4f3680bbd48aee3a8b224623e6097176aa284c5a11
+  metadata.gz: e2d2ead2f3e65f1c5142c4117234ba1e2da76bd37d8e1126c360b1b444e1b89aca41e308fc966804dbcec4329373079cb19fd6fe93f8d05fdb571363d472b2f9
+  data.tar.gz: 8ff4f833e385c975465eef632e3d234da54e186244fa17f13e7bc89723f67106a601ebe1917dd9cbdba89144638d27d3c54bab07f3da27b3721c47fc2d3187d4

data/README.md

@@ -43,7 +43,7 @@ rails g outbox:install
 
 ### Outbox/inbox items creation
 
-An ActiveRecord model can be generated for the outbox/ inbox item like this:
+An ActiveRecord model can be generated for the outbox/inbox item like this:
 
 ```shell
 rails g outbox:item MaybeNamespaced::SomeOutboxItem --kind outbox
@@ -85,6 +85,39 @@ transaction do
 end
 ```
 
+To create multiple Outbox items in batch, you should call the Interactor with the Item Model Class and batch attributes, each item should have same list of keys. Each item should have `event_key` element, it will be the Partitioning Key.
+
+```ruby
+transaction do
+  some_record.save!
+  another_record.save!
+
+  result = Sbmt::Outbox::CreateOutboxBatch.call(
+    MyOutboxItem,
+    batch_attributes: [
+      {
+        event_key: some_record.id,
+        payload: some_record.generate_payload,
+        options: {
+          key: some_record.id, # optional, may be used when producing to a Kafka topic
+          headers: {'FOO_BAR' => 'baz'} # optional, you can add custom headers
+        }
+      },
+      {
+        event_key: another_record.id,
+        payload: another_record.generate_payload,
+        options: {
+          key: another_record.id, # optional, may be used when producing to a Kafka topic
+          headers: {'FOO_BAR' => 'baz'} # optional, you can add custom headers
+        }
+      }
+    ]
+  )
+
+  raise result.failure unless result.success?
+end
+```
+
 ## Monitoring
 
 We use [Yabeda](https://github.com/yabeda-rb/yabeda) to collect [all kind of metrics](./config/initializers/yabeda.rb).
@@ -99,7 +132,7 @@ Example of a Grafana dashboard that you can import [from a file](./examples/graf
 
 ### `Outboxfile`
 
-First of all you shoudl create an `Outboxfile` at the root of your application with the following code:
+First of all you should create an `Outboxfile` at the root of your application with the following code:
 
 ```ruby
 # frozen_string_literal: true
@@ -135,8 +168,8 @@ Rails.application.config.outbox.tap do |config|
     pc.retryable_items_batch_size = 100
 
     # poll tactic: default is optimal for most cases: rate limit + redis job-queue size threshold
-    # poll tactic: aggressive is for high-intencity data: without rate limits + redis job-queue size threshold
-    # poll tactic: low-priority is for low-intencity data: rate limits + redis job-queue size threshold + + redis job-queue lag threshold
+    # poll tactic: aggressive is for high-intensity data: without rate limits + redis job-queue size threshold
+    # poll tactic: low-priority is for low-intensity data: rate limits + redis job-queue size threshold + + redis job-queue lag threshold
     pc.tactic = "default"
     # number of batches that one thread will process per rate interval
     pc.rate_limit = 60
@@ -257,7 +290,7 @@ production:
   bucket_size: 256
 ```
 __CAUTION__:
-- ⚠️ If this option is enabled and an error occurs while processing a message in a bucket, 
+- ⚠️ If this option is enabled and an error occurs while processing a message in a bucket,
 subsequent messages in that bucket won't be processed until the current message is either skipped or successfully processed
 - ⚠️ Cannot use `retry_strategies` and the `strict_order` option at the same time
 
@@ -435,6 +468,9 @@ Rails.application.config.outbox.tap do |config|
   config.create_item_middlewares.push(
     'MyCreateItemMiddleware'
   )
+  config.create_batch_middlewares.push(
+    'MyCreateBatchMiddleware'
+  )
 end
 
 # my_create_item_middleware.rb
@@ -445,6 +481,15 @@ class MyCreateItemMiddleware
     # your code
   end
 end
+
+# my_create_batch_middleware.rb
+class MyCreateBatchMiddleware
+  def call(item_class, batch_attributes)
+    # your code
+    yield
+    # your code
+  end
+end
 ```
 
 #### Server middlewares

data/app/interactors/sbmt/outbox/create_outbox_batch.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "sbmt/outbox/metrics/utils"
+
+# Provides ability to insert records in batches.
+# It allows to have different headers, event_key and partition_by between set of attributes,
+# but for predictability of metrics results better to have same headers across items
+
+module Sbmt
+  module Outbox
+    class CreateOutboxBatch < Outbox::DryInteractor
+      param :item_class, reader: :private
+      option :batch_attributes, reader: :private
+
+      delegate :box_type, :box_name, :owner, to: :item_class
+      delegate :create_batch_middlewares, to: "Sbmt::Outbox"
+
+      def call
+        middlewares = Middleware::Builder.new(create_batch_middlewares)
+        middlewares.call(item_class, batch_attributes) do
+          attributes_to_insert = batch_attributes.map do |attributes|
+            event_key = attributes[:event_key]
+            partition_by = attributes.delete(:partition_by) || event_key
+
+            return Failure(:missing_partition_by) unless partition_by
+            return Failure(:missing_event_key) unless event_key
+
+            # to get default values for some attributes, including uuid
+            record = item_class.new(attributes)
+
+            res = item_class.config.partition_strategy
+              .new(partition_by, item_class.config.bucket_size)
+              .call
+            record.bucket = res.value! if res.success?
+
+            # those 2 lines needed for rails 6, as it does not set timestamps
+            record.created_at ||= Time.zone.now
+            record.updated_at ||= record.created_at
+
+            record.attributes.reject { |_, value| value.nil? }
+          end
+
+          inserted_items = item_class.insert_all(attributes_to_insert, returning: [:id, :bucket])
+          inserted_items.rows.each do |(record_id, bucket)|
+            partition = item_class.bucket_partitions.fetch(bucket)
+            track_last_stored_id(record_id, partition)
+            track_counter(partition)
+          end
+
+          Success(inserted_items.rows.map(&:first))
+        rescue => e
+          Failure(e.message)
+        end
+      end
+
+      private
+
+      def track_last_stored_id(item_id, partition)
+        Yabeda
+          .outbox
+          .last_stored_event_id
+          .set({type: box_type, name: Sbmt::Outbox::Metrics::Utils.metric_safe(box_name), owner: owner, partition: partition}, item_id)
+      end
+
+      def track_counter(partition)
+        Yabeda
+          .outbox
+          .created_counter
+          .increment({type: box_type, name: Sbmt::Outbox::Metrics::Utils.metric_safe(box_name), owner: owner, partition: partition}, by: 1)
+      end
+    end
+  end
+end

data/app/interactors/sbmt/outbox/process_item.rb

@@ -93,7 +93,7 @@ module Sbmt
           track_discarded(item)
           Failure(:discard_item)
         else
-          track_failed("retry stratagy returned unknown failure: #{result.failure}")
+          track_failed("retry strategy returned unknown failure: #{result.failure}")
           Failure(:retry_strategy_failure)
         end
       end

data/lib/sbmt/outbox/engine.rb

@@ -63,6 +63,7 @@ module Sbmt
         c.batch_process_middlewares = []
         c.item_process_middlewares = []
         c.create_item_middlewares = []
+        c.create_batch_middlewares = []
 
         if defined?(::Sentry)
           c.batch_process_middlewares.push("Sbmt::Outbox::Middleware::Sentry::TracingBatchProcessMiddleware")

data/lib/sbmt/outbox/instrumentation/open_telemetry_loader.rb

@@ -5,6 +5,7 @@ require "opentelemetry-common"
 require "opentelemetry-instrumentation-base"
 
 require_relative "../middleware/open_telemetry/tracing_create_item_middleware"
+require_relative "../middleware/open_telemetry/tracing_create_batch_middleware"
 require_relative "../middleware/open_telemetry/tracing_item_process_middleware"
 
 module Sbmt
@@ -15,6 +16,7 @@ module Sbmt
           require_dependencies
 
           ::Sbmt::Outbox.config.create_item_middlewares.push("Sbmt::Outbox::Middleware::OpenTelemetry::TracingCreateItemMiddleware")
+          ::Sbmt::Outbox.config.create_batch_middlewares.push("Sbmt::Outbox::Middleware::OpenTelemetry::TracingCreateBatchMiddleware")
           ::Sbmt::Outbox.config.item_process_middlewares.push("Sbmt::Outbox::Middleware::OpenTelemetry::TracingItemProcessMiddleware")
         end
 
@@ -26,6 +28,7 @@ module Sbmt
 
         def require_dependencies
           require_relative "../middleware/open_telemetry/tracing_create_item_middleware"
+          require_relative "../middleware/open_telemetry/tracing_create_batch_middleware"
           require_relative "../middleware/open_telemetry/tracing_item_process_middleware"
         end
       end

data/lib/sbmt/outbox/middleware/open_telemetry/tracing_create_batch_middleware.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    module Middleware
+      module OpenTelemetry
+        class TracingCreateBatchMiddleware
+          def call(item_class, batch_attributes)
+            return yield unless defined?(::OpenTelemetry)
+
+            span_attributes = {
+              "messaging.system" => "outbox",
+              "messaging.outbox.box_type" => item_class.box_type.to_s,
+              "messaging.outbox.box_name" => item_class.box_name,
+              "messaging.outbox.owner" => item_class.owner,
+              "messaging.destination" => item_class.name,
+              "messaging.destination_kind" => "database"
+            }
+
+            tracer.in_span(span_name(item_class), attributes: span_attributes.compact, kind: :producer) do
+              batch_attributes.each do |item_attributes|
+                options = item_attributes[:options] ||= {}
+                headers = options[:headers] || options["headers"] || {}
+                ::OpenTelemetry.propagation.inject(headers)
+              end
+
+              yield
+            end
+          end
+
+          private
+
+          def tracer
+            ::Sbmt::Outbox::Instrumentation::OpenTelemetryLoader.instance.tracer
+          end
+
+          def span_name(item_class)
+            "#{item_class.box_type}/#{item_class.box_name} create batch"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/version.rb

@@ -2,6 +2,6 @@
 
 module Sbmt
   module Outbox
-    VERSION = "6.6.0"
+    VERSION = "6.7.0"
   end
 end

data/lib/sbmt/outbox.rb

@@ -166,5 +166,9 @@ module Sbmt
     def create_item_middlewares
       @create_item_middlewares ||= config.create_item_middlewares.map(&:constantize)
     end
+
+    def create_batch_middlewares
+      @create_batch_middlewares ||= config.create_batch_middlewares.map(&:constantize)
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sbmt-outbox
 version: !ruby/object:Gem::Version
-  version: 6.6.0
+  version: 6.7.0
 platform: ruby
 authors:
 - Sbermarket Ruby-Platform Team
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-06-20 00:00:00.000000000 Z
+date: 2024-08-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: connection_pool
@@ -519,6 +519,7 @@ files:
 - app/controllers/sbmt/outbox/root_controller.rb
 - app/interactors/sbmt/outbox/base_create_item.rb
 - app/interactors/sbmt/outbox/create_inbox_item.rb
+- app/interactors/sbmt/outbox/create_outbox_batch.rb
 - app/interactors/sbmt/outbox/create_outbox_item.rb
 - app/interactors/sbmt/outbox/dry_interactor.rb
 - app/interactors/sbmt/outbox/partition_strategies/hash_partitioning.rb
@@ -581,6 +582,7 @@ files:
 - lib/sbmt/outbox/logger.rb
 - lib/sbmt/outbox/metrics/utils.rb
 - lib/sbmt/outbox/middleware/builder.rb
+- lib/sbmt/outbox/middleware/open_telemetry/tracing_create_batch_middleware.rb
 - lib/sbmt/outbox/middleware/open_telemetry/tracing_create_item_middleware.rb
 - lib/sbmt/outbox/middleware/open_telemetry/tracing_item_process_middleware.rb
 - lib/sbmt/outbox/middleware/runner.rb
@@ -636,7 +638,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.5.11
 signing_key:
 specification_version: 4
 summary: Outbox service