tobox 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ecf0e9576a8a59834eb6b84eed5c648a2fda3e50b4be49fba068f2bcfc785978
-  data.tar.gz: fb8dc2819591dfe63a177dc1f27b9a947712db00bf451a8f3633d305ee12172b
+  metadata.gz: 33a40c07b2d7f36933540d390287aeeb7d4e0aa39a7d19a439eaca6a2f8fc23f
+  data.tar.gz: 939699d801afdb03ed85c0048eee3b00c50a786a345b0731c6dba910fafb51ad
 SHA512:
-  metadata.gz: 4b0d7eae10175ce51332e32ad89613196f7b56d01f54670bd78a36af0d411081fda20af93f7c2eee0fd032da84f7c098ad662b6ce00e01b8ab603b6f1f06cc4f
-  data.tar.gz: 6db739a849af63cee915970ea1abc7df4a9d4aff91edb4c76cc4c78273e7231750771638ff0dc477bbb586fc24f0dfab1de793d875cd22bb395c5ff1d3772641
+  metadata.gz: 5f5567a9f8e43fbc0f22e10465a0026a4984932183ae7d70c6fa95cdcb092995cbedd44422fa8a65b6c2d321be3ae15a0b72e905edd43c96427fbaac7cc73e34
+  data.tar.gz: f7823d88490ab6edc02ed6a8fd846f7e6cf40ce042fb046e502e3024c26f24ced5ee143de80669ffad49d5077dc97e338c363eda1e0ea1d3196479447c6aa7f6

data/CHANGELOG.md

@@ -1,5 +1,32 @@
 ## [Unreleased]
 
+## [0.3.0] - 2022-12-12
+
+### Features
+
+#### Inbox
+
+Implementation of the "inbox pattern", which ensures that events are processed to completion only once.
+
+```ruby
+# create an inbox table and reference it
+create_table(:inbox) do
+  column :id, :varchar, null: true, primary_key: true
+  # ...
+create_table(:outbox) do
+  column :inbox_id, :varchar
+  foreign_key :inbox_id, :inbox
+  # ...
+
+# tobox.rb
+inbox_table :inbox
+inbox_column :inbox_id
+
+# event production
+DB[:outbox].insert(event_type: "order_created", inbox_id: "order_created_#{order.id}", ....
+DB[:outbox].insert(event_type: "billing_event_started", inbox_id: "billing_event_started_#{order.id}", ....
+```
+
 ## [0.2.0] - 2022-12-05
 
 ### Features

data/README.md

@@ -15,6 +15,7 @@ Simple, data-first events processing framework based on the [transactional outbo
 - [Event](#event)
 - [Features](#features)
   - [Ordered event processing](#ordered-event-processing)
+  - [Inbox](#inbox)
 - [Plugins](#plugins)
   - [Zeitwerk](#zeitwerk)
   - [Sentry](#sentry)
@@ -299,6 +300,14 @@ Overrides the default log level ("info" when in "production" environment, "debug
 
 Defines the column to be used for event grouping, when [ordered processing of events is a requirement](#ordered-event-processing).
 
+### inbox table
+
+Defines the name of the table to be used for inbox, when [inbox usage is a requirement](#inbox).
+
+### inbox column
+
+Defines the column in the outbox table which references the inbox table, when one is set.
+
 <a id="markdown-event" name="event"></a>
 ## Event
 
@@ -361,6 +370,59 @@ end
 # "order_created" will be processed first
 # "billing_event_created" will only start processing once "order_created" finishes
 ```
+<a id="inbox" name="inbox"></a>
+### Inbox
+
+`tobox` also supports the [inbox pattern](https://event-driven.io/en/outbox_inbox_patterns_and_delivery_guarantees_explained/), to ensure "exactly-once" processing of events. This is achieved by "tagging" events with a unique identifier, and registering them in the inbox before processing (and if they're there, ignoring it altogether).
+
+In order to do so, you'll have to:
+
+1. add an "inbox" table in the database
+
+```ruby
+create_table(:inbox) do
+  column :inbox_id, :varchar, null: true, primary_key: true # it can also be a uuid, you decide
+  column :created_at, "timestamp without time zone", null: false, default: Sequel::CURRENT_TIMESTAMP
+end
+```
+
+2. add the unique id reference in the outbox table:
+
+```ruby
+create_table(:outbox) do
+  primary_key :id
+  column :type, :varchar, null: false
+  column :inbox_id, :varchar, null: true
+  # ...
+  foreign_key :inbox_id, :inbox
+```
+
+3. reference them in the configuration
+
+```ruby
+# tobox.rb
+inbox_table :inbox
+inbox_column :inbox_id
+```
+
+4. insert related outbox events with an inbox id
+
+```ruby
+order = Order.new(
+  item_id: item.id,
+  price: 20_20,
+  currency: "EUR"
+)
+DB.transaction do
+  order.save
+  DB[:outbox].insert(event_type: "order_created", inbox_id: "ord_crt_#{order.id}", data_after: order.to_hash)
+  DB[:outbox].insert(event_type: "billing_event_started", inbox_id: "bil_evt_std_#{order.id}", data_after: order.to_hash)
+end
+
+# assuming this bit above runs two times in two separate workers, each will be processed by tobox only once.
+```
+
+**NOTE**: make sure you keep cleaning the inbox periodically from older messages, once there's no more danger of receiving them again.
 
 <a id="markdown-plugins" name="plugins"></a>
 ## Plugins

data/lib/tobox/configuration.rb

@@ -18,6 +18,8 @@ module Tobox
       database_uri: nil,
       table: :outbox,
       group_column: nil,
+      inbox_table: nil,
+      inbox_column: nil,
       max_attempts: 10,
       exponential_retry_factor: 4,
       wait_for_events_delay: 5,

data/lib/tobox/fetcher.rb

@@ -24,6 +24,9 @@ module Tobox
 
       max_attempts = configuration[:max_attempts]
 
+      @inbox_table = configuration[:inbox_table]
+      @inbox_column = configuration[:inbox_column]
+
       @ds = @db[@table]
 
       run_at_conds = [
@@ -80,14 +83,17 @@ module Tobox
             if blk
               num_events = events.size
 
-              events.each do |ev|
-                ev[:metadata] = try_json_parse(ev[:metadata])
-                handle_before_event(ev)
-                yield(to_message(ev))
+              events.map! do |ev|
+                try_insert_inbox(ev) do
+                  ev[:metadata] = try_json_parse(ev[:metadata])
+                  handle_before_event(ev)
+                  yield(to_message(ev))
+                  ev
+                end
               rescue StandardError => e
                 error = e
                 raise Sequel::Rollback
-              end
+              end.compact!
             else
               events.map!(&method(:to_message))
             end
@@ -148,6 +154,16 @@ module Tobox
       data
     end
 
+    def try_insert_inbox(event)
+      return yield unless @inbox_table && @inbox_column
+
+      ret = @db[@inbox_table].insert_conflict.insert(@inbox_column => event[@inbox_column])
+
+      return unless ret
+
+      yield
+    end
+
     def handle_before_event(event)
       @logger.debug do
         log_message("outbox event (type: \"#{event[:type]}\", attempts: #{event[:attempts]}) starting...")

data/lib/tobox/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tobox
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tobox
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - HoneyryderChuck
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-12-05 00:00:00.000000000 Z
+date: 2022-12-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel