staged_event 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 626ddab89e43654e2f9f7444efbbb778e40a02882142e5bab971a253d0586278
+  data.tar.gz: 77bf67900459f69099d58913d8ae49140c934d49239f1ebf043e0924d127632e
+SHA512:
+  metadata.gz: 19f8edb5ba417a59bae2d5e62f68437d34a1563b3996fe2d848c6515292136970f071ef7ed75bbc6e9bdac7404a317b5330ba8d1cd3a4bfca40ca0160ea7ba1b
+  data.tar.gz: 7dc404d84752a7aeeda6897cbb47dda3168d86219e3fdc809e21b5dac230a829d2b266fb361f51ac21d0acf0b5ab3bd566a5121181b4853644b4b272a1f55f4e

data/README.md

@@ -0,0 +1,59 @@
+# StagedEvent
+
+StagedEvent is a Ruby implementation of the [transactional outbox](https://microservices.io/patterns/data/transactional-outbox.html) architectural pattern.
+
+This gem is relevant for any process that wants to publish events to a set of subscribers. It automatically saves outgoing events as records in a database table, from which a separate background process reads, publishes, and (once publishing is confirmed) deletes them. StagedEvent also provides a listener to subscribe to and receive these events from other processes.
+
+Staging events this way allows them to be persisted within the same database transaction as associated domain-specific records, guaranteeing at-least-once delivery and thus eventual consistency with the receiving system(s).
+
+For now, StagedEvent makes the following assumptions:
+- Events are defined as protobufs (and you have generated Ruby classes for them)
+- [Google Pub/Sub](https://cloud.google.com/pubsub/) is the underyling messaging infrastructure
+- You're using ActiveRecord (with a Postgres database >= version 9.5, due to the usage of the SQL `SKIP LOCKED` clause).
+
+### Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem "staged_event"
+```
+
+Then, in your application directory:
+
+```bash
+$ bundle install
+$ rails generate staged_event:install
+```
+
+This will create an initializer at `config/initializers/staged_event.rb`, which you should modify with settings specific to your application.
+
+The installer will also generate a migration to create the database table for staged events waiting to be published. Remember to run it using:
+
+```bash
+rails db:migrate
+```
+
+### Publishing Events
+
+In order for events to actually be published, StagedEvent provides a rake task that should be kept running alongside your application server(s):
+
+```bash
+rake staged_event:publisher
+```
+
+### Receiving Events
+
+In order to receive events from publishers, StagedEvent provides a rake task that should be kept running alongside your application server(s):
+
+```bash
+rake staged_event:subscriber
+```
+
+In order to process incoming events, you define a callback in the staged_event initializer file.
+
+### Regenerating Ruby from protobufs
+
+StagedEvent uses a one-off protobuf definition to serialize and deserialize events that are defined as protobufs. In case it becomes necessary to recreate the auto-generated ruby, the command for that (from the repository root) is:
+
+protoc --ruby_out=./ "./lib/staged_event/event_envelope.proto"

data/lib/generators/staged_event/install_generator.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require "rails/generators/active_record"
+
+module StagedEvent
+  class InstallGenerator < Rails::Generators::Base
+    include Rails::Generators::Migration
+
+    class << self
+      delegate :next_migration_number, to: ActiveRecord::Generators::Base
+    end
+
+    source_paths << File.join(File.dirname(__FILE__), "templates")
+
+    def create_migration_file
+      migration_template "create_staged_events.rb.erb", "db/migrate/create_staged_events.rb"
+    end
+
+    def create_initializer
+      template "initializer.rb.erb", "config/initializers/staged_event.rb"
+    end
+  end
+end

data/lib/generators/staged_event/templates/create_staged_events.rb.erb

@@ -0,0 +1,12 @@
+class CreateStagedEvents < ActiveRecord::Migration[6.0]
+  def change
+    enable_extension "uuid-ossp"
+    enable_extension "pgcrypto"
+
+    create_table :staged_events, id: :uuid do |t|
+      t.string :topic
+      t.binary :data, null: false
+      t.datetime :created_at, null: false
+    end
+  end
+end

data/lib/generators/staged_event/templates/initializer.rb.erb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+StagedEvent::Configuration.configure do |config|
+  config.google_pubsub do |google_pubsub|
+    # The contents of the Pub/Sub keyfile as a Hash
+    google_pubsub.credentials = {}
+
+    # This specifies the mapping between the "topic" values stored on events in
+    # the database, and the associated Pub/Sub topic id. This level of indirection
+    # lets you use arbitrary topic names in your application code instead of being
+    # tied to names stored in a third-party infrastructure.
+    # The first entry in the map is used as the default for when the event does
+    # not specify a topic.
+    google_pubsub.topic_map = {
+      default: "topic_id_1",
+    }
+
+    google_pubsub.subscription_ids = [ "subscription_id_1" ]
+  end
+
+  config.publisher do |publisher|
+    # The number of events that a publisher will transactionally publish and remove
+    # from the database in each iteration.
+    publisher.batch_size = 5
+  end
+
+  config.subscriber do |subscriber|
+    # This callback will be run for every message received, and is passed the serialized
+    # event data.
+    subscriber.event_received_callback = lambda do |serialized_data|
+      #
+      # To respond to the the event, one could either
+      # 1) enqueue a background job with serialized_data as a parameter
+      # 2) handle the event in-process. For example:
+      #
+      #   event = StagedEvent.deserialize_event(serialized_data)
+      #
+      #   event.id is globally-unique and can be used for idempotency.
+      #   event.data is the event deserialized into its original form (typically a
+      #   protobuf object).
+      #
+      #   if event.data.is_a?(MyEvents::SomethingHappened)
+      #     handle_something_happening
+      #   end
+      #
+      # WARNING: At the moment, the google pub/sub subscriber only has one thread
+      # per subscription id, so execution of this callback _blocks_ receipt of
+      # subsequent events for the subscription.
+    end
+  end
+end

data/lib/staged_event/backoff_timer.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module StagedEvent
+  class BackoffTimer
+    INCREMENTS_BEFORE_BACKOFF = 5
+    MAX_VALUE = 25
+
+    def initialize
+      reset
+    end
+
+    def increment
+      @increments += 1
+    end
+
+    def reset
+      @increments = 0
+    end
+
+    def value
+      # returns 1 until the timer has been incremented n times, after which it
+      # returns n^2 (until reaching a set maximum)
+      backoff_time = [ 1, increments - INCREMENTS_BEFORE_BACKOFF + 1 ].max**2
+      [ backoff_time, MAX_VALUE ].min
+    end
+
+    private
+
+    attr_accessor :increments
+  end
+end

data/lib/staged_event/configuration.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require "directive"
+
+module StagedEvent
+  module Configuration
+    extend Directive
+
+    configuration_options do
+      nested :google_pubsub do
+        option :credentials
+        option :topic_map
+        option :subscription_ids
+      end
+
+      nested :publisher do
+        option :batch_size
+      end
+
+      nested :subscriber do
+        option :event_received_callback
+      end
+    end
+  end
+end

data/lib/staged_event/event_envelope.proto

@@ -0,0 +1,14 @@
+syntax = 'proto3';
+
+package staged_event;
+
+import "google/protobuf/any.proto";
+
+// EventEnvelope allows any protobuf object to be embedded into its Any field.
+// By doing so, type information about the embedded object is included. This allows
+// a receiver to deserialize the object without knowing its type ahead of time.
+
+message EventEnvelope {
+  google.protobuf.Any event = 1;
+  string uuid = 2;
+}

data/lib/staged_event/event_envelope_pb.rb

@@ -0,0 +1,18 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: lib/staged_event/event_envelope.proto
+
+require 'google/protobuf'
+
+require 'google/protobuf/any_pb'
+Google::Protobuf::DescriptorPool.generated_pool.build do
+  add_file("lib/staged_event/event_envelope.proto", :syntax => :proto3) do
+    add_message "staged_event.EventEnvelope" do
+      optional :event, :message, 1, "google.protobuf.Any"
+      optional :uuid, :string, 2
+    end
+  end
+end
+
+module StagedEvent
+  EventEnvelope = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("staged_event.EventEnvelope").msgclass
+end

data/lib/staged_event/google_pub_sub/helper.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module StagedEvent
+  module GooglePubSub
+    class Helper
+      class << self
+        def new_google_pubsub
+          credentials = Configuration.config.google_pubsub.credentials.to_h
+          project_id = credentials[:project_id]
+
+          Google::Cloud::PubSub.new(
+            project_id: project_id,
+            credentials: credentials,
+          )
+        end
+      end
+    end
+  end
+end

data/lib/staged_event/google_pub_sub/publisher.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module StagedEvent
+  module GooglePubSub
+    class Publisher < StagedEvent::Publisher
+      def initialize
+        @google_pubsub = Helper.new_google_pubsub
+
+        raise ArgumentError, "topic_map must be initialized" unless topic_map.is_a?(Hash) && topic_map.any?
+      end
+
+      def publish(model)
+        topic_name = model.topic || topic_map.keys.first
+        topic_id = topic_map.fetch(topic_name)
+        google_topic = google_pubsub.topic(topic_id, skip_lookup: true)
+
+        # https://github.com/googleapis/google-cloud-ruby/blob/720d14d3641a60c0fab0bf8519bdd730a753a897/google-cloud-pubsub/lib/google/cloud/pubsub/topic.rb#L655
+        google_topic.publish(model.data)
+      end
+
+      private
+
+      attr_reader :google_pubsub
+
+      def topic_map
+        Configuration.config.google_pubsub.topic_map
+      end
+    end
+  end
+end

data/lib/staged_event/google_pub_sub/subscriber.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module StagedEvent
+  module GooglePubSub
+    class Subscriber < StagedEvent::Subscriber
+      include Technologic
+
+      def initialize
+        @google_pubsub = Helper.new_google_pubsub
+
+        raise ArgumentError, "event_received_callback is undefined" unless event_received_callback.respond_to?(:call)
+      end
+
+      def receive_events
+        threads = []
+
+        subscription_ids.each do |subscription_id|
+          threads << Thread.new do
+            receive_events_from_subscription(subscription_id)
+          end
+        end
+
+        threads.each(&:join)
+      end
+
+      def receive_events_from_subscription(subscription_id)
+        subscription = google_pubsub.subscription(subscription_id)
+
+        loop do
+          received_messages = subscription.pull(immediate: false)
+          received_messages.each do |received_message|
+            event_received_callback.call(received_message.data)
+            received_message.acknowledge!
+          end
+        end
+      rescue StandardError => exception
+        error :subscription_failed, exception: exception.message
+        retry
+      end
+
+      # TODO: Google pub/sub has built-in multi-threaded listeners but I haven't
+      # been able to successfully receive any messages using that API yet.
+      #
+      # def receive_events_from_subscription(subscription_id)
+      #   subscription = google_pubsub.subscription(subscription_id)
+      #   subscriber = subscription.listen(threads: { callback: 5 }) do |message|
+      #     event_received_callback.call(message.data)
+      #     message.acknowledge!
+      #   end
+      #
+      #   # Start background threads that will call the block passed to listen.
+      #   subscriber.start
+      #
+      #   # Block, letting processing threads continue in the background
+      #   sleep
+      # end
+
+      private
+
+      attr_reader :google_pubsub
+
+      def subscription_ids
+        Configuration.config.google_pubsub.subscription_ids
+      end
+
+      def event_received_callback
+        Configuration.config.subscriber.event_received_callback
+      end
+    end
+  end
+end

data/lib/staged_event/model.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require "active_record"
+
+module StagedEvent
+  class Model < ActiveRecord::Base
+    self.table_name = :staged_events
+  end
+end

data/lib/staged_event/publisher.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module StagedEvent
+  class Publisher
+    def publish(_model)
+      raise StandardError, "You must implement the publish method"
+    end
+  end
+end

data/lib/staged_event/publisher_process.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module StagedEvent
+  class PublisherProcess
+    include Technologic
+
+    def initialize(publisher)
+      @publisher = publisher
+      @backoff_timer = BackoffTimer.new
+    end
+
+    def run
+      loop do
+        sleep(backoff_timer.value)
+        publish_next_batch
+      end
+    rescue StandardError => exception
+      error :publish_failed, exception: exception.message
+      backoff_timer.increment
+      retry
+    end
+
+    private
+
+    attr_reader :publisher, :backoff_timer
+
+    def publish_next_batch
+      ActiveRecord::Base.connection_pool.with_connection do
+        ActiveRecord::Base.transaction do
+          events = Model.order(:created_at).limit(batch_size).lock("FOR UPDATE SKIP LOCKED")
+          if events.any?
+            events.each do |event|
+              publisher.publish(event)
+            end
+            events.destroy_all
+            backoff_timer.reset
+          else
+            backoff_timer.increment
+          end
+        end
+      end
+    end
+
+    def batch_size
+      Configuration.config.publisher.batch_size
+    end
+  end
+end

data/lib/staged_event/railtie.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "rails/railtie"
+
+module Labyrinth
+  class Railtie < Rails::Railtie
+    rake_tasks do
+      load "tasks/publisher.rake"
+      load "tasks/subscriber.rake"
+    end
+  end
+end

data/lib/staged_event/subscriber.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module StagedEvent
+  class Subscriber
+    def receive_events
+      raise StandardError, "You must implement the receive_events method"
+    end
+  end
+end

data/lib/staged_event/subscriber_process.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module StagedEvent
+  class SubscriberProcess
+    include Technologic
+
+    def initialize(subscriber)
+      @subscriber = subscriber
+    end
+
+    def run
+      subscriber.receive_events
+    rescue StandardError => exception
+      error :subscriber_main_loop_failed, exception: exception.message
+      retry
+    end
+
+    private
+
+    attr_reader :subscriber
+  end
+end

data/lib/staged_event/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module StagedEvent
+  VERSION = "0.0.1"
+end

data/lib/staged_event.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require "google/cloud/pubsub"
+
+require_relative "staged_event/backoff_timer"
+require_relative "staged_event/configuration"
+require_relative "staged_event/event_envelope_pb"
+require_relative "staged_event/model"
+require_relative "staged_event/publisher"
+require_relative "staged_event/subscriber"
+require_relative "staged_event/publisher_process"
+require_relative "staged_event/subscriber_process"
+require_relative "staged_event/version"
+
+require_relative "staged_event/google_pub_sub/helper"
+require_relative "staged_event/google_pub_sub/publisher"
+require_relative "staged_event/google_pub_sub/subscriber"
+
+require_relative "staged_event/railtie" if defined?(Rails)
+
+module StagedEvent
+  class Error < StandardError; end
+
+  class DeserializationError < Error; end
+  class UnknownEventTypeError < Error; end
+
+  class << self
+    # Builds an ActiveRecord model from a proto object representing an event.
+    # If the model is committed to the database, it will be published by the
+    # publisher process.
+    #
+    # @param [Instance of a protobuf generated class] proto the event data to publish
+    # @option kwargs [String] :topic the topic that will receive the event
+    # @return [StagedEvent::Model] an ActiveRecord model ready to be saved
+    def from_proto(proto, **kwargs)
+      uuid = SecureRandom.uuid
+
+      envelope = EventEnvelope.new(
+        event: {
+          type_url: proto.class.descriptor.name,
+          value: proto.class.encode(proto),
+        },
+        uuid: uuid,
+      )
+
+      data = EventEnvelope.encode(envelope)
+      topic = kwargs.fetch(:topic, nil)
+
+      Model.new(id: uuid, data: data, topic: topic)
+    end
+
+    # Converts serialized event data received from a publisher into an object with
+    # the event (and its metadata) in accessible form
+    #
+    # @param [String] serialized_data the serialized event data
+    # @return [OpenStruct] an object representing the event
+    def deserialize_event(serialized_data)
+      envelope = EventEnvelope.decode(serialized_data)
+      event_descriptor = Google::Protobuf::DescriptorPool.generated_pool.lookup(envelope.event.type_url)
+      raise UnknownEventTypeError if event_descriptor.blank?
+
+      proto = event_descriptor.msgclass.decode(envelope.event.value)
+
+      OpenStruct.new(
+        id: envelope.uuid,
+        data: proto,
+      )
+    rescue Google::Protobuf::ParseError
+      raise DeserializationError
+    end
+  end
+end

data/lib/tasks/publisher.rake

@@ -0,0 +1,6 @@
+namespace :staged_event do
+  desc "Run the staged_event publisher process"
+  task :publisher => :environment do
+    StagedEvent::PublisherProcess.new(StagedEvent::GooglePubSub::Publisher.new).run
+  end
+end

data/lib/tasks/subscriber.rake

@@ -0,0 +1,6 @@
+namespace :staged_event do
+  desc "Run the staged_event subscriber process"
+  task :subscriber => :environment do |task, args|
+    StagedEvent::SubscriberProcess.new(StagedEvent::GooglePubSub::Subscriber.new).run
+  end
+end

metadata

@@ -0,0 +1,209 @@
+--- !ruby/object:Gem::Specification
+name: staged_event
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Andrew Cross
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2021-10-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 6.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 6.0.0
+- !ruby/object:Gem::Dependency
+  name: directive
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.23.1.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.23.1.1
+- !ruby/object:Gem::Dependency
+  name: google-protobuf
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.8.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 4.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.8.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 4.0.0
+- !ruby/object:Gem::Dependency
+  name: google-cloud-pubsub
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: database_cleaner
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: faker
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A transactional outbox implementation for Ruby/ActiveRecord
+email: andrew.cross@freshly.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- lib/generators/staged_event/install_generator.rb
+- lib/generators/staged_event/templates/create_staged_events.rb.erb
+- lib/generators/staged_event/templates/initializer.rb.erb
+- lib/staged_event.rb
+- lib/staged_event/backoff_timer.rb
+- lib/staged_event/configuration.rb
+- lib/staged_event/event_envelope.proto
+- lib/staged_event/event_envelope_pb.rb
+- lib/staged_event/google_pub_sub/helper.rb
+- lib/staged_event/google_pub_sub/publisher.rb
+- lib/staged_event/google_pub_sub/subscriber.rb
+- lib/staged_event/model.rb
+- lib/staged_event/publisher.rb
+- lib/staged_event/publisher_process.rb
+- lib/staged_event/railtie.rb
+- lib/staged_event/subscriber.rb
+- lib/staged_event/subscriber_process.rb
+- lib/staged_event/version.rb
+- lib/tasks/publisher.rake
+- lib/tasks/subscriber.rake
+homepage: https://github.com/Freshly/staged_event
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.15
+signing_key:
+specification_version: 4
+summary: StagedEvent
+test_files: []