journaled 2.2.0 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5f46775bfd5621ba7978529c64da7768d471ee6fd73cc8cf315d8fa60e98c537
-  data.tar.gz: f0bbb3d98be52d715d847a1ee44c843d8c4f654826939167a710b04a23c4adb6
+  metadata.gz: 358c3ced2b976724c62e951ca88b635709d1fb7da0276ed4538bdc44bc02f468
+  data.tar.gz: ca65a5b0531ba4540de1b79fd36d247c7c164f3aef8b397254ed7b34503ebf0f
 SHA512:
-  metadata.gz: 82c607495e4dc58e70005a73b32d473d84d85baaa4b3004d8ce018ea65d178a4eafafee7c64336ba3616dc4d6b65349bbc4d2209a770aa858cfdd92f9c07043d
-  data.tar.gz: 803c3f09b67ac93ff28df87ad40680f7c05daed85c7da7110c35c817ccd2cd4f2a4b1995540f5659ce5b3a839b124f4fccbeed91686f540280aa0abe25bb05c9
+  metadata.gz: 236c6169ef41b1af9043f2549248ae43fad665a91d43cdd2f579e891685788aa5a008cf4f1dc8aba13abcce5dccfa6513c82232fdaaf3ab4f42f8a7b1b222bf7
+  data.tar.gz: 9e325f6c1e41eb577f5e0d7320517f5435b9570d7b0274c5b6ad9c593abc2a4a66dc6b6eb10d668617e645e5731804ecd65d4b6403fda63005de129443ed9332

data/README.md

@@ -1,16 +1,16 @@
 # Journaled
 
-A Rails engine to durably deliver schematized events to Amazon Kinesis via DelayedJob.
+A Rails engine to durably deliver schematized events to Amazon Kinesis via ActiveJob.
 
 More specifically, `journaled` is composed of three opinionated pieces:
 schema definition/validation via JSON Schema, transactional enqueueing
-via Delayed::Job (specifically `delayed_job_active_record`), and event
+via ActiveJob (specifically, via a DB-backed queue adapter), and event
 transmission via Amazon Kinesis. Our current use-cases include
 transmitting audit events for durable storage in S3 and/or analytical
 querying in Amazon Redshift.
 
 Journaled provides an at-least-once event delivery guarantee assuming
-Delayed::Job is configured not to delete jobs on failure.
+ActiveJob's queue adapter is not configured to delete jobs on failure.
 
 Note: Do not use the journaled gem to build an event sourcing solution
 as it does not guarantee total ordering of events. It's possible we'll
@@ -20,9 +20,20 @@ durable, eventually consistent record that discrete events happened.
 
 ## Installation
 
-1. [Install `delayed_job_active_record`](https://github.com/collectiveidea/delayed_job_active_record#installation)
-if you haven't already.
+1. If you haven't already,
+[configure ActiveJob](https://guides.rubyonrails.org/active_job_basics.html)
+to use one of the following queue adapters:
 
+- `:delayed_job` (via `delayed_job_active_record`)
+- `:que`
+- `:good_job`
+- `:delayed`
+
+Ensure that your queue adapter is not configured to delete jobs on failure.
+
+**If you launch your application in production mode and the gem detects that
+`ActiveJob::Base.queue_adapter` is not in the above list, it will raise an exception
+and prevent your application from performing unsafe journaling.**
 
 2. To integrate Journaled into your application, simply include the gem in your
 app's Gemfile.
@@ -85,9 +96,34 @@ Journaling provides a number of different configuation options that can be set i
 
 #### `Journaled.job_priority` (default: 20)
 
-  This can be used to configure what `priority` the Delayed Jobs are enqueued with. This will be applied to all the Journaled::Devivery jobs that are created by this application.
+  This can be used to configure what `priority` the ActiveJobs are enqueued with. This will be applied to all the `Journaled::DeliveryJob`s that are created by this application.
   Ex: `Journaled.job_priority = 14`
 
+  _Note that job priority is only supported on Rails 6.0+. Prior Rails versions will ignore this parameter and enqueue jobs with the underlying ActiveJob adapter's default priority._
+
+#### `Journaled.http_idle_timeout` (default: 1 second)
+
+  The number of seconds a persistent connection is allowed to sit idle before it should no longer be used.
+
+#### `Journaled.http_open_timeout` (default: 2 seconds)
+
+  The number of seconds before the :http_handler should timeout while trying to open a new HTTP session.
+
+#### `Journaled.http_read_timeout` (default: 60 seconds)
+
+  The number of seconds before the :http_handler should timeout while waiting for a HTTP response.
+
+#### ActiveJob `set` options
+
+Both model-level directives accept additional options to be passed into ActiveJob's `set` method:
+
+```ruby
+# For change journaling:
+journal_changes_to :email, as: :identity_change, enqueue_with: { priority: 10 }
+
+# Or for custom journaling:
+journal_attributes :email, enqueue_with: { priority: 20, queue: 'journaled' }
+```
 
 ### Change Journaling
 

data/Rakefile

@@ -28,7 +28,13 @@ if %w(development test).include? Rails.env
   RuboCop::RakeTask.new
 
   task(:default).clear
-  task default: %i(rubocop spec)
+  if ENV['APPRAISAL_INITIALIZED'] || ENV['CI']
+    task default: %i(rubocop spec)
+  else
+    require 'appraisal'
+    Appraisal::Task.new
+    task default: :appraisal
+  end
 
   task 'db:test:prepare' => 'db:setup'
 end

data/app/jobs/journaled/application_job.rb

@@ -0,0 +1,4 @@
+module Journaled
+  class ApplicationJob < Journaled.job_base_class_name.constantize
+  end
+end

data/app/jobs/journaled/delivery_job.rb

@@ -0,0 +1,96 @@
+module Journaled
+  class DeliveryJob < ApplicationJob
+    DEFAULT_REGION = 'us-east-1'.freeze
+
+    rescue_from(Aws::Kinesis::Errors::InternalFailure, Aws::Kinesis::Errors::ServiceUnavailable, Aws::Kinesis::Errors::Http503Error) do |e|
+      Rails.logger.error "Kinesis Error - Server Error occurred - #{e.class}"
+      raise KinesisTemporaryFailure
+    end
+
+    rescue_from(Seahorse::Client::NetworkingError) do |e|
+      Rails.logger.error "Kinesis Error - Networking Error occurred - #{e.class}"
+      raise KinesisTemporaryFailure
+    end
+
+    def perform(serialized_event:, partition_key:, app_name:)
+      @serialized_event = serialized_event
+      @partition_key = partition_key
+      @app_name = app_name
+
+      journal!
+    end
+
+    def self.stream_name(app_name:)
+      env_var_name = [app_name&.upcase, 'JOURNALED_STREAM_NAME'].compact.join('_')
+      ENV.fetch(env_var_name)
+    end
+
+    def kinesis_client_config
+      {
+        region: ENV.fetch('AWS_DEFAULT_REGION', DEFAULT_REGION),
+        retry_limit: 0,
+        http_idle_timeout: Journaled.http_idle_timeout,
+        http_open_timeout: Journaled.http_open_timeout,
+        http_read_timeout: Journaled.http_read_timeout,
+      }.merge(credentials)
+    end
+
+    private
+
+    attr_reader :serialized_event, :partition_key, :app_name
+
+    def journal!
+      kinesis_client.put_record record if Journaled.enabled?
+    end
+
+    def record
+      {
+        stream_name: self.class.stream_name(app_name: app_name),
+        data: serialized_event,
+        partition_key: partition_key,
+      }
+    end
+
+    def kinesis_client
+      Aws::Kinesis::Client.new(kinesis_client_config)
+    end
+
+    def credentials
+      if ENV.key?('JOURNALED_IAM_ROLE_ARN')
+        {
+          credentials: iam_assume_role_credentials,
+        }
+      else
+        legacy_credentials_hash_if_present
+      end
+    end
+
+    def legacy_credentials_hash_if_present
+      if ENV.key?('RUBY_AWS_ACCESS_KEY_ID')
+        {
+          access_key_id: ENV.fetch('RUBY_AWS_ACCESS_KEY_ID'),
+          secret_access_key: ENV.fetch('RUBY_AWS_SECRET_ACCESS_KEY'),
+        }
+      else
+        {}
+      end
+    end
+
+    def sts_client
+      Aws::STS::Client.new({
+        region: ENV.fetch('AWS_DEFAULT_REGION', DEFAULT_REGION),
+      }.merge(legacy_credentials_hash_if_present))
+    end
+
+    def iam_assume_role_credentials
+      @iam_assume_role_credentials ||= Aws::AssumeRoleCredentials.new(
+        client: sts_client,
+        role_arn: ENV.fetch('JOURNALED_IAM_ROLE_ARN'),
+        role_session_name: "JournaledAssumeRoleAccess",
+      )
+    end
+
+    class KinesisTemporaryFailure < NotTrulyExceptionalError
+    end
+  end
+end

data/app/models/concerns/journaled/changes.rb

@@ -2,10 +2,9 @@ module Journaled::Changes
   extend ActiveSupport::Concern
 
   included do
-    cattr_accessor :_journaled_change_definitions
-    cattr_accessor :journaled_attribute_names
-    self._journaled_change_definitions = []
-    self.journaled_attribute_names = []
+    cattr_accessor(:_journaled_change_definitions) { [] }
+    cattr_accessor(:journaled_attribute_names) { [] }
+    cattr_accessor(:journaled_enqueue_opts, instance_writer: false) { {} }
 
     after_create do
       self.class._journaled_change_definitions.each do |definition|
@@ -57,7 +56,7 @@ module Journaled::Changes
   end
 
   class_methods do
-    def journal_changes_to(*attribute_names, as:) # rubocop:disable Naming/UncommunicativeMethodParamName
+    def journal_changes_to(*attribute_names, as:, enqueue_with: {}) # rubocop:disable Naming/UncommunicativeMethodParamName
       if attribute_names.empty? || attribute_names.any? { |n| !n.is_a?(Symbol) }
         raise "one or more symbol attribute_name arguments is required"
       end
@@ -66,6 +65,7 @@ module Journaled::Changes
 
       _journaled_change_definitions << Journaled::ChangeDefinition.new(attribute_names: attribute_names, logical_operation: as)
       journaled_attribute_names.concat(attribute_names)
+      journaled_enqueue_opts.merge!(enqueue_with)
     end
 
     if Rails::VERSION::MAJOR > 5 || (Rails::VERSION::MAJOR == 5 && Rails::VERSION::MINOR >= 2)

data/app/models/journaled/change.rb

@@ -7,6 +7,7 @@ class Journaled::Change
     :logical_operation,
     :changes,
     :journaled_app_name,
+    :journaled_enqueue_opts,
     :actor
 
   journal_attributes :table_name,
@@ -22,6 +23,7 @@ class Journaled::Change
                  logical_operation:,
                  changes:,
                  journaled_app_name:,
+                 journaled_enqueue_opts:,
                  actor:)
     @table_name = table_name
     @record_id = record_id
@@ -29,6 +31,7 @@ class Journaled::Change
     @logical_operation = logical_operation
     @changes = changes
     @journaled_app_name = journaled_app_name
+    @journaled_enqueue_opts = journaled_enqueue_opts
     @actor = actor
   end
 end

data/app/models/journaled/change_writer.rb

@@ -28,6 +28,7 @@ class Journaled::ChangeWriter
       logical_operation: logical_operation,
       changes: JSON.dump(changes),
       journaled_app_name: journaled_app_name,
+      journaled_enqueue_opts: model.journaled_enqueue_opts,
       actor: actor_uri,
     )
   end

data/app/models/journaled/delivery.rb

@@ -1,4 +1,4 @@
-class Journaled::Delivery
+class Journaled::Delivery # rubocop:disable Betterment/ActiveJobPerformable
   DEFAULT_REGION = 'us-east-1'.freeze
 
   def initialize(serialized_event:, partition_key:, app_name:)
@@ -26,6 +26,9 @@ class Journaled::Delivery
     {
       region: ENV.fetch('AWS_DEFAULT_REGION', DEFAULT_REGION),
       retry_limit: 0,
+      http_idle_timeout: Journaled.http_idle_timeout,
+      http_open_timeout: Journaled.http_open_timeout,
+      http_read_timeout: Journaled.http_read_timeout,
     }.merge(credentials)
   end
 
@@ -80,6 +83,6 @@ class Journaled::Delivery
     )
   end
 
-  class KinesisTemporaryFailure < NotTrulyExceptionalError
+  class KinesisTemporaryFailure < ::Journaled::NotTrulyExceptionalError
   end
 end

data/app/models/journaled/event.rb

@@ -2,7 +2,7 @@ module Journaled::Event
   extend ActiveSupport::Concern
 
   def journal!
-    Journaled::Writer.new(journaled_event: self, priority: Journaled.job_priority).journal!
+    Journaled::Writer.new(journaled_event: self).journal!
   end
 
   # Base attributes
@@ -42,8 +42,9 @@ module Journaled::Event
   private
 
   class_methods do
-    def journal_attributes(*args)
+    def journal_attributes(*args, enqueue_with: {})
       journaled_attributes.concat(args)
+      journaled_enqueue_opts.merge!(enqueue_with)
     end
 
     def journaled_attributes
@@ -56,6 +57,8 @@ module Journaled::Event
   end
 
   included do
+    cattr_accessor(:journaled_enqueue_opts, instance_writer: false) { {} }
+
     journal_attributes :id, :event_type, :created_at
   end
 end

data/app/models/journaled/writer.rb

@@ -4,9 +4,10 @@ class Journaled::Writer
     journaled_partition_key
     journaled_attributes
     journaled_app_name
+    journaled_enqueue_opts
   ).freeze
 
-  def initialize(journaled_event:, priority:)
+  def initialize(journaled_event:)
     raise "An enqueued event must respond to: #{EVENT_METHOD_NAMES.to_sentence}" unless respond_to_all?(journaled_event, EVENT_METHOD_NAMES)
 
     unless journaled_event.journaled_schema_name.present? &&
@@ -21,26 +22,27 @@ class Journaled::Writer
     end
 
     @journaled_event = journaled_event
-    @priority = priority
   end
 
   def journal!
     base_event_json_schema_validator.validate! serialized_event
     json_schema_validator.validate! serialized_event
-    Delayed::Job.enqueue journaled_delivery, priority: priority
+    Journaled::DeliveryJob
+      .set(journaled_enqueue_opts.reverse_merge(priority: Journaled.job_priority))
+      .perform_later(delivery_perform_args)
   end
 
   private
 
-  attr_reader :journaled_event, :priority
-  delegate :journaled_schema_name, :journaled_attributes, :journaled_partition_key, :journaled_app_name, to: :journaled_event
+  attr_reader :journaled_event
+  delegate(*EVENT_METHOD_NAMES, to: :journaled_event)
 
-  def journaled_delivery
-    @journaled_delivery ||= Journaled::Delivery.new(
+  def delivery_perform_args
+    {
       serialized_event: serialized_event,
       partition_key: journaled_partition_key,
       app_name: journaled_app_name,
-    )
+    }
   end
 
   def serialized_event

data/lib/journaled.rb

@@ -1,14 +1,19 @@
-require "aws-sdk-resources"
-require "delayed_job"
+require "aws-sdk-kinesis"
+require "active_job"
 require "json-schema"
 require "request_store"
 
 require "journaled/engine"
-require "journaled/job_priority"
 
 module Journaled
+  SUPPORTED_QUEUE_ADAPTERS = %w(delayed delayed_job good_job que).freeze
+
   mattr_accessor :default_app_name
-  mattr_accessor(:job_priority) { Journaled::JobPriority::EVENTUAL }
+  mattr_accessor(:job_priority) { 20 }
+  mattr_accessor(:http_idle_timeout) { 5 }
+  mattr_accessor(:http_open_timeout) { 2 }
+  mattr_accessor(:http_read_timeout) { 60 }
+  mattr_accessor(:job_base_class_name) { 'ActiveJob::Base' }
 
   def development_or_test?
     %w(development test).include?(Rails.env)
@@ -30,5 +35,21 @@ module Journaled
     Journaled::ActorUriProvider.instance.actor_uri
   end
 
-  module_function :development_or_test?, :enabled?, :schema_providers, :commit_hash, :actor_uri
+  def detect_queue_adapter!
+    adapter = job_base_class_name.constantize.queue_adapter.class.name.split('::').last.underscore.gsub("_adapter", "")
+    unless SUPPORTED_QUEUE_ADAPTERS.include?(adapter)
+      raise <<~MSG
+        Journaled has detected an unsupported ActiveJob queue adapter: `:#{adapter}`
+
+        Journaled jobs must be enqueued transactionally to your primary database.
+
+        Please install the appropriate gems and set `queue_adapter` to one of the following:
+        #{SUPPORTED_QUEUE_ADAPTERS.map { |a| "- `:#{a}`" }.join("\n")}
+
+        Read more at https://github.com/Betterment/journaled
+      MSG
+    end
+  end
+
+  module_function :development_or_test?, :enabled?, :schema_providers, :commit_hash, :actor_uri, :detect_queue_adapter!
 end

data/lib/journaled/engine.rb

@@ -1,4 +1,9 @@
 module Journaled
   class Engine < ::Rails::Engine
+    config.after_initialize do
+      ActiveSupport.on_load(:active_job) do
+        Journaled.detect_queue_adapter! unless Journaled.development_or_test?
+      end
+    end
   end
 end

data/lib/journaled/relation_change_protection.rb

@@ -1,5 +1,5 @@
 module Journaled::RelationChangeProtection
-  def update_all(updates, force: false) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/PerceivedComplexity
+  def update_all(updates, force: false) # rubocop:disable Metrics/AbcSize, Metrics/PerceivedComplexity
     unless force || !@klass.respond_to?(:journaled_attribute_names) || @klass.journaled_attribute_names.empty?
       conflicting_journaled_attribute_names = if updates.is_a?(Hash)
                                                 @klass.journaled_attribute_names & updates.keys.map(&:to_sym)

data/lib/journaled/version.rb

@@ -1,3 +1,3 @@
 module Journaled
-  VERSION = "2.2.0".freeze
+  VERSION = "3.0.0".freeze
 end