journaled 2.2.0 → 3.0.0

data/spec/dummy/config/application.rb

@@ -1,10 +1,9 @@
 require File.expand_path('boot', __dir__)
 
 require "active_record/railtie"
+require "active_job/railtie"
 require "active_model/railtie"
 require "action_controller/railtie"
-require "action_mailer/railtie"
-require "sprockets/railtie"
 
 Bundler.require(*Rails.groups)
 require "journaled"

data/spec/dummy/config/database.yml

@@ -1,21 +1,6 @@
-default: &default
-  pool: 5
-  encoding: unicode
-
-postgresql:
-  default: &postgres_default
-    adapter: postgresql
-    url: <%= ENV['DATABASE_URL'] %>
-  test: &postgres_test
-    <<: *postgres_default
-    url: <%= ENV['DATABASE_URL'] || "postgresql://localhost:#{ENV.fetch('PGPORT', 5432)}/journaled_test" %>
-    database: journaled_test
-  development: &postgres_development
-    <<: *postgres_default
-    url: <%= ENV['DATABASE_URL'] || "postgresql://localhost:#{ENV.fetch('PGPORT', 5432)}/journaled_development" %>
-    database: journaled_development
-
 development:
-  <<: *postgres_development
+  adapter: sqlite3
+  database: ":memory:"
 test:
-  <<: *postgres_test
+  adapter: sqlite3
+  database: ":memory:"

data/spec/dummy/config/environments/development.rb

@@ -13,25 +13,12 @@ Rails.application.configure do
   config.consider_all_requests_local       = true
   config.action_controller.perform_caching = false
 
-  # Don't care if the mailer can't send.
-  config.action_mailer.raise_delivery_errors = false
-
   # Print deprecation notices to the Rails logger.
   config.active_support.deprecation = :log
 
   # Raise an error on page load if there are pending migrations.
   config.active_record.migration_error = :page_load
 
-  # Debug mode disables concatenation and preprocessing of assets.
-  # This option may cause significant delays in view rendering with a large
-  # number of complex assets.
-  config.assets.debug = true
-
-  # Adds additional error checking when serving assets at runtime.
-  # Checks for improperly declared sprockets dependencies.
-  # Raises helpful error messages.
-  config.assets.raise_runtime_errors = true
-
   # Raises error for missing translations
   # config.action_view.raise_on_missing_translations = true
 end

data/spec/dummy/config/environments/test.rb

@@ -26,14 +26,12 @@ Rails.application.configure do
   # Disable request forgery protection in test environment.
   config.action_controller.allow_forgery_protection = false
 
-  # Tell Action Mailer not to deliver emails to the real world.
-  # The :test delivery method accumulates sent emails in the
-  # ActionMailer::Base.deliveries array.
-  config.action_mailer.delivery_method = :test
-
   # Print deprecation notices to the stderr.
   config.active_support.deprecation = :stderr
 
   # Raises error for missing translations
   # config.action_view.raise_on_missing_translations = true
+
+  # Use ActiveJob test adapter
+  config.active_job.queue_adapter = :test
 end

data/spec/dummy/db/schema.rb

@@ -11,21 +11,8 @@
 # It's strongly recommended that you check this file into your version control system.
 
 ActiveRecord::Schema.define(version: 20180606205114) do
-  # These are extensions that must be enabled in order to support this database
-  enable_extension "plpgsql"
-
-  create_table "delayed_jobs", force: :cascade do |t|
-    t.integer "priority", default: 0, null: false
-    t.integer "attempts", default: 0, null: false
-    t.text "handler", null: false
-    t.text "last_error"
-    t.datetime "run_at"
-    t.datetime "locked_at"
-    t.datetime "failed_at"
-    t.string "locked_by"
-    t.string "queue"
-    t.datetime "created_at"
-    t.datetime "updated_at"
-    t.index ["priority", "run_at"], name: "delayed_jobs_priority"
+  create_table "widgets", force: :cascade do |t|
+    t.string "name"
+    t.string "other_column"
   end
 end

data/spec/jobs/journaled/delivery_job_spec.rb

@@ -0,0 +1,221 @@
+require 'rails_helper'
+
+RSpec.describe Journaled::DeliveryJob do
+  let(:stream_name) { 'test_events' }
+  let(:partition_key) { 'fake_partition_key' }
+  let(:serialized_event) { '{"foo":"bar"}' }
+  let(:kinesis_client) { Aws::Kinesis::Client.new(stub_responses: true) }
+  let(:args) { { serialized_event: serialized_event, partition_key: partition_key, app_name: nil } }
+
+  around do |example|
+    with_env(JOURNALED_STREAM_NAME: stream_name) { example.run }
+  end
+
+  describe '#perform' do
+    let(:return_status_body) { { shard_id: '101', sequence_number: '101123' } }
+    let(:return_object) { instance_double Aws::Kinesis::Types::PutRecordOutput, return_status_body }
+
+    before do
+      allow(Aws::AssumeRoleCredentials).to receive(:new).and_call_original
+      allow(Aws::Kinesis::Client).to receive(:new).and_return kinesis_client
+      kinesis_client.stub_responses(:put_record, return_status_body)
+
+      allow(Journaled).to receive(:enabled?).and_return(true)
+    end
+
+    it 'makes requests to AWS to put the event on the Kinesis with the correct body' do
+      event = described_class.perform_now(args)
+
+      expect(event.shard_id).to eq '101'
+      expect(event.sequence_number).to eq '101123'
+    end
+
+    context 'when JOURNALED_IAM_ROLE_ARN is defined' do
+      let(:aws_sts_client) { Aws::STS::Client.new(stub_responses: true) }
+
+      around do |example|
+        with_env(JOURNALED_IAM_ROLE_ARN: 'iam-role-arn-for-assuming-kinesis-access') { example.run }
+      end
+
+      before do
+        allow(Aws::STS::Client).to receive(:new).and_return aws_sts_client
+        aws_sts_client.stub_responses(:assume_role, assume_role_response)
+      end
+
+      let(:assume_role_response) do
+        {
+          assumed_role_user: {
+            arn: 'iam-role-arn-for-assuming-kinesis-access',
+            assumed_role_id: "ARO123EXAMPLE123:Bob",
+          },
+          credentials: {
+            access_key_id: "AKIAIOSFODNN7EXAMPLE",
+            secret_access_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYzEXAMPLEKEY",
+            session_token: "EXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI",
+            expiration: Time.zone.parse("2011-07-15T23:28:33.359Z"),
+          },
+        }
+      end
+
+      it 'initializes a Kinesis client with assume role credentials' do
+        described_class.perform_now(args)
+
+        expect(Aws::AssumeRoleCredentials).to have_received(:new).with(
+          client: aws_sts_client,
+          role_arn: "iam-role-arn-for-assuming-kinesis-access",
+          role_session_name: "JournaledAssumeRoleAccess",
+        )
+      end
+    end
+
+    context 'when the stream name env var is NOT set' do
+      let(:stream_name) { nil }
+
+      it 'raises an KeyError error' do
+        expect { described_class.perform_now(args) }.to raise_error KeyError
+      end
+    end
+
+    context 'when Amazon responds with an InternalFailure' do
+      before do
+        kinesis_client.stub_responses(:put_record, 'InternalFailure')
+      end
+
+      it 'catches the error and re-raises a subclass of NotTrulyExceptionalError and logs about the failure' do
+        allow(Rails.logger).to receive(:error)
+        expect { described_class.perform_now(args) }.to raise_error described_class::KinesisTemporaryFailure
+        expect(Rails.logger).to have_received(:error).with(
+          "Kinesis Error - Server Error occurred - Aws::Kinesis::Errors::InternalFailure",
+        ).once
+      end
+    end
+
+    context 'when Amazon responds with a ServiceUnavailable' do
+      before do
+        kinesis_client.stub_responses(:put_record, 'ServiceUnavailable')
+      end
+
+      it 'catches the error and re-raises a subclass of NotTrulyExceptionalError and logs about the failure' do
+        allow(Rails.logger).to receive(:error)
+        expect { described_class.perform_now(args) }.to raise_error described_class::KinesisTemporaryFailure
+        expect(Rails.logger).to have_received(:error).with(/\AKinesis Error/).once
+      end
+    end
+
+    context 'when we receive a 504 Gateway timeout' do
+      before do
+        kinesis_client.stub_responses(:put_record, 'Aws::Kinesis::Errors::ServiceError')
+      end
+
+      it 'raises an error that subclasses Aws::Kinesis::Errors::ServiceError' do
+        expect { described_class.perform_now(args) }.to raise_error Aws::Kinesis::Errors::ServiceError
+      end
+    end
+
+    context 'when the IAM user does not have permission to put_record to the specified stream' do
+      before do
+        kinesis_client.stub_responses(:put_record, 'AccessDeniedException')
+      end
+
+      it 'raises an AccessDeniedException error' do
+        expect { described_class.perform_now(args) }.to raise_error Aws::Kinesis::Errors::AccessDeniedException
+      end
+    end
+
+    context 'when the request timesout' do
+      before do
+        kinesis_client.stub_responses(:put_record, Seahorse::Client::NetworkingError.new(Timeout::Error.new))
+      end
+
+      it 'catches the error and re-raises a subclass of NotTrulyExceptionalError and logs about the failure' do
+        allow(Rails.logger).to receive(:error)
+        expect { described_class.perform_now(args) }.to raise_error described_class::KinesisTemporaryFailure
+        expect(Rails.logger).to have_received(:error).with(
+          "Kinesis Error - Networking Error occurred - Seahorse::Client::NetworkingError",
+        ).once
+      end
+    end
+  end
+
+  describe ".stream_name" do
+    context "when app_name is unspecified" do
+      it "is fetched from a prefixed ENV var if specified" do
+        allow(ENV).to receive(:fetch).and_return("expected_stream_name")
+        expect(described_class.stream_name(app_name: nil)).to eq("expected_stream_name")
+        expect(ENV).to have_received(:fetch).with("JOURNALED_STREAM_NAME")
+      end
+    end
+
+    context "when app_name is specified" do
+      it "is fetched from a prefixed ENV var if specified" do
+        allow(ENV).to receive(:fetch).and_return("expected_stream_name")
+        expect(described_class.stream_name(app_name: "my_funky_app_name")).to eq("expected_stream_name")
+        expect(ENV).to have_received(:fetch).with("MY_FUNKY_APP_NAME_JOURNALED_STREAM_NAME")
+      end
+    end
+  end
+
+  describe "#kinesis_client_config" do
+    it "is in us-east-1 by default" do
+      with_env(AWS_DEFAULT_REGION: nil) do
+        expect(subject.kinesis_client_config).to include(region: 'us-east-1')
+      end
+    end
+
+    it "respects AWS_DEFAULT_REGION env var" do
+      with_env(AWS_DEFAULT_REGION: 'us-west-2') do
+        expect(subject.kinesis_client_config).to include(region: 'us-west-2')
+      end
+    end
+
+    it "doesn't limit retry" do
+      expect(subject.kinesis_client_config).to include(retry_limit: 0)
+    end
+
+    it "provides no AWS credentials by default" do
+      with_env(RUBY_AWS_ACCESS_KEY_ID: nil, RUBY_AWS_SECRET_ACCESS_KEY: nil) do
+        expect(subject.kinesis_client_config).not_to have_key(:access_key_id)
+        expect(subject.kinesis_client_config).not_to have_key(:secret_access_key)
+      end
+    end
+
+    it "will use legacy credentials if specified" do
+      with_env(RUBY_AWS_ACCESS_KEY_ID: 'key_id', RUBY_AWS_SECRET_ACCESS_KEY: 'secret') do
+        expect(subject.kinesis_client_config).to include(access_key_id: 'key_id', secret_access_key: 'secret')
+      end
+    end
+
+    it "will set http_idle_timeout by default" do
+      expect(subject.kinesis_client_config).to include(http_idle_timeout: 5)
+    end
+
+    it "will set http_open_timeout by default" do
+      expect(subject.kinesis_client_config).to include(http_open_timeout: 2)
+    end
+
+    it "will set http_read_timeout by default" do
+      expect(subject.kinesis_client_config).to include(http_read_timeout: 60)
+    end
+
+    context "when Journaled.http_idle_timeout is specified" do
+      it "will set http_idle_timeout by specified value" do
+        allow(Journaled).to receive(:http_idle_timeout).and_return(2)
+        expect(subject.kinesis_client_config).to include(http_idle_timeout: 2)
+      end
+    end
+
+    context "when Journaled.http_open_timeout is specified" do
+      it "will set http_open_timeout by specified value" do
+        allow(Journaled).to receive(:http_open_timeout).and_return(1)
+        expect(subject.kinesis_client_config).to include(http_open_timeout: 1)
+      end
+    end
+
+    context "when Journaled.http_read_timeout is specified" do
+      it "will set http_read_timeout by specified value" do
+        allow(Journaled).to receive(:http_read_timeout).and_return(2)
+        expect(subject.kinesis_client_config).to include(http_read_timeout: 2)
+      end
+    end
+  end
+end

data/spec/lib/journaled_spec.rb

@@ -49,4 +49,43 @@ RSpec.describe Journaled do
       expect(described_class.actor_uri).to eq "my actor uri"
     end
   end
+
+  describe '.detect_queue_adapter!' do
+    it 'raises an error unless the queue adapter is DB-backed' do
+      expect { described_class.detect_queue_adapter! }.to raise_error <<~MSG
+        Journaled has detected an unsupported ActiveJob queue adapter: `:test`
+
+        Journaled jobs must be enqueued transactionally to your primary database.
+
+        Please install the appropriate gems and set `queue_adapter` to one of the following:
+        - `:delayed`
+        - `:delayed_job`
+        - `:good_job`
+        - `:que`
+
+        Read more at https://github.com/Betterment/journaled
+      MSG
+    end
+
+    context 'when the queue adapter is supported' do
+      before do
+        stub_const("ActiveJob::QueueAdapters::DelayedAdapter", Class.new)
+        ActiveJob::Base.disable_test_adapter
+        ActiveJob::Base.queue_adapter = :delayed
+      end
+
+      around do |example|
+        begin
+          example.run
+        ensure
+          ActiveJob::Base.queue_adapter = :test
+          ActiveJob::Base.enable_test_adapter(ActiveJob::QueueAdapters::TestAdapter.new)
+        end
+      end
+
+      it 'does not raise an error' do
+        expect { described_class.detect_queue_adapter! }.not_to raise_error
+      end
+    end
+  end
 end

data/spec/models/concerns/journaled/changes_spec.rb

@@ -92,4 +92,15 @@ RSpec.describe Journaled::Changes do
     expect(change_writer).to have_received(:delete)
     expect(Journaled::ChangeWriter).to have_received(:new)
   end
+
+  context 'when DJ opts are provided' do
+    before do
+      klass.journal_changes_to :thing, as: :other_thing, enqueue_with: { asdf: 1, foo: 'bar' }
+    end
+
+    it 'sets them on the model' do
+      expect(klass.journaled_enqueue_opts).to eq(asdf: 1, foo: 'bar')
+      expect(klass.new.journaled_enqueue_opts).to eq(asdf: 1, foo: 'bar')
+    end
+  end
 end

data/spec/models/database_change_protection_spec.rb

@@ -4,38 +4,42 @@ if Rails::VERSION::MAJOR > 5 || (Rails::VERSION::MAJOR == 5 && Rails::VERSION::M
   # rubocop:disable Rails/SkipsModelValidations
   RSpec.describe "Raw database change protection" do
     let(:journaled_class) do
-      Class.new(Delayed::Job) do
+      Class.new(ActiveRecord::Base) do
         include Journaled::Changes
 
-        journal_changes_to :locked_at, as: :attempt
+        self.table_name = 'widgets'
+
+        journal_changes_to :name, as: :attempt
       end
     end
 
     let(:journaled_class_with_no_journaled_columns) do
-      Class.new(Delayed::Job) do
+      Class.new(ActiveRecord::Base) do
         include Journaled::Changes
+
+        self.table_name = 'widgets'
       end
     end
 
     describe "the relation" do
       describe "#update_all" do
         it "refuses on journaled columns passed as hash" do
-          expect { journaled_class.update_all(locked_at: nil) }.to raise_error(/aborted by Journaled/)
+          expect { journaled_class.update_all(name: nil) }.to raise_error(/aborted by Journaled/)
         end
 
         it "refuses on journaled columns passed as string" do
-          expect { journaled_class.update_all("\"locked_at\" = NULL") }.to raise_error(/aborted by Journaled/)
-          expect { journaled_class.update_all("locked_at = null") }.to raise_error(/aborted by Journaled/)
-          expect { journaled_class.update_all("delayed_jobs.locked_at = null") }.to raise_error(/aborted by Journaled/)
-          expect { journaled_class.update_all("last_error = 'locked_at'") }.not_to raise_error
+          expect { journaled_class.update_all("\"name\" = NULL") }.to raise_error(/aborted by Journaled/)
+          expect { journaled_class.update_all("name = null") }.to raise_error(/aborted by Journaled/)
+          expect { journaled_class.update_all("widgets.name = null") }.to raise_error(/aborted by Journaled/)
+          expect { journaled_class.update_all("other_column = 'name'") }.not_to raise_error
         end
 
         it "succeeds on unjournaled columns" do
-          expect { journaled_class.update_all(handler: "") }.not_to raise_error
+          expect { journaled_class.update_all(other_column: "") }.not_to raise_error
         end
 
         it "succeeds when forced on journaled columns" do
-          expect { journaled_class.update_all({ locked_at: nil }, force: true) }.not_to raise_error
+          expect { journaled_class.update_all({ name: nil }, force: true) }.not_to raise_error
         end
       end
 
@@ -69,29 +73,19 @@ if Rails::VERSION::MAJOR > 5 || (Rails::VERSION::MAJOR == 5 && Rails::VERSION::M
     end
 
     describe "an instance" do
-      let(:job) do
-        module TestJob
-          def perform
-            "foo"
-          end
-
-          module_function :perform
-        end
-      end
-
-      subject { journaled_class.enqueue(job) }
+      subject { journaled_class.create!(name: 'foo') }
 
       describe "#update_columns" do
         it "refuses on journaled columns" do
-          expect { subject.update_columns(locked_at: nil) }.to raise_error(/aborted by Journaled/)
+          expect { subject.update_columns(name: nil) }.to raise_error(/aborted by Journaled/)
         end
 
         it "succeeds on unjournaled columns" do
-          expect { subject.update_columns(handler: "") }.not_to raise_error
+          expect { subject.update_columns(other_column: "") }.not_to raise_error
         end
 
         it "succeeds when forced on journaled columns" do
-          expect { subject.update_columns({ locked_at: nil }, force: true) }.not_to raise_error
+          expect { subject.update_columns({ name: nil }, force: true) }.not_to raise_error
         end
       end
 
@@ -101,7 +95,7 @@ if Rails::VERSION::MAJOR > 5 || (Rails::VERSION::MAJOR == 5 && Rails::VERSION::M
         end
 
         it "succeeds if no journaled columns exist" do
-          instance = journaled_class_with_no_journaled_columns.enqueue(job)
+          instance = journaled_class_with_no_journaled_columns.create!
           expect { instance.delete }.not_to raise_error
         end