joblin 0.1.0

data/spec/models/background_task_spec.rb

@@ -0,0 +1,41 @@
+require 'spec_helper'
+
+RSpec.describe Joblin::BackgroundTask, type: :model do
+  let(:task) { Joblin::BackgroundTask.create!(type: 'Joblin::BackgroundTask') }
+
+  describe '#options' do
+    it 'saves changes' do
+      task.options['foo'] = 'bar'
+      task.save!
+      expect(task.reload.extra_options).to eq({ "foo" => 'bar' })
+    end
+
+    it 'stores references' do
+      task.options['ref'] = task
+      task.save!
+      task.reload
+      expect(task.options['ref']).to eq(task)
+      expect(task.extra_options.to_h).to eq({ 'ref' => task.to_gid.to_s })
+    end
+
+    it 'is indifferent' do
+      task.options[:foo] = 'bar'
+      task.save!
+      task.reload
+      expect(task.options[:foo]).to eq('bar')
+      expect(task.options['foo']).to eq('bar')
+      expect(task.options[:updated_at]).to eq(task.updated_at)
+      expect(task.options['updated_at']).to eq(task.updated_at)
+    end
+
+    it 'reads from columns' do
+      expect(task.options[:updated_at]).to eq(task.updated_at)
+    end
+
+    it 'writes to columns' do
+      dt = 3.days.ago
+      task.options[:updated_at] = dt
+      expect(task.updated_at).to eq(dt)
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,29 @@
+require 'bundler'
+
+Bundler.require :default, :development
+
+Combustion.initialize! :active_record, :active_job
+
+require 'redis'
+
+require 'rspec/rails'
+# If you're using Capybara:
+# require 'capybara/rails'
+
+require 'sidekiq'
+require 'sidekiq/rails'
+require 'sidekiq/testing'
+Sidekiq::Testing.fake!
+Sidekiq::Testing.server_middleware do |chain|
+  chain.add Joblin::Batching::Compat::Sidekiq::ServerMiddleware
+end
+
+Dir[File.dirname(__FILE__) + "/batching/support/**/*.rb"].each {|f| require f }
+
+RSpec.configure do |config|
+  config.use_transactional_fixtures = true
+end
+
+def set_batch_context(ctx)
+  allow_any_instance_of(ActiveJob::Base).to receive(:batch_context).and_return(ctx.with_indifferent_access)
+end

data/spec/uniqueness/compat/active_job_spec.rb

@@ -0,0 +1,49 @@
+require 'uniqueness/spec_helper'
+
+RSpec.describe Joblin::Uniqueness::Compat::ActiveJob do
+  context 'Job Extension' do
+    it 'includes UniqueJobExtension' do
+      expect(ActiveJob::Base < Joblin::Uniqueness::Compat::ActiveJob::JobExtension).to be true
+    end
+
+    it 'has the ensure_uniqueness method' do
+      expect(ActiveJob::Base.method(:ensure_uniqueness)).to be_present
+    end
+  end
+
+  context 'Job' do
+    let(:test_job) do
+      Class.new(ActiveJob::Base) do
+        ensure_uniqueness(
+          strategy: :until_executed,
+        )
+        def perform; end
+      end
+    end
+
+    before(:each) do
+      stub_const('TestJob', test_job)
+      ActiveJob::Base.queue_adapter = :sidekiq
+    end
+
+    it 'runs as expected' do
+      strategy = Joblin::Uniqueness::Strategy::UntilExecuted.new(nil)
+      allow_any_instance_of(Joblin::Uniqueness::LockContext).to receive(:lock_strategy) do |lock_context|
+        strategy.instance_variable_set(:@lock_context, lock_context)
+        strategy
+      end
+      allow(Joblin::Uniqueness::Strategy::UntilExecuted).to receive(:new).and_return(strategy)
+
+      expect(strategy).to receive(:on_enqueue).and_call_original
+      expect(strategy).to receive(:on_perform).and_call_original
+      expect(strategy).to receive(:batch_callback).with(:complete, anything).and_call_original
+      expect(strategy).to receive(:batch_callback).with(:success, anything).and_call_original
+
+      expect_any_instance_of(test_job).to receive(:perform)
+
+      Sidekiq::Testing.inline! do
+        test_job.perform_later
+      end
+    end
+  end
+end

data/spec/uniqueness/compat/sidekiq_spec.rb

@@ -0,0 +1,68 @@
+require 'uniqueness/spec_helper'
+
+RSpec.describe Joblin::Uniqueness::Compat::Sidekiq do
+  let(:config) { defined?(Sidekiq::Config) ? double(Sidekiq::Config) : class_double(Sidekiq) }
+  let(:client_middleware) { double(Sidekiq::Middleware::Chain) }
+
+  context 'client' do
+    it 'adds client middleware' do
+      allow(Sidekiq).to receive(:configure_client).and_yield(config)
+      expect(config).to receive(:client_middleware).and_yield(client_middleware)
+      expect(client_middleware).to receive(:insert_before).with(anything, Joblin::Uniqueness::Compat::Sidekiq::ClientMiddleware)
+      Joblin::Uniqueness::Compat::Sidekiq.configure
+    end
+  end
+
+  context 'server' do
+    let(:server_middleware) { double(Sidekiq::Middleware::Chain) }
+
+    it 'adds client and server middleware' do
+      allow(Sidekiq).to receive(:configure_server).and_yield(config)
+      expect(config).to receive(:client_middleware).and_yield(client_middleware)
+      expect(config).to receive(:server_middleware).and_yield(server_middleware)
+      expect(client_middleware).to receive(:insert_before).with(anything, Joblin::Uniqueness::Compat::Sidekiq::ClientMiddleware)
+      expect(server_middleware).to receive(:insert_after).with(anything, Joblin::Uniqueness::Compat::Sidekiq::ServerMiddleware)
+      Joblin::Uniqueness::Compat::Sidekiq.configure
+    end
+  end
+
+  context 'Job Extension' do
+    let!(:dummy_worker) do
+      Class.new do
+        include Sidekiq::Worker
+      end
+    end
+
+    it 'includes UniqueJobExtension' do
+      expect(dummy_worker < Joblin::Uniqueness::Compat::Sidekiq::WorkerExtension).to be true
+    end
+
+    it 'has the ensure_uniqueness method' do
+      expect(dummy_worker.method(:ensure_uniqueness)).to be_present
+    end
+  end
+
+  context 'Job' do
+    include_context 'with TestWorker'
+
+    it 'runs as expected' do
+      strategy = Joblin::Uniqueness::Strategy::UntilExecuted.new(nil)
+      allow_any_instance_of(Joblin::Uniqueness::LockContext).to receive(:lock_strategy) do |lock_context|
+        strategy.instance_variable_set(:@lock_context, lock_context)
+        strategy
+      end
+      allow(Joblin::Uniqueness::Strategy::UntilExecuted).to receive(:new).and_return(strategy)
+
+      expect(strategy).to receive(:on_enqueue).and_call_original
+      expect(strategy).to receive(:on_perform).and_call_original
+      expect(strategy).to receive(:batch_callback).with(:complete, anything).and_call_original
+      expect(strategy).to receive(:batch_callback).with(:success, anything).and_call_original
+
+      expect_any_instance_of(TestWorker).to receive(:perform)
+
+      Sidekiq::Testing.inline! do
+        worker.perform_async
+      end
+    end
+  end
+end

data/spec/uniqueness/lock_context_spec.rb

@@ -0,0 +1,106 @@
+require 'uniqueness/spec_helper'
+
+RSpec.describe Joblin::Uniqueness::LockContext do
+  let!(:worker) do
+    Class.new do
+      include Sidekiq::Worker
+
+      ensure_uniqueness(
+        strategy: :until_executed,
+      )
+
+      def perform; end
+    end
+  end
+
+  before(:each) { stub_const('TestWorker', worker) }
+
+  let(:lock_context) { described_class.new(context_data) }
+  let(:context_data) { { queue: 'default', job_clazz: 'TestWorker', args: [1,2,3], kwargs: { foo: 'bar' } } }
+
+  describe '#base_key' do
+    let(:scope) { :per_queue }
+    before(:each) do
+      TestWorker.unique_job_options[:scope] = scope
+    end
+
+    it 'returns matching keys for equal hashes' do
+      context_data[:args] = [{ foo: 'bar', bar: 'foo' }]
+      key1 = lock_context.base_key
+      lock_context.instance_variable_set(:@base_key, nil)
+
+      context_data[:args] = [{ bar: 'foo', foo: 'bar' }]
+      key2 = lock_context.base_key
+
+      expect(key1).to eq key2
+    end
+
+    context 'when scope is a Proc' do
+      let(:scope) { ->(queue:) { "blob" } }
+
+      it 'returns the base key' do
+        expect(lock_context.base_key).to match /^uniquejob:blob:.*/
+      end
+    end
+
+    context 'when scope is :global' do
+      let(:scope) { :global }
+
+      it 'returns the base key' do
+        expect(lock_context.base_key).to match /^uniquejob:TestWorker:.*/
+      end
+    end
+
+    context 'when scope is :per_queue' do
+      let(:scope) { :per_queue }
+
+      it 'returns the base key' do
+        expect(lock_context.base_key).to match /^uniquejob:TestWorker:default:.*/
+      end
+    end
+
+    context 'when scope is not a Proc, :global, or :per_queue' do
+      let(:scope) { 'foo' }
+
+      it 'returns the base key' do
+        expect(lock_context.base_key).to match /^uniquejob:foo:.*/
+      end
+    end
+  end
+
+  describe '#call_conflict_strategy' do
+    before(:each) do
+      worker.unique_job_options[:on_conflict] = :raise
+    end
+
+    it 'invokes the conflict strategy' do
+      expect_any_instance_of(Joblin::Uniqueness::OnConflict::Raise).to receive(:call)
+      lock_context.send(:call_conflict_strategy, :perform)
+    end
+  end
+
+  describe '#conflict_strategy_for' do
+    before(:each) do
+      worker.unique_job_options[:on_conflict] = :raise
+    end
+
+    it 'resolves a united conflict strategy' do
+      expect(lock_context.send(:conflict_strategy_for, :perform)).to be_a Joblin::Uniqueness::OnConflict::Raise
+    end
+
+    it 'resolves enqueue strategy' do
+      worker.unique_job_options[:on_conflict] = { enqueue: :log, perform: :raise }
+      expect(lock_context.send(:conflict_strategy_for, :enqueue)).to be_a Joblin::Uniqueness::OnConflict::Log
+    end
+
+    it 'resolves perform strategy' do
+      worker.unique_job_options[:on_conflict] = { enqueue: :log, perform: :reject }
+      expect(lock_context.send(:conflict_strategy_for, :perform)).to be_a Joblin::Uniqueness::OnConflict::Reject
+    end
+
+    it 'resolves null strategy' do
+      worker.unique_job_options[:on_conflict] = { enqueue: :log }
+      expect(lock_context.send(:conflict_strategy_for, :perform)).to be_a Joblin::Uniqueness::OnConflict::NullStrategy
+    end
+  end
+end

data/spec/uniqueness/on_conflict/log_spec.rb

@@ -0,0 +1,11 @@
+require 'uniqueness/spec_helper'
+
+RSpec.describe Joblin::Uniqueness::OnConflict::Log do
+  include_context "on_conflict specs"
+  include_examples "OnConflict is compatible with", %i[until_executing while_executing until_executed until_and_while_executing until_expired]
+
+  it "logs" do
+    expect(Joblin::Uniqueness.logger).to receive(:info)
+    on_conflict.call
+  end
+end

data/spec/uniqueness/on_conflict/raise_spec.rb

@@ -0,0 +1,10 @@
+require 'uniqueness/spec_helper'
+
+RSpec.describe Joblin::Uniqueness::OnConflict::Raise do
+  include_context "on_conflict specs"
+  include_examples "OnConflict is compatible with", %i[until_executing while_executing until_executed until_and_while_executing until_expired]
+
+  it "raises an error" do
+    expect { on_conflict.call }.to raise_error(Joblin::Uniqueness::Conflict)
+  end
+end

data/spec/uniqueness/on_conflict/reschedule_spec.rb

@@ -0,0 +1,63 @@
+require 'uniqueness/spec_helper'
+
+RSpec.describe Joblin::Uniqueness::OnConflict::Reschedule do
+  include_context "on_conflict specs"
+  include_examples "OnConflict is compatible with", %i[while_executing]
+
+  it "calls reenqueue" do
+    expect(lock_context).to receive(:reenqueue) do |*args, **kwargs|
+      expect(Thread.current[:unique_jobs_previous_context]).to be_present
+    end
+    on_conflict.call
+  end
+
+  it "reschedules successfully" do
+    TestWorker.unique_job_options.merge!({ strategy: :until_and_while_executing, on_conflict: { enqueue: :raise, perform: :reschedule }})
+
+    lock_context.handle_lifecycle!(:enqueue) {}
+    lock_context.handle_lifecycle!(:perform) {}
+
+    lock_context2 = Joblin::Uniqueness::Compat::Sidekiq::SidekiqLockContext.new({ jid: 'j2', queue: 'default', job_clazz: 'TestWorker'}, job_instance: {})
+    lock_context2.handle_lifecycle!(:enqueue) {}
+    lock_context2.handle_lifecycle!(:perform) {}
+  end
+
+  it "maintains a consistent lock_id" do
+    TestWorker.clear
+    TestWorker.unique_job_options.merge!({ strategy: :while_executing, on_conflict: :reschedule })
+
+    lock_context.handle_lifecycle!(:enqueue) {}
+    lock_context.handle_lifecycle!(:perform) {}
+
+    TestWorker.perform_async
+    j1 = TestWorker.jobs[0]
+    TestWorker.perform_one
+    j2 = TestWorker.jobs[0]
+
+    expect(j1).not_to eq(j2)
+    expect(j1['uniqueness_cache_data']['lid']).to be_present
+    expect(j1['uniqueness_cache_data']['lid']).to eq(j2['uniqueness_cache_data']['lid'])
+  end
+
+  it "unlocks after rescheduling" do
+    TestWorker.clear
+    TestWorker.unique_job_options.merge!({ strategy: :while_executing, on_conflict: :reschedule })
+
+    lock_context.handle_lifecycle!(:enqueue) {}
+    lock_context.handle_lifecycle!(:perform) {}
+
+    TestWorker.perform_async
+    expect(TestWorker.jobs.size).to eq(1)
+    TestWorker.perform_one
+    expect(TestWorker.jobs.size).to eq(1)
+
+    lock_context.lock_strategy.send(:unlock)
+    Sidekiq::Worker.drain_all
+    expect(TestWorker.jobs.size).to eq(0)
+
+    # Run another worker to validate that the lock was released
+    TestWorker.perform_async
+    TestWorker.perform_one
+    expect(TestWorker.jobs.size).to eq(0)
+  end
+end

data/spec/uniqueness/on_conflict_spec.rb

@@ -0,0 +1,16 @@
+require 'uniqueness/spec_helper'
+
+RSpec.describe Joblin::Uniqueness::OnConflict do
+
+  describe ".validate!" do
+    it "passes a valid pair" do
+      described_class.validate!(:reject, :while_executing)
+    end
+
+    it "raises an error if the strategy is not valid for the given action" do
+      expect do
+        described_class.validate!(:reject, :until_executed)
+      end.to raise_error(ArgumentError)
+    end
+  end
+end

data/spec/uniqueness/spec_helper.rb

@@ -0,0 +1,19 @@
+require_relative "../spec_helper"
+require "joblin/uniqueness/job_uniqueness"
+
+require 'sidekiq/testing'
+
+require 'redis'
+Redis.silence_deprecations = true
+
+Dir[File.join(File.dirname(__FILE__), "./support/**/*.rb")].sort.each { |f| require f }
+
+Sidekiq::Testing.server_middleware do |chain|
+  chain.insert_after Joblin::Batching::Compat::Sidekiq::ServerMiddleware, Joblin::Uniqueness::Compat::Sidekiq::ServerMiddleware
+end
+
+RSpec.configure do |config|
+  config.before(:each) do
+    Joblin.redis.flushdb
+  end
+end

data/spec/uniqueness/strategy/base_spec.rb

@@ -0,0 +1,100 @@
+require 'uniqueness/spec_helper'
+
+RSpec.describe Joblin::Uniqueness::Strategy::Base do
+  include_context 'with TestWorker'
+
+  let!(:jid) { worker.perform_async }
+  let(:context) { Joblin::Uniqueness::LockContext.new({ jid: jid, queue: 'default', job_clazz: 'TestWorker' }) }
+  let(:strategy) { Joblin::Uniqueness::Strategy::UntilExecuted.new(context) }
+
+  describe '#wrap_in_batch' do
+    it 'does not wrap if re-enqueuing' do
+      Thread.current[:unique_jobs_previous_context] = context
+      expect(Joblin::Batching::Batch).not_to receive(:new)
+      strategy.send(:wrap_in_batch, &->{ })
+    ensure
+      Thread.current[:unique_jobs_previous_context] = nil
+    end
+
+    context 'when the job fails' do
+      before(:each) do
+        TestWorker.unique_job_options[:strategy] = :until_executed
+        allow_any_instance_of(worker).to receive(:perform) do |w|
+          @bid = w.bid
+          raise 'error'
+        end
+      end
+
+      context 'unlock_on_failure: :any' do
+        before(:each) { TestWorker.unique_job_options[:unlock_on_failure] = :any }
+
+        it 'unlocks on any failure' do
+          expect(strategy).to be_locked
+          expect{ Sidekiq::Worker.drain_all }.to raise_error('error')
+          Sidekiq::Worker.drain_all
+          expect(strategy).not_to be_locked
+        end
+      end
+
+      context 'unlock_on_failure: :death' do
+        before(:each) { TestWorker.unique_job_options[:unlock_on_failure] = :death }
+
+        it 'does not unlock on any failure' do
+          expect(strategy).to be_locked
+          expect{ Sidekiq::Worker.drain_all }.to raise_error('error')
+          Sidekiq::Worker.drain_all
+          expect(strategy).to be_locked
+        end
+
+        it 'it unlocks on death' do
+          expect{ Sidekiq::Worker.drain_all }.to raise_error('error')
+          Sidekiq::Worker.drain_all
+          expect(strategy).to be_locked
+          Joblin::Batching::Batch.process_dead_job(@bid, jid)
+          Sidekiq::Worker.drain_all
+          expect(strategy).not_to be_locked
+        end
+      end
+
+      context 'unlock_on_failure: :stagnant' do
+        before(:each) { TestWorker.unique_job_options[:unlock_on_failure] = :stagnant }
+
+        it 'does not unlock on any failure' do
+          expect(strategy).to be_locked
+          expect{ Sidekiq::Worker.drain_all }.to raise_error('error')
+          Sidekiq::Worker.drain_all
+          expect(strategy).to be_locked
+        end
+
+        it 'it unlocks on stagnation' do
+          expect{ Sidekiq::Worker.drain_all }.to raise_error('error')
+          Sidekiq::Worker.drain_all
+          expect(strategy).to be_locked
+          Joblin::Batching::Batch.process_dead_job(@bid, jid)
+          Sidekiq::Worker.drain_all
+          expect(strategy).not_to be_locked
+        end
+      end
+
+      context 'unlock_on_failure: :expire' do
+        before(:each) { TestWorker.unique_job_options[:unlock_on_failure] = :expire }
+
+        it 'does not unlock on any failure' do
+          expect(strategy).to be_locked
+          expect{ Sidekiq::Worker.drain_all }.to raise_error('error')
+          Sidekiq::Worker.drain_all
+          expect(strategy).to be_locked
+        end
+
+        it 'does not unlock on death' do
+          expect{ Sidekiq::Worker.drain_all }.to raise_error('error')
+          Sidekiq::Worker.drain_all
+          expect(strategy).to be_locked
+          Joblin::Batching::Batch.process_dead_job(@bid, jid)
+          Sidekiq::Worker.drain_all
+          expect(strategy).to be_locked
+        end
+      end
+    end
+  end
+end

data/spec/uniqueness/strategy/until_and_while_executing_spec.rb

@@ -0,0 +1,48 @@
+require 'uniqueness/spec_helper'
+
+RSpec.describe Joblin::Uniqueness::Strategy::UntilAndWhileExecuting do
+  include_context 'lock strategy specs', strategy: :until_and_while_executing
+
+  include_examples "a lock implementation"
+
+  describe "#on_enqueue" do
+    it "locks the queue" do
+      process_one.on_enqueue {}
+      expect(process_one).to be_locked
+    end
+
+    it "does not lock the runtime lock" do
+      process_one.on_enqueue {}
+      expect(process_one.send(:runtime_lock)).to_not be_locked
+    end
+  end
+
+  describe "#on_perform" do
+    it "unlocks the queue" do
+      process_one.on_perform do
+        expect(process_one).to_not be_locked
+      end
+    end
+
+    it "locks the runtime lock" do
+      process_one.on_perform do
+        expect(process_one.send(:runtime_lock)).to be_locked
+      end
+    end
+
+    it "unlocks the runtime lock after processing" do
+      process_one.on_perform do
+        process_one.send(:runtime_lock).batch_callback(:success, nil)
+        expect(process_one.send(:runtime_lock)).to_not be_locked
+      end
+    end
+
+    it "prevents other processes from executing" do
+      process_one.on_perform do
+        expect do
+          process_two.on_perform { }
+        end.to raise_error(Joblin::Uniqueness::CouldNotLockError)
+      end
+    end
+  end
+end

data/spec/uniqueness/strategy/until_executed_spec.rb

@@ -0,0 +1,23 @@
+require 'uniqueness/spec_helper'
+
+RSpec.describe Joblin::Uniqueness::Strategy::UntilExecuted do
+  include_context 'lock strategy specs', strategy: :until_executed
+
+  include_examples "a lock implementation"
+
+  describe "#on_enqueue" do
+    it "locks the queue" do
+      process_one.on_enqueue {}
+      expect(process_one).to be_locked
+    end
+  end
+
+  describe "#on_perform" do
+    it "unlocks after processing" do
+      process_one.on_enqueue {}
+      process_one.on_perform {}
+      process_one.batch_callback(:success, nil)
+      expect(process_one).to_not be_locked
+    end
+  end
+end

data/spec/uniqueness/strategy/until_executing_spec.rb

@@ -0,0 +1,23 @@
+require 'uniqueness/spec_helper'
+
+RSpec.describe Joblin::Uniqueness::Strategy::UntilExecuting do
+  include_context 'lock strategy specs', strategy: :until_executing
+
+  include_examples "a lock implementation"
+
+  describe "#on_enqueue" do
+    it "locks the queue" do
+      process_one.on_enqueue {}
+      expect(process_one).to be_locked
+    end
+  end
+
+  describe "#on_perform" do
+    it "unlocks before processing" do
+      process_one.on_enqueue {}
+      process_one.on_perform do
+        expect(process_one).to_not be_locked
+      end
+    end
+  end
+end

data/spec/uniqueness/strategy/until_expired_spec.rb

@@ -0,0 +1,23 @@
+require 'uniqueness/spec_helper'
+
+RSpec.describe Joblin::Uniqueness::Strategy::UntilExpired do
+  include_context 'lock strategy specs', strategy: :until_expired
+
+  # include_examples "a lock implementation"
+
+  describe "#on_enqueue" do
+    it "locks the queue" do
+      process_one.on_enqueue {}
+      expect(process_one).to be_locked
+    end
+  end
+
+  describe "#on_perform" do
+    it "does not unlock after processing" do
+      process_one.on_enqueue {}
+      process_one.on_perform {}
+      process_one.batch_callback(:success, nil)
+      expect(process_one).to be_locked
+    end
+  end
+end

data/spec/uniqueness/strategy/while_executing_spec.rb

@@ -0,0 +1,33 @@
+require 'uniqueness/spec_helper'
+
+RSpec.describe Joblin::Uniqueness::Strategy::WhileExecuting do
+  include_context 'lock strategy specs', strategy: :while_executing
+
+  include_examples "a lock implementation"
+
+  describe "#on_enqueue" do
+    it "does not lock jobs" do
+      expect(process_one.on_enqueue {})
+      expect(process_one).not_to be_locked
+
+      expect(process_two.on_enqueue {})
+      expect(process_two).not_to be_locked
+    end
+  end
+
+  describe "#on_perform" do
+    it "locks the process" do
+      process_one.on_perform do
+        expect(process_one).to be_locked
+      end
+    end
+
+    it "prevents other processes from executing" do
+      process_one.on_perform do
+        expect do
+          process_two.on_perform { }
+        end.to raise_error(Joblin::Uniqueness::CouldNotLockError)
+      end
+    end
+  end
+end