jiggler 0.1.0.rc2

data/spec/fixtures/jobs.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require_relative './my_job'
+require_relative './my_failed_job'
+require_relative './my_job_with_args'

data/spec/fixtures/my_failed_job.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+class MyFailedJob
+  include Jiggler::Job
+  job_options queue: 'test', retries: 3
+
+  def perform(name = 'Yo')
+    raise StandardError, 'Oh no!'
+  end
+end

data/spec/fixtures/my_job.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class MyJob
+  include Jiggler::Job
+
+  def perform
+    puts 'Hello World'
+  end
+end

data/spec/fixtures/my_job_with_args.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class MyJobWithArgs
+  include Jiggler::Job
+  job_options retries: 2
+
+  def perform(str, int, float, bool, array, hash)
+    if !(str.is_a?(String) &&
+      int.is_a?(Integer) &&
+      float.is_a?(Float) &&
+      bool.is_a?(TrueClass) &&
+      array.is_a?(Array) &&
+      hash.is_a?(Hash))
+
+      raise StandardError, 'Args are not the correct type ◉ ︵ ◉'
+    end
+  end
+end

data/spec/jiggler/cleaner_spec.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+RSpec.describe Jiggler::Cleaner do
+  let(:config) do
+    Jiggler::Config.new(
+      concurrency: 1,
+      timeout: 1,
+      poller_enabled: false
+    )
+  end
+  let(:cleaner) { described_class.new(config) }
+  after(:all) { Jiggler.config.cleaner.prune_all }
+
+  describe '#prune_failures_counter' do
+    it 'prunes the failures counter' do
+      config.client_redis_pool.acquire do |conn|
+        conn.call('SET', config.failures_counter, 5)
+      end
+      cleaner.prune_failures_counter
+      config.client_redis_pool.acquire do |conn|
+        expect(conn.call('GET', config.failures_counter)).to be nil
+      end
+    end
+  end
+
+  describe '#prune_processed_counter' do
+    it 'prunes the processed counter' do
+      config.client_redis_pool.acquire do |conn|
+        conn.call('SET', config.processed_counter, 5)
+      end
+      cleaner.prune_processed_counter
+      config.client_redis_pool.acquire do |conn|
+        expect(conn.call('GET', config.processed_counter)).to be nil
+      end
+    end
+  end
+
+  describe '#prune_all_processes' do
+    it 'prunes all processes' do
+      config.client_redis_pool.acquire do |conn|
+        conn.call(
+          'SET',
+          'jiggler:svr:uuid-cleaner-test-0:process', 
+          '{}',
+          ex: 10
+        )
+      end
+      cleaner.prune_all_processes
+      config.client_redis_pool.acquire do |conn|
+        expect(conn.call('SCAN', '0', 'MATCH', config.process_scan_key).last).to be_empty
+      end
+    end
+  end
+
+  describe '#prune_process' do
+    let(:uuid) { 'jiggler:svr:uuid-cleaner-test-1:process' }
+    it 'prunes a process by its uuid' do
+      config.client_redis_pool.acquire do |conn|
+        conn.call(
+          'SET',
+          uuid,
+          '{}',
+          ex: 10
+        )
+      end
+      cleaner.prune_process(uuid: uuid)
+      config.client_redis_pool.acquire do |conn|
+        expect(
+          conn.call('GET', uuid)
+        ).to be nil
+      end
+    end
+  end
+
+  describe '#prune_process_by_hex' do
+    let(:uuid) { 'jiggler:svr:uuid-cleaner-test-2:process' }
+    it 'prunes a process by its hex' do
+      config.client_redis_pool.acquire do |conn|
+        conn.call(
+          'SET',
+          uuid,
+          '{}',
+          ex: 10
+        )
+      end
+      cleaner.prune_process_by_hex(hex: 'jiggler:svr:uuid-cleaner-test-2')
+      config.client_redis_pool.acquire do |conn|
+        expect(
+          conn.call('GET', uuid)
+        ).to be nil
+      end
+    end
+  end
+
+  describe '#prune_dead_set' do
+    it 'prunes the dead set' do
+      config.client_redis_pool.acquire do |conn|
+        conn.call('ZADD', config.dead_set, 1, '{}')
+      end
+      cleaner.prune_dead_set
+      config.client_redis_pool.acquire do |conn|
+        expect(conn.call('ZCARD', config.dead_set)).to be 0
+      end
+    end
+  end
+
+  describe '#prune_retries_set' do
+    it 'prunes the retries set' do
+      config.client_redis_pool.acquire do |conn|
+        conn.call('ZADD', config.retries_set, 1, '{}')
+      end
+      cleaner.prune_retries_set
+      config.client_redis_pool.acquire do |conn|
+        expect(conn.call('ZCARD', config.retries_set)).to be 0
+      end
+    end
+  end
+
+  describe '#prune_scheduled_set' do
+    it 'prunes the scheduled set' do
+      config.client_redis_pool.acquire do |conn|
+        conn.call('ZADD', config.scheduled_set, 1, '{}')
+      end
+      cleaner.prune_scheduled_set
+      config.client_redis_pool.acquire do |conn|
+        expect(conn.call('ZCARD', config.scheduled_set)).to be 0
+      end
+    end
+  end
+
+  describe '#prune_all_queues' do
+    it 'prunes all queues' do
+      config.client_redis_pool.acquire do |conn|
+        conn.call('LPUSH', 'jiggler:list:cleaner-test', '{}')
+      end
+      cleaner.prune_all_queues
+      config.client_redis_pool.acquire do |conn|
+        expect(conn.call('LRANGE', 'jiggler:list:cleaner-test-0', 0, -1)).to be_empty
+      end
+    end
+  end
+
+  describe '#prune_queue' do
+    it 'prunes a queue by its name' do
+      config.client_redis_pool.acquire do |conn|
+        conn.call('LPUSH', 'jiggler:list:cleaner-test-1', '{}')
+        conn.call('LPUSH', 'jiggler:list:cleaner-test-2', '{}')
+      end
+      cleaner.prune_queue(name: 'cleaner-test-1')
+      config.client_redis_pool.acquire do |conn|
+        expect(conn.call('LRANGE', 'jiggler:list:cleaner-test-1', 0, -1)).to be_empty
+        expect(conn.call('LRANGE', 'jiggler:list:cleaner-test-2', 0, -1)).to eq(['{}'])
+      end
+    end
+  end
+
+  describe '#prune_all' do
+    it 'prunes all data' do
+      cleaner.prune_all
+      config.client_redis_pool.acquire do |conn|
+        expect(conn.call('SCAN', '0', 'MATCH', config.process_scan_key).last).to be_empty
+        expect(conn.call('SMEMBERS', config.dead_set)).to be_empty
+        expect(conn.call('SMEMBERS', config.retries_set)).to be_empty
+        expect(conn.call('SMEMBERS', config.scheduled_set)).to be_empty
+        expect(conn.call('KEYS', "#{config.queue_prefix}*")).to be_empty
+        expect(conn.call('GET', config.failures_counter)).to be nil
+        expect(conn.call('GET', config.processed_counter)).to be nil      
+      end
+    end
+  end
+end

data/spec/jiggler/cli_spec.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require_relative '../../lib/jiggler/cli.rb'
+
+RSpec.describe Jiggler::CLI do
+  let(:cli) { Jiggler::CLI.instance }
+
+  describe '.parse_and_init' do
+    context 'with no args' do
+      it 'uses default config' do
+        cli.parse_and_init
+        expect(cli.config[:concurrency]).to be 10
+        expect(cli.config[:queues]).to eq ['default']
+        expect(cli.config[:require]).to be nil
+        expect(cli.config[:timeout]).to be 25
+      end
+    end
+
+    context 'with args' do
+      before { cli.instance_variable_set(:@config, Jiggler::Config.new(verbose: true)) }
+      after { cli.instance_variable_set(:@config, Jiggler::Config.new) }
+      let(:path) { './spec/fixtures/config/jiggler.yml' }
+
+      it 'fetches config file' do
+        cli.parse_and_init(['-C', path])
+        expect(cli.config[:config_file]).to eq(path)
+        expect(cli.config[:concurrency]).to be 1
+        expect(cli.config[:queues]).to eq(['users', 'blep'])
+      end
+
+      it 'fetches concurrency' do
+        cli.parse_and_init(['-c', '11'])
+        expect(cli.config[:concurrency]).to be 11
+      end
+
+      it 'fetches queues' do  
+        cli.parse_and_init(['-q', 'test,qwerty'])
+        expect(cli.config[:queues]).to eq(['test', 'qwerty'])
+      end
+
+      it 'fetches require' do
+        cli.parse_and_init(['-r', path])
+        expect(cli.config[:require]).to eq(path)
+      end
+
+      it 'fetches timeout' do
+        cli.parse_and_init(['-t', '10'])
+        expect(cli.config[:timeout]).to be 10
+      end
+
+      it 'fetches verbose' do 
+        cli.parse_and_init(['-v'])
+        expect(cli.config[:verbose]).to be true
+      end
+
+      it 'fetches version & exits' do
+        expect do
+          expect { cli.parse_and_init(['-V']) }.to output('Jiggler #{Jiggler::VERSION}').to_stdout
+        end.to raise_error(SystemExit)
+      end
+
+      it 'fetches help & exits' do
+        expect do
+          expect { cli.parse_and_init(['-h']) }.to output.to_stdout
+        end.to raise_error(SystemExit)
+      end
+
+      it 'fetches environment' do
+        cli.parse_and_init(['-e', 'test'])
+        expect(cli.config[:environment]).to eq('test')
+      end
+    end
+
+    context 'with invalid args' do
+      after { cli.instance_variable_set(:@config, Jiggler::Config.new) }
+
+      it { expect { cli.parse_and_init(['-c', 'invalid']) }.to raise_error(ArgumentError) }
+      it { expect { cli.parse_and_init(['-c', '-1']) }.to raise_error(ArgumentError) }
+      it { expect { cli.parse_and_init(['-t', 'yo']) }.to raise_error(ArgumentError) }
+      it do 
+        cli.parse_and_init(['-r', 'test.rb'])
+        expect { cli.send(:load_app) }.to raise_error(SystemExit) 
+      end
+      it { expect { cli.parse_and_init(['-q', 'in:va:lid']) }.to raise_error(ArgumentError) }
+    end
+  end
+end

data/spec/jiggler/config_spec.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+RSpec.describe Jiggler::Config do
+  let(:config) do
+    Jiggler::Config.new(
+      concurrency: 1, 
+      client_concurrency: 1,
+      environment: 'test',
+      timeout: 1, 
+      verbose: true,
+      queues: ['test', 'test2'],
+      require: 'test.rb',
+      max_dead_jobs: 100,
+      dead_timeout: 100,
+      redis_url: 'redis://localhost:6379'
+    ) 
+  end
+
+  describe '#initialize' do
+    it 'sets correct attrs' do
+      expect(config[:concurrency]).to be 1
+      expect(config[:environment]).to eq 'test'
+      expect(config[:timeout]).to be 1
+      expect(config[:verbose]).to be true
+      expect(config[:queues]).to eq ['test', 'test2']
+      expect(config[:require]).to eq 'test.rb'
+      expect(config[:max_dead_jobs]).to be 100
+      expect(config[:dead_timeout]).to be 100
+      expect(config[:stats_interval]).to be 10
+      expect(config[:poller_enabled]).to be true
+      expect(config[:poll_interval]).to be 5
+      expect(config[:client_async]).to be false
+    end
+
+    it 'generates prefixed queues' do
+      expect(config.prefixed_queues).to eq ['jiggler:list:test', 'jiggler:list:test2']
+    end
+
+    it 'gets redis options for server' do
+      expect(config.redis_options).to eq(
+        concurrency: 4,
+        async: true,
+        redis_url: 'redis://localhost:6379'
+      )
+    end
+
+    it 'gets redis options for client' do
+      expect(config.client_redis_options).to eq(
+        concurrency: 1,
+        async: false,
+        client_redis_pool: nil,
+        redis_url: 'redis://localhost:6379'
+      )
+    end
+  end
+end

data/spec/jiggler/core_spec.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+RSpec.describe 'Core' do
+  describe '.configure' do
+    it 'applies the configuration' do
+      Jiggler.configure do |config|
+        config[:concurrency] = 1
+        config[:client_concurrency] = 2
+        config[:timeout] = 3
+        config[:stats_interval] = 4
+        config[:max_dead_jobs] = 5
+        config[:dead_timeout] = 6
+        config[:poll_interval] = 7
+        config[:poller_enabled] = false
+        config[:client_async] = true
+        config[:queues] = %w[foo bar]
+        config[:require] = 'foo'
+        config[:environment] = 'bar'
+      end
+      expect(Jiggler.config[:concurrency]).to be 1
+      expect(Jiggler.config[:client_concurrency]).to be 2
+      expect(Jiggler.config[:timeout]).to be 3
+      expect(Jiggler.config[:stats_interval]).to be 4
+      expect(Jiggler.config[:max_dead_jobs]).to be 5
+      expect(Jiggler.config[:dead_timeout]).to be 6
+      expect(Jiggler.config[:poll_interval]).to be 7
+      expect(Jiggler.config[:poller_enabled]).to be false
+      expect(Jiggler.config[:client_async]).to be true
+      expect(Jiggler.config[:queues]).to eq %w[foo bar]
+      expect(Jiggler.config[:require]).to eq 'foo'
+      expect(Jiggler.config[:environment]).to eq 'bar'
+    end
+  end
+end

data/spec/jiggler/job_spec.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+RSpec.describe Jiggler::Job do
+  before(:all) do 
+    Jiggler.instance_variable_set(:@config, Jiggler::Config.new(
+      concurrency: 1,
+      client_concurrency: 1,
+      timeout: 1,
+      poller_enabled: false
+    ))
+  end
+  after(:all) do
+    Jiggler.instance_variable_set(:@config, Jiggler::Config.new)
+  end
+
+  describe '.job_options' do
+    context 'on default' do
+      let(:job) { MyJob.new }
+
+      before do
+        MyJob.job_options # reset to default
+      end
+
+      it 'has correct attrs' do
+        expect(job.class.queue).to eq 'default'
+        expect(job.class.retries).to be 0
+        expect(job.class.retry_queue).to eq 'default'
+        expect(job.class.name).to eq 'MyJob'
+      end
+    end
+
+    context 'with custom options' do
+      let(:job) { MyJob.new }
+
+      before do
+        MyJob.job_options(queue: 'custom', retries: 3, retry_queue: 'custom_retry')
+      end
+
+      it 'has correct attrs' do
+        expect(job.class.queue).to eq 'custom'
+        expect(job.class.retries).to be 3
+        expect(job.class.retry_queue).to eq 'custom_retry'
+        expect(job.class.name).to eq 'MyJob'
+      end
+    end
+  end
+
+  describe '.with_options' do
+    it 'allows to override options on job level' do
+      expect { MyJob.with_options(queue: 'woo').enqueue }.to change { 
+        Jiggler.config.client_redis_pool.acquire { |conn| conn.call('LLEN', 'jiggler:list:woo') }
+      }.by(1)
+      Jiggler.config.client_redis_pool.acquire { |conn| conn.call('DEL', 'jiggler:list:woo') }
+    end
+  end
+
+  describe '.enqueue' do
+    it 'adds the job to the queue' do
+      expect { MyJob.with_options(queue: 'mine').enqueue }.to change { 
+        Jiggler.config.client_redis_pool.acquire { |conn| conn.call('LLEN', 'jiggler:list:mine') }
+      }.by(1)
+      Jiggler.config.client_redis_pool.acquire { |conn| conn.call('DEL', 'jiggler:list:mine') }
+    end
+  end
+
+  describe '.enqueue_in' do
+    it 'adds the job to the scheduled set' do
+      expect { MyJob.with_options(queue: 'mine').enqueue_in(1) }.to change { 
+        Jiggler.config.client_redis_pool.acquire do |conn| 
+          conn.call('ZCARD', Jiggler.config.scheduled_set) 
+        end
+      }.by(1)
+    end
+
+    it 'supports multiple args' do
+      expect { MyJobWithArgs.with_options(queue: 'mine').enqueue_in(1, '1', 1, 1.0, true, [1], { '1' => 1 }) }.to change { 
+        Jiggler.config.client_redis_pool.acquire do |conn| 
+          conn.call('ZCARD', Jiggler.config.scheduled_set) 
+        end
+      }.by(1)
+    end
+  end
+
+  describe '#enqueue_bulk' do
+    let(:args_arr) do
+      [
+        ['1', 1, 1.0, true, [1], { '1' => 1 }],
+        ['2', 2, 2.0, true, [2], { '2' => 2 }],
+        ['3', 3, 3.0, true, [3], { '3' => 3 }]
+      ]
+    end
+
+    it 'adds the jobs to the queue' do
+      expect { MyJobWithArgs.enqueue_bulk(args_arr) }.to change { 
+        Jiggler.config.client_redis_pool.acquire { |conn| conn.call('LLEN', 'jiggler:list:default') }
+      }.by(3)
+    end
+  end
+end

data/spec/jiggler/launcher_spec.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+RSpec.describe Jiggler::Launcher do
+  let(:config) do
+    Jiggler::Config.new(
+      concurrency: 1,
+      timeout: 1,
+      server_mode: true
+    )
+    end
+  let(:launcher) { described_class.new(config) }
+
+  describe '#initialize' do
+    it 'sets correct attrs' do
+      expect(launcher.config).to eq config
+    end
+  end
+
+  describe '#start' do
+    it 'starts the launcher' do
+      task = Async do
+        Async do
+          expect(launcher.send(:manager)).to receive(:start).and_call_original
+          expect(launcher.send(:poller)).to receive(:start).and_call_original
+          expect(launcher.send(:monitor)).to receive(:start).and_call_original
+          launcher.start
+        end
+        sleep(1)
+        launcher.stop
+      end
+      task.wait
+    end
+  end
+
+  describe '#suspend' do
+    it 'suspends the launcher' do
+      task = Async do
+        Async do
+          expect(launcher.send(:manager)).to receive(:suspend).and_call_original
+          expect(launcher.send(:poller)).to receive(:terminate).and_call_original
+          expect(launcher.send(:monitor)).to receive(:terminate).and_call_original
+          launcher.start
+        end
+        sleep(1)
+        launcher.suspend
+      end
+      task.wait
+    end
+  end
+
+  describe '#stop' do
+    it 'stops the launcher' do
+      task = Async do
+        Async do
+          expect(launcher.send(:manager)).to receive(:terminate).and_call_original
+          expect(launcher.send(:poller)).to receive(:terminate).and_call_original
+          expect(launcher.send(:monitor)).to receive(:terminate).and_call_original
+          launcher.start
+        end
+        sleep(1)
+        launcher.stop
+      end
+      task.wait
+    end
+  end
+end

data/spec/jiggler/manager_spec.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+RSpec.describe Jiggler::Manager do
+  let(:config) do 
+    Jiggler::Config.new(
+      concurrency: 4,
+      timeout: 1,
+      server_mode: true
+    )
+  end
+  let(:collection) { Jiggler::Stats::Collection.new(config) }
+  let(:manager) { described_class.new(config, collection) }
+
+  it { expect(manager.instance_variable_get(:@workers).count).to be 4 }
+
+  describe '#start' do
+    it 'starts the manager' do
+      expect(manager.instance_variable_get(:@workers)).to all(receive(:run))
+      Async { manager.start }
+      sleep(0.5)
+      manager.terminate
+    end
+  end
+
+  describe '#suspend' do
+    it 'suspends the manager' do
+      expect(manager.instance_variable_get(:@workers)).to all(receive(:suspend).and_call_original)
+      task = Async do
+        Async { manager.start }
+        sleep(0.5)
+        manager.suspend
+        expect(manager.instance_variable_get(:@done)).to be true
+      end
+      task.wait
+    end
+  end
+
+  describe '#terminate' do
+    it 'terminates the manager' do
+      expect(manager.instance_variable_get(:@workers)).to all(receive(:suspend).and_call_original)
+      expect(manager.instance_variable_get(:@workers)).to all(receive(:terminate).and_call_original)
+      task = Async do
+        Async { manager.start }
+        sleep(0.5)
+        manager.terminate
+        sleep(2) # wait for timeout
+        expect(manager.instance_variable_get(:@done)).to be true
+      end
+      task.wait
+    end
+  end
+end

data/spec/jiggler/redis_store_spec.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+RSpec.describe Jiggler::RedisStore do
+  describe '#pool' do
+    let(:options) { { concurrency: 4, async: true } }
+    let(:redis_store) { described_class.new(options) }
+
+    it 'returns an async pool' do
+      expect(redis_store.pool).to be_a Async::Pool::Controller
+    end
+
+    context 'when connection is sync' do
+      let(:options) { { concurrency: 4, async: false } }
+
+      it 'returns a sync pool' do
+        expect(redis_store.pool).to be_a RedisClient::Pooled
+      end
+    end
+  end
+end

data/spec/jiggler/retrier_spec.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+RSpec.describe Jiggler::Retrier do
+  let(:config) do
+    Jiggler::Config.new(
+      concurrency: 1,
+      timeout: 1,
+      queues: ['test']
+    ) 
+  end
+  let(:collection) { Jiggler::Stats::Collection.new('test-retrier-uuid') }
+  let(:retrier) { Jiggler::Retrier.new(config, collection) }
+
+  describe '#wrapped' do
+    context 'failed jobs' do
+      let(:job) { MyFailedJob.new }
+
+      it 'increments attempt if mex retries are not reached' do
+        msg = { 'jid' => '1', 'retries' => 1 }
+        Sync { config.cleaner.prune_failures_counter }
+        expect(config.logger).to receive(:error).twice
+        retrier.wrapped(job, msg, 'test') do
+          job.perform
+        end
+        expect(msg['attempt']).to be 1
+        expect(msg['error_message']).to eq 'Oh no!'
+        expect(msg['error_class']).to eq 'StandardError'
+        expect(collection.data[:failures]).to be 1
+      end
+
+      it 'does not increment attempt if max retries are reached' do
+        msg = { 'attempt' => 3, 'jid' => '123', 'name' => 'MyFailedJob' }
+        Sync { config.cleaner.prune_failures_counter }
+        expect(config.logger).to receive(:error).twice
+        retrier.wrapped(job, msg, 'test') do
+          job.perform
+        end
+
+        expect(msg['attempt']).to be 3
+        expect(msg['error_message']).to eq 'Oh no!'
+        expect(msg['error_class']).to eq 'StandardError'
+        expect(collection.data[:failures]).to be 1
+      end
+    end
+
+    context 'successful jobs' do
+      it 'does not throw any exceptions' do
+        msg = { 'jid' => '2' }
+        expect(config.logger).to_not receive(:error)
+        retrier.wrapped(MyJob.new, msg, 'default') { 'success' }
+        expect(collection.data[:failures]).to be 0
+      end
+    end
+  end
+end