jiggler 0.1.0.rc2

data/spec/jiggler/scheduled/enqueuer_spec.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+RSpec.describe Jiggler::Scheduled::Enqueuer do
+  let(:config) do 
+    Jiggler::Config.new(
+      concurrency: 1,
+      timeout: 1,
+      queues: ['default', 'mine'],
+      server_mode: true
+    )
+  end
+  let(:enqueuer) { described_class.new(config) }
+
+  describe '#push_job' do
+    it 'pushes an empty job to default queue' do
+      expect do
+        config.with_sync_redis do |conn|
+          enqueuer.push_job(conn, '{ "name": "MyJob", "jid": "001" }')
+        end
+      end.to change { 
+        config.with_sync_redis { |conn| conn.call('LLEN', 'jiggler:list:default') }
+      }.by(1)
+    end
+
+    it 'pushes job to queue from args' do
+      expect do
+        config.with_sync_redis do |conn|
+          enqueuer.push_job(
+            conn, 
+            '{ "name": "MyJob", "queue": "mine", "jid": "111" }'
+          )
+        end
+      end.to change { 
+        config.with_sync_redis { |conn| conn.call('LLEN', 'jiggler:list:mine') }
+      }.by(1)
+      config.with_sync_redis { |conn| conn.call('DEL', 'jiggler:list:mine') }
+    end
+
+    it 'does not drop job if queue is not configured' do
+      expect do
+        config.with_sync_redis do |conn|
+          enqueuer.push_job(
+            conn, 
+            '{ "name": "MyJob", "queue": "unknown", "jid": "011" }'
+          )
+        end
+      end.to change { 
+        config.with_sync_redis { |conn| conn.call('LLEN', 'jiggler:list:unknown') }
+      }.by(1)
+    end
+  end
+
+  describe '#enqueue_jobs' do
+    it 'enqueues jobs if their zrange is in the past' do
+      expect do
+        config.with_sync_redis do |conn| 
+          conn.call(
+            'ZADD',
+            config.retries_set, 
+            (Time.now.to_f - 10.0).to_s, 
+            '{ "name": "MyJob", "queue": "mine", "jid": "002" }'
+          )
+          conn.call(
+            'ZADD',
+            config.retries_set, 
+            (Time.now.to_f + 10.0).to_s,
+            '{ "name": "MyFailedJob", "queue": "mine", "jid": "012" }'
+          )
+          config.logger.debug('Enqueuer Test') { conn.call('ZRANGE', config.retries_set, -5, -1) }
+        end
+        expect(enqueuer).to receive(:push_job).at_least(:once).and_call_original
+        Sync do
+          enqueuer.enqueue_jobs
+        end
+      end.to change {
+        config.with_sync_redis { |conn| conn.call('LLEN', 'jiggler:list:mine') }
+      }.by(1)
+      config.with_sync_redis { |conn| conn.call('DEL', 'jiggler:list:mine') }
+    end
+  end
+end

data/spec/jiggler/scheduled/poller_spec.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+RSpec.describe Jiggler::Scheduled::Poller do
+  let(:config) do
+    Jiggler::Config.new(
+      concurrency: 1,
+      timeout: 1,
+      poll_interval: 1
+    )
+  end
+  let(:poller) { described_class.new(config) }
+
+  describe '#start' do
+    it 'starts the poller' do
+      task = Async do
+        expect(poller).to receive(:initial_wait)
+        expect(poller).to receive(:enqueue).at_least(:once).and_call_original
+        Async { poller.start }
+        expect(poller.instance_variable_get(:@done)).to be false
+        sleep(1)
+        poller.terminate
+      end
+      task.wait
+    end
+  end
+
+  describe '#terminate' do 
+    it 'terminates the poller' do
+      expect(poller.instance_variable_get(:@enqueuer)).to receive(:terminate).
+        and_call_original
+      task = Async do
+        Async { poller.start }
+        sleep(1)
+        poller.terminate
+        expect(poller.instance_variable_get(:@done)).to be true
+      end
+      task.wait
+    end
+  end
+end

data/spec/jiggler/stats/monitor_spec.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+RSpec.describe Jiggler::Stats::Monitor do
+  let(:config) do 
+    Jiggler::Config.new(
+      timeout: 1,
+      stats_interval: 1
+    )
+  end
+  let(:uuid) { 'monitor-test-uuid' }
+  let(:collection) { Jiggler::Stats::Collection.new(uuid) }
+  let(:monitor) { described_class.new(config, collection) }
+
+  describe '#start' do
+    it 'starts the monitor' do
+      task = Async do
+        expect(monitor).to receive(:load_data_into_redis).at_least(:once).and_call_original
+        Async { monitor.start }
+        expect(monitor.instance_variable_get(:@done)).to be false
+        sleep(1)
+        monitor.terminate
+      end
+      task.wait
+    end
+  end
+
+  describe '#terminate' do
+    it 'terminates the monitor' do
+      task = Async do
+        Async { monitor.start }
+        sleep(1)
+        expect(monitor).to receive(:cleanup).and_call_original
+        monitor.terminate
+        expect(monitor.instance_variable_get(:@done)).to be true
+        expect(config.client_redis_pool.acquire { |conn| conn.call('GET', uuid) }).to be nil
+      end
+      task.wait
+    end
+  end
+end

data/spec/jiggler/summary_spec.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+RSpec.describe Jiggler::Summary do
+  let(:queues) { ['default', 'queue1'] }
+  let(:config) do
+    Jiggler::Config.new(
+      concurrency: 1,
+      timeout: 1,
+      stats_interval: 1,
+      queues: queues,
+      poller_enabled: false
+    )
+  end
+  let(:collection) { Jiggler::Stats::Collection.new('summary-test-uuid') } 
+  let(:summary) { described_class.new(config) }
+
+  describe '.all' do
+    it 'has correct keys and data types' do
+      subject = summary.all
+      expect(subject).to be_a Hash
+      expect(subject.keys).to eq Jiggler::Summary::KEYS
+      expect(subject['retry_jobs_count']).to be_a Integer
+      expect(subject['dead_jobs_count']).to be_a Integer
+      expect(subject['scheduled_jobs_count']).to be_a Integer
+      expect(subject['processed_count']).to be_a Integer
+      expect(subject['failures_count']).to be_a Integer
+      expect(subject['monitor_enabled']).to be_a(String).or be_a(NilClass)
+      expect(subject['processes']).to be_a Hash
+      expect(subject['queues']).to be_a Hash
+    end
+
+    it 'gets latest data' do
+      Sync do
+        config.cleaner.prune_all
+        launcher = Jiggler::Launcher.new(config)
+        uuid = launcher.send(:uuid)
+        MyJob.with_options(queue: 'queue1').enqueue
+        
+        first_summary = summary.all
+        expect(first_summary['queues']).to include({
+          'queue1' => 1
+        })
+        expect(first_summary['processes'].keys).to_not include(uuid)
+        
+        second_summary = nil
+        launcher_task = Async { launcher.start }
+        stop_task = Async do
+          sleep(2)
+          second_summary = summary.all
+          launcher.stop
+        end
+        launcher_task.wait
+        stop_task.wait
+
+        expect(second_summary['queues']).to_not include({
+          'queue1' => 1
+        })
+        expect(second_summary['processes'].keys).to include(uuid)
+        expect(second_summary['processes'][uuid]).to include({
+          'queues' => queues.join(','),
+          'hostname' => Socket.gethostname,
+          'pid' => Process.pid.to_s,
+          'concurrency' => '1',
+          'timeout' => '1',
+          'poller_enabled' => false,
+          'current_jobs' => {}
+        })
+      end
+    end
+
+    context 'key dissasembly' do
+      let(:uuid) { 'jiggler:svr:2aa9:10:15:drop,bark,meow:1:1673820635:29677:dyno:1' }
+
+      it 'has correct process metadata' do
+        allow(summary).to receive(:fetch_processes).and_return([uuid])
+        subject = summary.all
+        expect(subject['processes'][uuid]).to include({
+          'name' => 'jiggler:svr:2aa9',
+          'concurrency' => '10',
+          'timeout' => '15',
+          'queues' => 'drop,bark,meow',
+          'pid' => '29677',
+          'poller_enabled' => true,
+          'started_at' => '1673820635',
+          'hostname' => 'dyno:1',
+        })
+      end
+    end
+  end
+
+  describe '#last_dead_jobs' do
+    it 'returns last n dead jobs' do
+      Sync do 
+        config.cleaner.prune_all
+        expect(summary.last_dead_jobs(1)).to be_empty
+        MyFailedJob.with_options(queue: 'queue1', retries: 0).enqueue('yay')
+      end
+      worker = Jiggler::Worker.new(config, collection) do
+        config.logger.info('Doing some weird dead testings')
+      end
+      task = Async do
+        Async do
+          worker.run
+        end
+        sleep(1)
+        worker.terminate
+      end
+      task.wait
+      jobs = Sync { summary.last_dead_jobs(3) }
+      expect(jobs.count).to be 1
+      expect(jobs.first).to include({
+        'name' => 'MyFailedJob',
+        'args' => ['yay']
+      })
+    end
+  end
+
+  describe '#last_retry_jobs' do
+    it 'returns last n retry jobs' do
+      Sync do
+        config.cleaner.prune_all
+        expect(summary.last_retry_jobs(3)).to be_empty
+        5.times do |i|
+          MyFailedJob.with_options(
+            queue: 'queue1', 
+            retries: 1
+          ).enqueue("yay-#{i}")
+        end
+      end
+      worker = Jiggler::Worker.new(config, collection) do
+        config.logger.info('Doing some weird retry testings')
+      end
+      task = Async do
+        Async do
+          worker.run
+        end
+        sleep(1)
+        worker.terminate
+      end
+      task.wait
+      jobs = Sync { summary.last_retry_jobs(3) }
+      expect(jobs.count).to be 3
+      expect(jobs.first).to include({
+        'name' => 'MyFailedJob',
+        'args' => ['yay-4']
+      })
+    end
+  end
+
+  describe '#last_scheduled_jobs' do
+    it 'returns last n scheduled jobs' do
+      Sync do 
+        config.cleaner.prune_all
+        expect(summary.last_scheduled_jobs(3)).to be_empty
+        MyFailedJob.with_options(
+          queue: 'queue1'
+        ).enqueue_in(100, 'yay-scheduled')
+        jobs = summary.last_scheduled_jobs(3)
+        expect(jobs.count).to be 1
+        expect(jobs.first).to include({
+          'name' => 'MyFailedJob',
+          'args' => ['yay-scheduled']
+        })
+        expect(jobs.first['scheduled_at']).to be_a Float
+      end
+    end
+  end
+end

data/spec/jiggler/web_spec.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+RSpec.describe Jiggler::Web do
+  let(:web) { Jiggler::Web.new }
+
+  describe '#time_ago_in_words' do
+    it 'returns the correct string for a timestamp' do
+      expect(web.time_ago_in_words(Time.now.to_i - 120)).to eq '2 minutes ago'
+    end
+
+    it 'returns nil for nil' do
+      expect(web.time_ago_in_words(nil)).to be_nil
+    end
+  end
+
+  describe '#format_memory' do
+    it 'returns the correct string for a memory size' do
+      expect(web.format_memory(1024)).to eq '1.0 MB'
+    end
+
+    it 'returns ? for nil' do
+      expect(web.format_memory(nil)).to eq '?'
+    end
+  end
+
+  describe '#outdated_heartbeat?' do
+    it 'returns true if the heartbeat is older than 2 stats intervals' do
+      expect(
+        web.outdated_heartbeat?(Time.now.to_i - Jiggler.config[:stats_interval] * 2 - 1)
+      ).to be true
+    end
+
+    it 'returns false if the heartbeat is newer than 2 stats intervals' do
+      expect(web.outdated_heartbeat?(Time.now.to_i - Jiggler.config[:stats_interval])).to be false
+    end
+  end
+end

data/spec/jiggler/worker_spec.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+RSpec.describe Jiggler::Worker do
+  before(:all) do 
+    Jiggler.instance_variable_set(:@config, Jiggler::Config.new(
+      concurrency: 1,
+      client_concurrency: 1,
+      timeout: 1,
+      queues: ['default', 'test']
+    ) )
+  end
+  after(:all) do
+    Jiggler.instance_variable_set(:@config, Jiggler::Config.new)
+  end
+
+  let(:config) { Jiggler.config }
+  let(:collection) { Jiggler::Stats::Collection.new(config) }
+  let(:worker) do 
+    described_class.new(config, collection) do
+      config.logger.debug("Callback called: #{rand(100)}")
+    end
+  end
+
+  describe '#run' do
+    it 'runs the worker and performs the job' do
+      MyJob.enqueue
+      task = Async do
+        Async do
+          expect(worker).to receive(:fetch_one).at_least(:once).and_call_original
+          expect(worker).to receive(:execute_job).at_least(:once).and_call_original
+          worker.run
+        end
+        sleep(1)
+        worker.terminate
+      end
+      task.wait
+    end
+
+    it 'runs the worker and performs the job with args' do
+      expect do
+        MyJobWithArgs.enqueue('str', 1, 2.0, true, [1, 2], { a: 1, b: 2 })
+        task = Async do
+          Async do
+            expect(worker).to receive(:fetch_one).at_least(:once).and_call_original
+            expect(worker).to receive(:execute_job).at_least(:once).and_call_original
+            worker.run
+          end
+          sleep(1)
+          worker.terminate
+        end
+        task.wait 
+      end.to_not change { 
+        config.with_sync_redis { |conn| conn.call('ZCARD', config.retries_set) }
+      }
+    end
+
+    it 'runs the worker and adds the job to retry queue' do
+      expect do
+        MyFailedJob.enqueue
+        task = Async do
+          Async do
+            expect(worker).to receive(:fetch_one).at_least(:once).and_call_original
+            expect(worker).to receive(:execute_job).at_least(:once).and_call_original
+            worker.run
+          end
+          sleep(1)
+          worker.terminate
+        end
+        task.wait
+      end.to change { 
+        config.with_sync_redis { |conn| conn.call('ZCARD', config.retries_set) }
+      }.by(1)
+      config.with_sync_redis { |conn| conn.call('DEL', 'jiggler:set:retries') }
+    end
+  end
+
+  describe '#execute_job' do
+    it 'executes the job' do
+      expect do
+        worker.instance_variable_set(
+          :@current_job, 
+          Jiggler::Worker::CurrentJob.new(queue: 'default', args: '{ "name": "MyJob", "jid": "321" }')
+        )
+        worker.send(:execute_job)
+      end.to output("Hello World\n").to_stdout
+    end
+  end
+
+  describe '#terminate' do
+    context 'when worker is running' do
+      it 'terminates the worker' do
+        task = Async do
+          expect(worker.done).to be false
+          Async { worker.run }
+          worker.terminate
+        end
+        task.wait
+        expect(worker.done).to be true
+      end
+    end
+
+    context 'when worker is not running' do
+      it do
+        expect(worker.done).to be false
+        worker.terminate
+        expect(worker.done).to be true
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require 'debug'
+
+# client
+require 'jiggler'
+require 'jiggler/web'
+
+# server
+require 'jiggler/support/helper'
+require 'jiggler/scheduled/enqueuer'
+require 'jiggler/scheduled/poller'
+require 'jiggler/stats/collection'
+require 'jiggler/stats/monitor'
+require 'jiggler/errors'
+require 'jiggler/retrier'
+require 'jiggler/launcher'
+require 'jiggler/manager'
+require 'jiggler/worker'
+require 'jiggler/cli'
+
+require_relative './fixtures/jobs'
+
+RSpec.configure do |config|
+  config.expect_with :rspec do |expectations|
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  config.mock_with :rspec do |mocks|
+    mocks.verify_partial_doubles = true
+  end
+
+  config.shared_context_metadata_behavior = :apply_to_host_groups
+
+  config.filter_run_when_matching :focus
+  config.example_status_persistence_file_path = 'spec/examples.txt'
+  config.disable_monkey_patching!
+  config.warnings = true
+
+  if config.files_to_run.one?
+    config.default_formatter = 'doc'
+  end
+
+  config.around(:each) do |example|
+    Timeout.timeout(10) do
+      example.run
+    end
+  end
+
+  config.profile_examples = 10
+  config.order = :random
+
+  Kernel.srand config.seed
+end