jiggler 0.1.0.rc4 → 0.1.0.rc6

data/spec/jiggler/manager_spec.rb

@@ -1,52 +1,61 @@
 # frozen_string_literal: true
 
 RSpec.describe Jiggler::Manager do
-  let(:config) do 
-    Jiggler::Config.new(
-      concurrency: 4,
-      timeout: 1,
-      server_mode: true
-    )
-  end
-  let(:collection) { Jiggler::Stats::Collection.new(config) }
-  let(:manager) { described_class.new(config, collection) }
+  let(:uuid) { "#{SecureRandom.hex(3)}-test" }
+  let(:collection) { Jiggler::Stats::Collection.new(uuid, uuid) }
 
-  it { expect(manager.instance_variable_get(:@workers).count).to be 4 }
+  [:at_most_once, :at_least_once].each do |mode|
+    context "with #{mode} mode" do
+      let(:config) do 
+        Jiggler::Config.new(
+          concurrency: 4,
+          timeout: 1,
+          mode: mode
+        )
+      end
+      let(:manager) { described_class.new(config, collection) }
 
-  describe '#start' do
-    it 'starts the manager' do
-      expect(manager.instance_variable_get(:@workers)).to all(receive(:run))
-      Async { manager.start }
-      sleep(0.5)
-      manager.terminate
-    end
-  end
+      it { expect(manager.instance_variable_get(:@workers).count).to be 4 }
 
-  describe '#suspend' do
-    it 'suspends the manager' do
-      expect(manager.instance_variable_get(:@workers)).to all(receive(:suspend).and_call_original)
-      task = Async do
-        Async { manager.start }
-        sleep(0.5)
-        manager.suspend
-        expect(manager.instance_variable_get(:@done)).to be true
+      describe '#start' do
+        it 'starts the manager' do
+          expect(manager.instance_variable_get(:@workers)).to all(receive(:run))
+          task = Async do
+            Async { manager.start }
+            sleep(0.5)
+            manager.terminate
+          end
+          task.wait
+        end
       end
-      task.wait
-    end
-  end
-
-  describe '#terminate' do
-    it 'terminates the manager' do
-      expect(manager.instance_variable_get(:@workers)).to all(receive(:suspend).and_call_original)
-      expect(manager.instance_variable_get(:@workers)).to all(receive(:terminate).and_call_original)
-      task = Async do
-        Async { manager.start }
-        sleep(0.5)
-        manager.terminate
-        sleep(2) # wait for timeout
-        expect(manager.instance_variable_get(:@done)).to be true
+    
+      describe '#suspend' do
+        it 'suspends the manager' do
+          expect(manager.instance_variable_get(:@fetcher)).to receive(:suspend).and_call_original
+          task = Async do
+            Async { manager.start }
+            sleep(0.5)
+            manager.suspend
+            expect(manager.instance_variable_get(:@done)).to be true
+            manager.terminate
+          end
+          task.wait
+        end
+      end
+    
+      describe '#terminate' do
+        it 'terminates the manager' do
+          expect(manager.instance_variable_get(:@fetcher)).to receive(:suspend).and_call_original
+          task = Async do
+            Async { manager.start }
+            sleep(0.5)
+            manager.terminate
+            sleep(2) # wait for timeout
+            expect(manager.instance_variable_get(:@done)).to be true
+          end
+          task.wait
+        end
       end
-      task.wait
     end
   end
 end

data/spec/jiggler/retrier_spec.rb

@@ -8,7 +8,9 @@ RSpec.describe Jiggler::Retrier do
       queues: ['test']
     ) 
   end
-  let(:collection) { Jiggler::Stats::Collection.new('test-retrier-uuid') }
+  let(:uuid) { "#{SecureRandom.hex(3)}-test" }
+  let(:identity) { "#{uuid}-identity" }
+  let(:collection) { Jiggler::Stats::Collection.new(uuid, identity) }
   let(:retrier) { Jiggler::Retrier.new(config, collection) }
 
   describe '#wrapped' do

data/spec/jiggler/scheduled/requeuer_spec.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+RSpec.describe Jiggler::Scheduled::Requeuer do
+  let(:config) do
+    Jiggler::Config.new(
+      concurrency: 1,
+      timeout: 1,
+      poller_enabled: false,
+      queues: ['my_queue']
+    )
+  end
+  let(:launcher) { Jiggler::Launcher.new(config) }
+  let(:identity) { launcher.send(:identity) }
+  let(:uuid) { launcher.send(:uuid) }
+  let(:requeuer) { Jiggler::Scheduled::Requeuer.new(config) }
+
+  after { config.cleaner.prune_all }
+
+  describe '#running_processes_uuid' do
+    it 'fetches process uuids' do
+      config.with_sync_redis do |conn|
+        conn.call('SET', identity, 'test')
+      end
+      expect(requeuer.send(:running_processes_uuid)).to eq([uuid])
+    end
+  end
+
+  describe '#requeue_data' do
+    it 'selects data without running processes' do
+      config.with_sync_redis do |conn|
+        conn.call('SET', identity, 'test')
+        conn.call('LPUSH', "jiggler:list:my_queue:in_progress:#{uuid}", 'test')
+      end
+      expect(requeuer.send(:requeue_data)).to eq([])
+      config.with_sync_redis do |conn|
+        conn.call('DEL', identity)
+      end
+      expect(requeuer.send(:requeue_data)).to eq([
+        ["jiggler:list:my_queue:in_progress:#{uuid}", "jiggler:list:my_queue", uuid]
+      ])
+    end
+  end
+
+  describe '#handle_stale' do
+    it 'requeues stale data' do
+      config.with_sync_redis do |conn|
+        conn.call('LPUSH', "jiggler:list:my_queue:in_progress:#{uuid}", 'test')
+        expect(conn.call('LLEN', 'jiggler:list:my_queue')).to be 0
+      end
+      requeuer.handle_stale
+      config.with_sync_redis do |conn|
+        expect(conn.call('LLEN', "jiggler:list:my_queue:in_progress:#{uuid}")).to be 0
+        expect(conn.call('LLEN', 'jiggler:list:my_queue')).to be 1
+      end
+    end
+  end
+end

data/spec/jiggler/stats/monitor_spec.rb

@@ -8,7 +8,8 @@ RSpec.describe Jiggler::Stats::Monitor do
     )
   end
   let(:uuid) { 'monitor-test-uuid' }
-  let(:collection) { Jiggler::Stats::Collection.new(uuid) }
+  let(:identity) { "#{uuid}-identity" }
+  let(:collection) { Jiggler::Stats::Collection.new(uuid, identity) }
   let(:monitor) { described_class.new(config, collection) }
 
   describe '#start' do
@@ -32,7 +33,7 @@ RSpec.describe Jiggler::Stats::Monitor do
         expect(monitor).to receive(:cleanup).and_call_original
         monitor.terminate
         expect(monitor.instance_variable_get(:@done)).to be true
-        expect(config.client_redis_pool.acquire { |conn| conn.call('GET', uuid) }).to be nil
+        expect(config.client_redis_pool.acquire { |conn| conn.call('GET', identity) }).to be nil
       end
       task.wait
     end

data/spec/jiggler/summary_spec.rb

@@ -8,10 +8,14 @@ RSpec.describe Jiggler::Summary do
       timeout: 1,
       stats_interval: 1,
       queues: queues,
-      poller_enabled: false
+      poller_enabled: false,
+      mode: :at_most_once
     )
   end
-  let(:collection) { Jiggler::Stats::Collection.new('summary-test-uuid') } 
+  let(:uuid) { "#{SecureRandom.hex(3)}-test" }
+  let(:collection) { Jiggler::Stats::Collection.new(uuid, uuid) }
+  let(:acknowledger) { Jiggler::AtMostOnce::Acknowledger.new(config) }
+  let(:fetcher) { Jiggler::AtMostOnce::Fetcher.new(config, collection) }
   let(:summary) { described_class.new(config) }
 
   describe '.all' do
@@ -33,7 +37,7 @@ RSpec.describe Jiggler::Summary do
       Sync do
         config.cleaner.prune_all
         launcher = Jiggler::Launcher.new(config)
-        uuid = launcher.send(:uuid)
+        uuid = launcher.send(:identity)
         MyJob.with_options(queue: 'queue1').enqueue
         
         first_summary = summary.all
@@ -90,12 +94,13 @@ RSpec.describe Jiggler::Summary do
 
   describe '#last_dead_jobs' do
     it 'returns last n dead jobs' do
+      # clean data and enqueue a failling job with no retries
       Sync do 
         config.cleaner.prune_all
         expect(summary.last_dead_jobs(1)).to be_empty
         MyFailedJob.with_options(queue: 'queue1', retries: 0).enqueue('yay')
       end
-      worker = Jiggler::Worker.new(config, collection) do
+      worker = Jiggler::Worker.new(config, collection, acknowledger, fetcher) do
         config.logger.info('Doing some weird dead testings')
       end
       task = Async do
@@ -106,6 +111,8 @@ RSpec.describe Jiggler::Summary do
         worker.terminate
       end
       task.wait
+
+      # ensure the job is saved as dead
       jobs = Sync { summary.last_dead_jobs(3) }
       expect(jobs.count).to be 1
       expect(jobs.first).to include({
@@ -117,6 +124,7 @@ RSpec.describe Jiggler::Summary do
 
   describe '#last_retry_jobs' do
     it 'returns last n retry jobs' do
+      # clean data and enqueue 5 failling jobs
       Sync do
         config.cleaner.prune_all
         expect(summary.last_retry_jobs(3)).to be_empty
@@ -127,9 +135,12 @@ RSpec.describe Jiggler::Summary do
           ).enqueue("yay-#{i}")
         end
       end
-      worker = Jiggler::Worker.new(config, collection) do
+      
+      worker = Jiggler::Worker.new(config, collection, acknowledger, fetcher) do
         config.logger.info('Doing some weird retry testings')
       end
+      
+      # start worker, wait for 1 sec, terminate worker
       task = Async do
         Async do
           worker.run
@@ -138,7 +149,10 @@ RSpec.describe Jiggler::Summary do
         worker.terminate
       end
       task.wait
+
+      # fetch last 3 retry jobs to ensure they are in the right order
       jobs = Sync { summary.last_retry_jobs(3) }
+      puts jobs.first
       expect(jobs.count).to be 3
       expect(jobs.first).to include({
         'name' => 'MyFailedJob',

data/spec/jiggler/worker_spec.rb

@@ -1,22 +1,26 @@
 # frozen_string_literal: true
 
 RSpec.describe Jiggler::Worker do
-  before(:all) do 
+  before(:all) do
     Jiggler.instance_variable_set(:@config, Jiggler::Config.new(
       concurrency: 1,
       client_concurrency: 1,
       timeout: 1,
-      queues: ['default', 'test']
-    ) )
+      queues: [['default', 2], ['test', 1]],
+      mode: :at_most_once
+    ))
   end
   after(:all) do
     Jiggler.instance_variable_set(:@config, Jiggler::Config.new)
   end
 
   let(:config) { Jiggler.config }
-  let(:collection) { Jiggler::Stats::Collection.new(config) }
+  let(:uuid) { "#{SecureRandom.hex(3)}-test" }
+  let(:collection) { Jiggler::Stats::Collection.new(uuid, uuid) }
+  let(:acknowledger) { Jiggler::AtMostOnce::Acknowledger.new(config) }
+  let(:fetcher) { Jiggler::AtMostOnce::Fetcher.new(config, collection) }
   let(:worker) do 
-    described_class.new(config, collection) do
+    described_class.new(config, collection, acknowledger, fetcher) do
       config.logger.debug("Callback called: #{rand(100)}")
     end
   end
@@ -90,20 +94,12 @@ RSpec.describe Jiggler::Worker do
     context 'when worker is running' do
       it 'terminates the worker' do
         task = Async do
-          expect(worker.done).to be false
           Async { worker.run }
+          sleep(0.5)
           worker.terminate
         end
         task.wait
-        expect(worker.done).to be true
-      end
-    end
-
-    context 'when worker is not running' do
-      it do
-        expect(worker.done).to be false
-        worker.terminate
-        expect(worker.done).to be true
+        expect(worker.instance_variable_get(:@runner)).to be_nil
       end
     end
   end

data/spec/spec_helper.rb

@@ -10,12 +10,19 @@ require 'jiggler/web'
 require 'jiggler/support/helper'
 require 'jiggler/scheduled/enqueuer'
 require 'jiggler/scheduled/poller'
+require 'jiggler/scheduled/requeuer'
 require 'jiggler/stats/collection'
 require 'jiggler/stats/monitor'
 require 'jiggler/errors'
 require 'jiggler/retrier'
 require 'jiggler/launcher'
 require 'jiggler/manager'
+require 'jiggler/base_acknowledger'
+require 'jiggler/base_fetcher'
+require 'jiggler/at_most_once/acknowledger'
+require 'jiggler/at_most_once/fetcher'
+require 'jiggler/at_least_once/acknowledger'
+require 'jiggler/at_least_once/fetcher'
 require 'jiggler/worker'
 require 'jiggler/cli'
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: jiggler
 version: !ruby/object:Gem::Version
-  version: 0.1.0.rc4
+  version: 0.1.0.rc6
 platform: ruby
 authors:
 - Julija Alieckaja
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-01-18 00:00:00.000000000 Z
+date: 2023-02-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: async
@@ -59,42 +59,56 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.11'
+        version: '0.12'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.11'
+        version: '0.12'
 - !ruby/object:Gem::Dependency
   name: oj
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.13'
+        version: '3.14'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.13'
+        version: '3.14'
+- !ruby/object:Gem::Dependency
+  name: priority_queue_cxx
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.3'
+        version: '2.4'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.3'
+        version: '2.4'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -135,7 +149,13 @@ files:
 - README.md
 - bin/jiggler
 - lib/jiggler.rb
+- lib/jiggler/at_least_once/acknowledger.rb
+- lib/jiggler/at_least_once/fetcher.rb
+- lib/jiggler/at_most_once/acknowledger.rb
+- lib/jiggler/at_most_once/fetcher.rb
 - lib/jiggler/base.rb
+- lib/jiggler/base_acknowledger.rb
+- lib/jiggler/base_fetcher.rb
 - lib/jiggler/cleaner.rb
 - lib/jiggler/cli.rb
 - lib/jiggler/client.rb
@@ -149,6 +169,7 @@ files:
 - lib/jiggler/retrier.rb
 - lib/jiggler/scheduled/enqueuer.rb
 - lib/jiggler/scheduled/poller.rb
+- lib/jiggler/scheduled/requeuer.rb
 - lib/jiggler/server.rb
 - lib/jiggler/stats/collection.rb
 - lib/jiggler/stats/monitor.rb
@@ -165,6 +186,9 @@ files:
 - spec/fixtures/my_failed_job.rb
 - spec/fixtures/my_job.rb
 - spec/fixtures/my_job_with_args.rb
+- spec/jiggler/at_least_once/acknowledger_spec.rb
+- spec/jiggler/at_least_once/fetcher_spec.rb
+- spec/jiggler/at_most_once/fetcher_spec.rb
 - spec/jiggler/cleaner_spec.rb
 - spec/jiggler/cli_spec.rb
 - spec/jiggler/config_spec.rb
@@ -176,6 +200,7 @@ files:
 - spec/jiggler/retrier_spec.rb
 - spec/jiggler/scheduled/enqueuer_spec.rb
 - spec/jiggler/scheduled/poller_spec.rb
+- spec/jiggler/scheduled/requeuer_spec.rb
 - spec/jiggler/stats/monitor_spec.rb
 - spec/jiggler/summary_spec.rb
 - spec/jiggler/web_spec.rb
@@ -204,7 +229,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubygems_version: 3.4.3
+rubygems_version: 3.4.6
 signing_key:
 specification_version: 4
 summary: Ruby background job processor
@@ -215,6 +240,9 @@ test_files:
 - spec/fixtures/my_failed_job.rb
 - spec/fixtures/my_job.rb
 - spec/fixtures/my_job_with_args.rb
+- spec/jiggler/at_least_once/acknowledger_spec.rb
+- spec/jiggler/at_least_once/fetcher_spec.rb
+- spec/jiggler/at_most_once/fetcher_spec.rb
 - spec/jiggler/cleaner_spec.rb
 - spec/jiggler/cli_spec.rb
 - spec/jiggler/config_spec.rb
@@ -226,6 +254,7 @@ test_files:
 - spec/jiggler/retrier_spec.rb
 - spec/jiggler/scheduled/enqueuer_spec.rb
 - spec/jiggler/scheduled/poller_spec.rb
+- spec/jiggler/scheduled/requeuer_spec.rb
 - spec/jiggler/stats/monitor_spec.rb
 - spec/jiggler/summary_spec.rb
 - spec/jiggler/web_spec.rb