sqewer 5.0.0 → 5.0.1

data/spec/spec_helper.rb

@@ -15,12 +15,18 @@ RSpec.configure do |config|
   config.order = 'random'
   config.around :each do | example |
     if example.metadata[:sqs]
-      queue_name = 'conveyor-belt-test-queue-%s' % SecureRandom.hex(6)
+      queue_name = 'sqewer-test-queue-%s' % SecureRandom.hex(6)
       client = Aws::SQS::Client.new
       resp = client.create_queue(queue_name: queue_name)
       ENV['SQS_QUEUE_URL'] = resp.queue_url
+      
       example.run
-      resp = client.delete_queue(queue_url: ENV.fetch('SQS_QUEUE_URL'))
+      
+      # Sometimes the queue is already deleted before the example completes. If the test has passed,
+      # we do not really care whether this invocation raises an exception about a non-existent queue since
+      # all we care about is the queue _being gone_ at the end of the example.
+      client.delete_queue(queue_url: ENV.fetch('SQS_QUEUE_URL')) rescue Aws::SQS::Errors::NonExistentQueue
+      
       ENV.delete('SQS_QUEUE_URL')
     else
       example.run

data/spec/sqewer/active_job_spec.rb

@@ -0,0 +1,113 @@
+require_relative '../spec_helper'
+require 'securerandom'
+require 'active_job'
+require 'active_record'
+require 'global_id'
+require_relative '../../lib/sqewer/extensions/active_job_adapter'
+
+class CreateFileJob < ActiveJob::Base
+  def perform(file)
+    File.open(file, 'w') {}
+  end
+end
+
+class DeleteFileJob < ActiveJob::Base
+  def perform(file)
+    File.unlink(file)
+  end
+end
+
+class ActivateUser < ActiveJob::Base
+  def perform(user)
+    user.active = true
+    user.save!
+  end
+end
+
+GlobalID.app = 'test-app'
+class User < ActiveRecord::Base
+  include GlobalID::Identification
+end
+
+describe ActiveJob::QueueAdapters::SqewerAdapter, :sqs => true do
+  let(:file) { File.join(Dir.tmpdir, "file_active_job_test_1") }
+  let(:client) { ::Aws::SQS::Client.new }
+
+  after :all do
+    # Ensure database files get killed afterwards
+    File.unlink(ActiveRecord::Base.connection_config[:database]) rescue nil
+  end
+
+  before :all do
+    ActiveJob::Base.queue_adapter = ActiveJob::QueueAdapters::SqewerAdapter
+
+    test_seed_name = SecureRandom.hex(4)
+    ActiveRecord::Base.establish_connection(adapter: 'sqlite3', database: ('master_db_%s.sqlite3' % test_seed_name))
+
+    ActiveRecord::Migration.suppress_messages do
+      ActiveRecord::Schema.define(:version => 1) do
+        create_table :users do |t|
+          t.string :email, :null => true
+          t.boolean :active, default: false
+          t.timestamps :null => false
+        end
+      end
+    end
+  end
+
+  before do
+    @queue_url_hash = { queue_url: ENV['SQS_QUEUE_URL'] }
+  end
+
+  it "sends job to the queue" do
+    CreateFileJob.perform_later(file)
+    resp = client.get_queue_attributes(@queue_url_hash.merge(attribute_names: ["ApproximateNumberOfMessages"]))
+    expect(resp.attributes["ApproximateNumberOfMessages"].to_i).to eq(1)
+  end
+
+  it "correctly serializes the job into a Sqewer job" do
+    job = CreateFileJob.perform_later(file)
+    resp = client.receive_message(@queue_url_hash)
+    serialized_job = JSON.parse(resp.messages.last.body)
+
+    expect(serialized_job["_job_class"]).to eq("ActiveJob::QueueAdapters::SqewerAdapter::Performable")
+    expect(serialized_job["_job_params"]["job"]["job_id"]).to eq(job.job_id)
+  end
+
+  it "executes job from the queue" do
+    file_delayed = "#{file}_delayed"
+    CreateFileJob.perform_later(file)
+    CreateFileJob.perform_later(file_delayed)
+    w = Sqewer::Worker.default
+    w.start
+    begin
+      wait_for { File.exist?(file) }.to eq(true)
+      File.unlink(file)
+      
+      wait_for { File.exist?(file_delayed) }.to eq(true)
+      DeleteFileJob.set(wait: 5.seconds).perform_later(file_delayed)
+      
+      sleep 6
+      
+      expect(File.exist?(file_delayed)).to eq(false)
+    ensure
+      w.stop
+    end
+  end
+
+  it "serializes and deserializes active record using GlobalID" do
+    user = User.create(email: 'test@wetransfer.com')
+    expect(user.active).to eq(false)
+    ActivateUser.perform_later(user)
+    w = Sqewer::Worker.default
+    begin
+      w.start
+      sleep 4
+      user.reload
+      expect(user.active).to eq(true)
+    ensure
+      w.stop
+    end
+  end
+
+end

data/spec/sqewer/cli_spec.rb

@@ -4,63 +4,80 @@ describe Sqewer::CLI, :sqs => true, :wait => {timeout: 120} do
   after :each do
     Dir.glob('*-result').each{|path| File.unlink(path) }
   end
-    
+  
+  describe 'with a mock Worker' do
+    it 'uses just three methods' do
+      mock_worker = Class.new do
+        def self.start; end
+        def self.stop; end
+        def self.debug_thread_information!; end
+      end
+      
+      worker_pid = fork do
+        Sqewer::CLI.start(mock_worker)
+      end
+      sleep 1
+      
+      begin
+        Process.kill('INFO', worker_pid) # Calls debug_thread_information!
+      rescue ArgumentError, Errno::ENOTSUP # on Linux
+      end
+      Process.kill('TERM', worker_pid) # Terminates the worker
+      
+      wait_for { 
+        _, status = Process.wait2(worker_pid)
+        expect(status.exitstatus).to be_zero # Must have quit cleanly
+      }
+    end
+  end
+  
   describe 'runs the commandline app, executes jobs and then quits cleanly' do
     it 'on a USR1 signal' do
       submitter = Sqewer::Connection.default
     
-      stderr = Tempfile.new('worker-stderr')
-      stderr.sync = true
-      
-      pid = fork { $stderr.reopen(stderr); $stderr.sync = true; exec("ruby #{__dir__}/cli_app.rb") }
-  
+      pid = fork { exec("ruby #{__dir__}/cli_app.rb") }
+
       Thread.new do
         20.times do
-          j = {job_class: 'MyJob', first_name: 'John', last_name: 'Doe'}
+          j = {"_job_class" => 'MyJob', "_job_params" => {first_name: 'John', last_name: 'Doe'}}
           submitter.send_message(JSON.dump(j))
         end
       end
-   
-      sleep 10 # Give it some time to process all the jobs
-      Process.kill("USR1", pid)
-      wait_for { Process.wait(pid) }
+
+      sleep 8
+      wait_for { 
+        Process.kill("USR1", pid)
+        _, status = Process.wait2(pid)
+        expect(status.exitstatus).to be_zero # Must have quit cleanly
+      }
       
       generated_files = Dir.glob('*-result')
       expect(generated_files).not_to be_empty
-    
-      stderr.rewind
-      log_output = stderr.read
-      # This assertion frequently fails (probably because STDERR doesn't get flushed properly)
-      # expect(log_output).to include('Stopping (clean shutdown)')
+      generated_files.each{|path| File.unlink(path) }
     end
     
     it 'on a TERM signal' do
       submitter = Sqewer::Connection.default
     
-      stderr = Tempfile.new('worker-stderr')
-      stderr.sync = true
-    
-      pid = fork { $stderr.reopen(stderr); $stderr.sync; exec("ruby #{__dir__}/cli_app.rb") }
-  
+      pid = fork { exec("ruby #{__dir__}/cli_app.rb") }
+
       Thread.new do
         20.times do
-          j = {job_class: 'MyJob', first_name: 'John', last_name: 'Doe'}
+          j = {"_job_class" => 'MyJob', "_job_params" => {first_name: 'John', last_name: 'Doe'}}
           submitter.send_message(JSON.dump(j))
         end
       end
-   
-      sleep 4
-      Process.kill("TERM", pid)
-      wait_for { Process.wait(pid) }
+
+      sleep 8
+      wait_for { 
+        Process.kill("TERM", pid)
+        _, status = Process.wait2(pid)
+        expect(status.exitstatus).to be_zero # Must have quit cleanly
+      }
       
       generated_files = Dir.glob('*-result')
       expect(generated_files).not_to be_empty
       generated_files.each{|path| File.unlink(path) }
-    
-      stderr.rewind
-      log_output = stderr.read
-      # This assertion frequently fails (probably because STDERR doesn't get flushed properly)
-      # expect(log_output).to include('Stopping (clean shutdown)')
     end
   end
 end

data/spec/sqewer/serializer_spec.rb

@@ -31,6 +31,17 @@ describe Sqewer::Serializer do
       expect(described_class.new.serialize(job)).to eq("{\"_job_class\":\"SomeJob\",\"_job_params\":{\"one\":123,\"two\":[456]}}")
     end
     
+    it 'adds _execute_after when the value is given' do
+      class ThirdJob < Struct.new :one, :two
+      end
+      
+      job = ThirdJob.new(123, [456])
+      res = described_class.new.serialize(job, Time.now.to_i + 1500)
+      parsed = JSON.load(res)
+      
+      expect(parsed["_execute_after"]).to be_within(10).of(Time.now.to_i + 1500)
+    end
+    
     it 'raises an exception if the object is of an anonymous class' do
       s = Struct.new(:foo)
       o = s.new(1)
@@ -58,66 +69,50 @@ describe Sqewer::Serializer do
   end
   
   describe '#unserialize' do
-    describe 'with the old ticket format' do
-      it 'builds a job without keyword arguments if its constructor does not need any kwargs' do
-        class VerySimpleJob; end
-        blob  = '{"job_class": "VerySimpleJob"}'
-        built_job = described_class.new.unserialize(blob)
-        expect(built_job).to be_kind_of(VerySimpleJob)
-      end
+    it 'wraps the job with a Resubmit when the _execute_after key hints that it is too early' do
+      class EvenSimplerJob; end
     
-      it 'raises an error if the job does not accept the keyword arguments given in the ticket' do
-        class OtherVerySimpleJob; end
-        blob  = '{"job_class": "OtherVerySimpleJob", "foo": 1}'
-      
-        expect {
-          described_class.new.unserialize(blob)
-        }.to raise_error(described_class::ArityMismatch)
-      end
+      timestamp_way_in_the_future = Time.now.to_i + (60 * 60 * 24 * 3)
+      blob  = '{"_job_class": "EvenSimplerJob", "_execute_after": %d}' % timestamp_way_in_the_future
+      built_job = described_class.new.unserialize(blob)
+    
+      expect(built_job).to be_kind_of(Sqewer::Resubmit)
+      expect(built_job.execute_after).to eq(timestamp_way_in_the_future)
       
-      it 'instantiates the job with keyword arguments' do
-        ValidJob = Ks.strict(:foo)
-        
-        blob  = '{"job_class": "ValidJob", "foo": 1}'
-        built_job = described_class.new.unserialize(blob)
-        
-        expect(built_job).to be_kind_of(ValidJob)
-        expect(built_job.foo).to eq(1)
-      end
+      embedded_job = built_job.job
+      expect(embedded_job).to be_kind_of(EvenSimplerJob)
     end
     
-    describe 'with the new ticket format' do
-      it 'builds a job without keyword arguments if its constructor does not need any kwargs' do
-        class EvenSimplerJob; end
-        
-        blob  = '{"_job_class": "EvenSimplerJob"}'
-        built_job = described_class.new.unserialize(blob)
-        
-        expect(built_job).to be_kind_of(EvenSimplerJob)
-        
-        blob  = '{"_job_class": "EvenSimplerJob", "_job_params": null}'
-        built_job = described_class.new.unserialize(blob)
-        
-        expect(built_job).to be_kind_of(EvenSimplerJob)
-      end
-      
-      it 'raises an error if the job does not accept the keyword arguments given in the ticket' do
-        class MicroJob; end
-        blob  = '{"_job_class": "MicroJob", "_job_params":{"foo": 1}}'
-        expect {
-          described_class.new.unserialize(blob)
-        }.to raise_error(described_class::ArityMismatch)
-      end
-      
-      it 'instantiates the job with keyword arguments' do
-        OtherValidJob = Ks.strict(:foo)
-        
-        blob  = '{"_job_class": "OtherValidJob", "_job_params": {"foo": 1}}'
-        built_job = described_class.new.unserialize(blob)
-        
-        expect(built_job).to be_kind_of(OtherValidJob)
-        expect(built_job.foo).to eq(1)
-      end
+    it 'builds a job without keyword arguments if its constructor does not need any kwargs' do
+      class EvenSimplerJob; end
+    
+      blob  = '{"_job_class": "EvenSimplerJob"}'
+      built_job = described_class.new.unserialize(blob)
+    
+      expect(built_job).to be_kind_of(EvenSimplerJob)
+    
+      blob  = '{"_job_class": "EvenSimplerJob", "_job_params": null}'
+      built_job = described_class.new.unserialize(blob)
+    
+      expect(built_job).to be_kind_of(EvenSimplerJob)
+    end
+  
+    it 'raises an error if the job does not accept the keyword arguments given in the ticket' do
+      class MicroJob; end
+      blob  = '{"_job_class": "MicroJob", "_job_params":{"foo": 1}}'
+      expect {
+        described_class.new.unserialize(blob)
+      }.to raise_error(ArgumentError)
+    end
+  
+    it 'instantiates the job with keyword arguments' do
+      OtherValidJob = Ks.strict(:foo)
+    
+      blob  = '{"_job_class": "OtherValidJob", "_job_params": {"foo": 1}}'
+      built_job = described_class.new.unserialize(blob)
+    
+      expect(built_job).to be_kind_of(OtherValidJob)
+      expect(built_job.foo).to eq(1)
     end
   end
 end

data/spec/sqewer/submitter_spec.rb

@@ -39,5 +39,23 @@ describe Sqewer::Submitter do
       subject = described_class.new(fake_connection, fake_serializer)
       subject.submit!(:some_object, delay_seconds: 5)
     end
+    
+    it 'handles the massively delayed execution by clamping the delay_seconds to the SQS maximum, and saving the _execute_after' do
+      fake_serializer = double('Some serializer')
+      allow(fake_serializer).to receive(:serialize) {|object_to_serialize, timestamp_seconds|
+        
+        delay_by = Time.now.to_i + 4585659855
+        expect(timestamp_seconds).to be_within(20).of(delay_by)
+        
+        expect(object_to_serialize).not_to be_nil
+        'serialized-object-data'
+      }
+      
+      fake_connection = double('Some SQS connection')
+      expect(fake_connection).to receive(:send_message).with('serialized-object-data', {delay_seconds: 899})
+      
+      subject = described_class.new(fake_connection, fake_serializer)
+      subject.submit!(:some_object, delay_seconds: 4585659855)
+    end
   end
 end

data/spec/sqewer/worker_spec.rb

@@ -22,6 +22,11 @@ describe Sqewer::Worker, :sqs => true do
     expect(default_worker).to respond_to(:stop)
   end
     
+  it 'instantiates a new worker object on every call to .default' do
+    workers = (1..10).map { described_class.default }
+    expect(workers.uniq.length).to eq(10)
+  end
+  
   it 'instantiates a Logger to STDERR by default' do
     expect(Logger).to receive(:new).with(STDERR)
     worker = described_class.new
@@ -57,7 +62,7 @@ describe Sqewer::Worker, :sqs => true do
   
   context 'when the job cannot be instantiated due to an unknown class' do
     it 'is able to cope with an exception when the job class is unknown (one of generic exceptions)' do
-      payload = JSON.dump({job_class: 'UnknownJobClass', arg1: 'some value'})
+      payload = JSON.dump({_job_class: 'UnknownJobClass', _job_params: {arg1: 'some value'}})
     
       client = Aws::SQS::Client.new
       client.send_message(queue_url: ENV.fetch('SQS_QUEUE_URL'), message_body: payload)
@@ -77,18 +82,18 @@ describe Sqewer::Worker, :sqs => true do
     it 'sets up the processing pipeline so that jobs can execute in sequence (with threads)' do
       class SecondaryJob
         def run
-          File.open('secondary-job-run','w') {}
+          File.open(File.join(Dir.tmpdir, 'secondary-job-run'),'w') {}
         end
       end
       
       class InitialJob
         def run(executor)
-          File.open('initial-job-run','w') {}
+          File.open(File.join(Dir.tmpdir, 'initial-job-run'),'w') {}
           executor.submit!(SecondaryJob.new)
         end
       end
       
-      payload = JSON.dump({job_class: 'InitialJob'})
+      payload = JSON.dump({_job_class: 'InitialJob'})
       client = Aws::SQS::Client.new
       client.send_message(queue_url: ENV.fetch('SQS_QUEUE_URL'), message_body: payload)
       
@@ -97,11 +102,8 @@ describe Sqewer::Worker, :sqs => true do
       worker.start
       
       begin
-        wait_for { File.exist?('initial-job-run') }.to eq(true)
-        wait_for { File.exist?('secondary-job-run') }.to eq(true)
-        
-        File.unlink('initial-job-run')
-        File.unlink('secondary-job-run')
+        wait_for { File.exist?(File.join(Dir.tmpdir, 'initial-job-run')) }.to eq(true)
+        wait_for { File.exist?(File.join(Dir.tmpdir, 'secondary-job-run')) }.to eq(true)
       ensure
         worker.stop
       end