sqewer 1.0.0

data/spec/conveyor_belt_spec.rb

@@ -0,0 +1,14 @@
+require_relative 'spec_helper'
+
+describe Sqewer do
+  it 'provides a #submit!() method that is a shortcut to the default submitter' do
+    fake_submitter = double('Submitter')
+    expect(Sqewer::Submitter).to receive(:default) { fake_submitter }
+    
+    first_job = double('Job1')
+    second_job = double('Job2')
+    
+    expect(fake_submitter).to receive(:submit!).with(first_job, second_job, {})
+    Sqewer.submit!(first_job, second_job)
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,49 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'rspec'
+require 'simplecov'
+
+require 'dotenv'
+Dotenv.load
+
+require 'aws-sdk'
+
+SimpleCov.start
+require 'sqewer'
+
+module Polling
+  # Call the given block every N seconds, and return once the
+  # block returns a truthy value. If it still did not return
+  # the value after fail_after, fail the spec.
+  def poll(every: 0.5, fail_after:, &check_block)
+    started_polling = Time.now
+    loop do
+      return if check_block.call
+      sleep(every)
+      if (Time.now - started_polling) > fail_after
+        fail "Waited for #{fail_after} seconds for the operation to complete but it didnt"
+      end
+    end
+  end
+end
+
+RSpec.configure do |config| 
+  config.order = 'random'
+  config.include Polling
+  
+  config.around :each do | example |
+    if example.metadata[:sqs]
+      queue_name = 'conveyor-belt-test-queue-%s' % SecureRandom.hex(6)
+      client = Aws::SQS::Client.new
+      resp = client.create_queue(queue_name: queue_name)
+      ENV['SQS_QUEUE_URL'] = resp.queue_url
+      example.run
+      resp = client.delete_queue(queue_url: ENV.fetch('SQS_QUEUE_URL'))
+      ENV.delete('SQS_QUEUE_URL')
+    else
+      example.run
+    end
+  end
+end
+

data/spec/sqewer/atomic_counter_spec.rb

@@ -0,0 +1,15 @@
+require_relative '../spec_helper'
+
+describe Sqewer::AtomicCounter do
+  it 'is atomic' do
+    c = described_class.new
+    expect(c.to_i).to be_zero
+    
+    threads = (1..64).map do
+      Thread.new { sleep(rand); c.increment! }
+    end
+    threads.map(&:join)
+    
+    expect(c.to_i).to eq(threads.length)
+  end
+end

data/spec/sqewer/cli_app.rb

@@ -0,0 +1,13 @@
+require_relative '../../lib/sqewer'
+
+class MyJob
+  include Sqewer::SimpleJob
+  attr_accessor :first_name
+  attr_accessor :last_name
+  
+  def run(executor)
+    File.open("#{SecureRandom.hex(3)}-result", 'w') {|f| f << [first_name, last_name].join }
+  end
+end
+
+Sqewer::CLI.start

data/spec/sqewer/cli_spec.rb

@@ -0,0 +1,57 @@
+require_relative '../spec_helper'
+
+describe Sqewer::CLI, :sqs => true do
+  describe 'runs the commandline app, executes jobs and then quits cleanly' do
+    it 'on a USR1 signal' do
+      submitter = Sqewer::Connection.default
+    
+      stderr = Tempfile.new('worker-stderr')
+    
+      pid = fork { $stderr.reopen(stderr); exec("ruby #{__dir__}/cli_app.rb") }
+  
+      Thread.new do
+        20.times do
+          j = {job_class: 'MyJob', first_name: 'John', last_name: 'Doe'}
+          submitter.send_message(JSON.dump(j))
+        end
+      end
+   
+      sleep 4
+      Process.kill("USR1", pid)
+      
+      generated_files = Dir.glob('*-result')
+      expect(generated_files).not_to be_empty
+      generated_files.each{|path| File.unlink(path) }
+    
+      stderr.rewind
+      log_output = stderr.read
+      expect(log_output).to include('Stopping (clean shutdown)')
+    end
+    
+    it 'on a TERM signal' do
+      submitter = Sqewer::Connection.default
+    
+      stderr = Tempfile.new('worker-stderr')
+    
+      pid = fork { $stderr.reopen(stderr); exec("ruby #{__dir__}/cli_app.rb") }
+  
+      Thread.new do
+        20.times do
+          j = {job_class: 'MyJob', first_name: 'John', last_name: 'Doe'}
+          submitter.send_message(JSON.dump(j))
+        end
+      end
+   
+      sleep 4
+      Process.kill("TERM", pid)
+    
+      generated_files = Dir.glob('*-result')
+      expect(generated_files).not_to be_empty
+      generated_files.each{|path| File.unlink(path) }
+    
+      stderr.rewind
+      log_output = stderr.read
+      expect(log_output).to include('Stopping (clean shutdown)')
+    end
+  end
+end

data/spec/sqewer/connection_spec.rb

@@ -0,0 +1,57 @@
+require_relative '../spec_helper'
+
+describe Sqewer::Connection do
+  describe '.default' do
+    it 'returns a new Connection with the SQS queue location picked from SQS_QUEUE_URL envvar' do
+      expect(ENV).to receive(:fetch).with('SQS_QUEUE_URL').and_return('https://aws-fake-queue.com')
+      default = described_class.default
+      expect(default).to be_kind_of(described_class)
+    end
+  end
+  
+  describe '#send_message' do
+    it 'sends the message to the SQS client created with the URL given to the constructor' do
+      fake_sqs_client = double('Client')
+      expect(Aws::SQS::Client).to receive(:new) { fake_sqs_client }
+      expect(fake_sqs_client).to receive(:send_message).
+        with({:queue_url=>"https://fake-queue.com", :message_body=>"abcdef"})
+      
+      conn = described_class.new('https://fake-queue.com')
+      conn.send_message('abcdef')
+    end
+    
+    it 'passes keyword args to Aws::SQS::Client' do
+      fake_sqs_client = double('Client')
+      expect(Aws::SQS::Client).to receive(:new) { fake_sqs_client }
+      expect(fake_sqs_client).to receive(:send_message).
+        with({:queue_url=>"https://fake-queue.com", :message_body=>"abcdef", delay_seconds: 5})
+      
+      conn = described_class.new('https://fake-queue.com')
+      conn.send_message('abcdef', delay_seconds: 5)
+    end
+  end
+  
+  describe '#poll' do
+    it 'uses the batched receive feature' do
+      s = described_class.new('https://fake-queue')
+      
+      fake_poller = double('QueuePoller')
+      expect(::Aws::SQS::QueuePoller).to receive(:new).with('https://fake-queue') { fake_poller }
+      expect(fake_poller).to receive(:poll) {|*a, **k, &blk|
+        expect(k[:max_number_of_messages]).to be > 1
+        bulk = (1..5).map do
+          double('SQSMessage', receipt_handle: SecureRandom.hex(4), body: 'Some message')
+        end
+        # Yields arrays of messages, so...
+        blk.call(bulk)
+      }
+      
+      receives = []
+      s.poll do | sqs_message_handle, sqs_message_body |
+        receives << [sqs_message_handle, sqs_message_body]
+      end
+      
+      expect(receives.length).to eq(5)
+    end
+  end
+end

data/spec/sqewer/execution_context_spec.rb

@@ -0,0 +1,43 @@
+require_relative '../spec_helper'
+
+describe Sqewer::ExecutionContext do
+  it 'offers a submit! that goes through the given Submitter argument' do
+    fake_submitter = double('Submitter')
+    expect(fake_submitter).to receive(:submit!).with(:fake_job, {})
+    
+    subject = described_class.new(fake_submitter)
+    subject.submit!(:fake_job)
+  end
+  
+  it 'offers arbitrary key/value storage' do
+    fake_submitter = double('Submitter')
+    subject = described_class.new(fake_submitter)
+    
+    subject['foo'] = 123
+    expect(subject['foo']).to eq(123)
+    expect(subject[:foo]).to eq(123)
+    expect(subject.fetch(:foo)).to eq(123)
+    
+    expect {
+      subject.fetch(:bar)
+    }.to raise_error(KeyError)
+    
+    default_value = subject.fetch(:bar) { 123 }
+    expect(default_value).to eq(123)
+  end
+  
+  it 'returns the NullLogger from #logger if no logger was passed to the constructor' do
+    fake_submitter = double('Submitter')
+    
+    subject = described_class.new(fake_submitter)
+    expect(subject.logger).to eq(Sqewer::NullLogger)
+  end
+  
+  it 'offers access to the given "logger" extra param if it was given to the constructor' do
+    fake_submitter = double('Submitter')
+    fake_logger = double('Logger')
+    
+    subject = described_class.new(fake_submitter, {'logger' => fake_logger})
+    expect(subject.logger).to eq(fake_logger)
+  end
+end

data/spec/sqewer/middleware_stack_spec.rb

@@ -0,0 +1,69 @@
+require_relative '../spec_helper'
+
+describe Sqewer::MiddlewareStack do
+  it 'returns a default instance' do
+    stacks = (1..100).map { described_class.default }
+    expect(stacks.uniq.length).to eq(1)
+  end
+  
+  describe '#around_deserialization' do
+    it 'works without any handlers added' do
+      stack = described_class.new
+      prepare_result = stack.around_deserialization(nil, 'msg-123', '{"body":"some text"}') { :prepared }
+      expect(prepare_result).to eq(:prepared)
+    end
+    
+    it 'works with an entire stack' do
+      called = []
+      handler = double('Some middleware')
+      allow(handler).to receive(:around_deserialization){|*a, &blk|
+        called << a
+        blk.call 
+      } 
+      
+      stack = described_class.new
+      stack << handler
+      stack << double('Object that does not handle around_deserialization')
+      stack << handler
+      stack << handler
+      result = stack.around_deserialization(nil, 'msg-123', '{"body":"some text"}') { :foo }
+      expect(result).to eq(:foo)
+      expect(called).not_to be_empty
+      
+      expect(called).to eq([
+          [nil, "msg-123", "{\"body\":\"some text\"}"],
+          [nil, "msg-123", "{\"body\":\"some text\"}"],
+          [nil, "msg-123", "{\"body\":\"some text\"}"]]
+      )
+    end
+  end
+  
+  describe '#around_execution' do
+    it 'works without any handlers added' do
+      stack = described_class.new
+      prepare_result = stack.around_execution(nil, nil) { :prepared }
+      expect(prepare_result).to eq(:prepared)
+    end
+    
+    it 'works with an entire stack' do
+      called = []
+      handler = double('Some middleware')
+      allow(handler).to receive(:around_execution){|*a, &blk|
+        called << a
+        blk.call
+      } 
+      
+      stack = described_class.new
+      stack << handler
+      stack << handler
+      stack << double('Object that does not handle around_execution')
+      stack << handler
+      
+      result = stack.around_execution(:some_job, :some_context) { :executed }
+      expect(result).to eq(:executed)
+      expect(called).not_to be_empty
+      
+      expect(called).to eq([[:some_job, :some_context], [:some_job, :some_context], [:some_job, :some_context]])
+    end
+  end
+end

data/spec/sqewer/serializer_spec.rb

@@ -0,0 +1,123 @@
+require_relative '../spec_helper'
+
+describe Sqewer::Serializer do
+  describe '.default' do
+    it 'returns the same Serializer instance' do
+      instances = (1..1000).map{ described_class.default }
+      instances.uniq!
+      expect(instances).to be_one
+      
+      the_instance = instances[0]
+      expect(the_instance).to respond_to(:serialize)
+      expect(the_instance).to respond_to(:unserialize)
+    end
+  end
+  
+  describe '#serialize' do
+    
+    it 'serializes a Job that has no to_h support without its kwargs' do
+      class JobWithoutToHash
+      end
+      job = JobWithoutToHash.new
+      expect(described_class.new.serialize(job)).to eq("{\"_job_class\":\"JobWithoutToHash\",\"_job_params\":null}")
+    end
+    
+    it 'serializes a Struct along with its members and the class name' do
+      class SomeJob < Struct.new :one, :two
+      end
+      
+      job = SomeJob.new(123, [456])
+      
+      expect(described_class.new.serialize(job)).to eq("{\"_job_class\":\"SomeJob\",\"_job_params\":{\"one\":123,\"two\":[456]}}")
+    end
+    
+    it 'raises an exception if the object is of an anonymous class' do
+      s = Struct.new(:foo)
+      o = s.new(1)
+      expect {
+        described_class.new.serialize(o)
+      }.to raise_error(described_class::AnonymousJobClass)
+    end
+  end
+  
+  it 'is able to roundtrip a job with a parameter' do
+    require 'ks'
+    
+    class LeJob < Ks.strict(:some_data)
+    end
+  
+    job = LeJob.new(some_data: 123)
+  
+    subject = described_class.new
+  
+    serialized = subject.serialize(job)
+    restored = subject.unserialize(serialized)
+  
+    expect(restored).to be_kind_of(LeJob)
+    expect(restored.some_data).to eq(123)
+  end
+  
+  describe '#unserialize' do
+    describe 'with the old ticket format' do
+      it 'builds a job without keyword arguments if its constructor does not need any kwargs' do
+        class VerySimpleJob; end
+        blob  = '{"job_class": "VerySimpleJob"}'
+        built_job = described_class.new.unserialize(blob)
+        expect(built_job).to be_kind_of(VerySimpleJob)
+      end
+    
+      it 'raises an error if the job does not accept the keyword arguments given in the ticket' do
+        class OtherVerySimpleJob; end
+        blob  = '{"job_class": "OtherVerySimpleJob", "foo": 1}'
+      
+        expect {
+          described_class.new.unserialize(blob)
+        }.to raise_error(described_class::ArityMismatch)
+      end
+      
+      it 'instantiates the job with keyword arguments' do
+        ValidJob = Ks.strict(:foo)
+        
+        blob  = '{"job_class": "ValidJob", "foo": 1}'
+        built_job = described_class.new.unserialize(blob)
+        
+        expect(built_job).to be_kind_of(ValidJob)
+        expect(built_job.foo).to eq(1)
+      end
+    end
+    
+    describe 'with the new ticket format' do
+      it 'builds a job without keyword arguments if its constructor does not need any kwargs' do
+        class EvenSimplerJob; end
+        
+        blob  = '{"_job_class": "EvenSimplerJob"}'
+        built_job = described_class.new.unserialize(blob)
+        
+        expect(built_job).to be_kind_of(EvenSimplerJob)
+        
+        blob  = '{"_job_class": "EvenSimplerJob", "_job_params": null}'
+        built_job = described_class.new.unserialize(blob)
+        
+        expect(built_job).to be_kind_of(EvenSimplerJob)
+      end
+      
+      it 'raises an error if the job does not accept the keyword arguments given in the ticket' do
+        class MicroJob; end
+        blob  = '{"_job_class": "MicroJob", "_job_params":{"foo": 1}}'
+        expect {
+          described_class.new.unserialize(blob)
+        }.to raise_error(described_class::ArityMismatch)
+      end
+      
+      it 'instantiates the job with keyword arguments' do
+        OtherValidJob = Ks.strict(:foo)
+        
+        blob  = '{"_job_class": "OtherValidJob", "_job_params": {"foo": 1}}'
+        built_job = described_class.new.unserialize(blob)
+        
+        expect(built_job).to be_kind_of(OtherValidJob)
+        expect(built_job.foo).to eq(1)
+      end
+    end
+  end
+end