sqewer 5.0.3 → 5.0.4

data/spec/spec_helper.rb

@@ -1,36 +0,0 @@
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-$LOAD_PATH.unshift(File.dirname(__FILE__))
-
-require 'rspec'
-require 'rspec/wait'
-require 'dotenv'
-require 'aws-sdk'
-require 'simplecov'
-Dotenv.load
-
-SimpleCov.start
-require 'sqewer'
-
-RSpec.configure do |config| 
-  config.order = 'random'
-  config.around :each do | example |
-    if example.metadata[:sqs]
-      queue_name = 'sqewer-test-queue-%s' % SecureRandom.hex(6)
-      client = Aws::SQS::Client.new
-      resp = client.create_queue(queue_name: queue_name)
-      ENV['SQS_QUEUE_URL'] = resp.queue_url
-      
-      example.run
-      
-      # Sometimes the queue is already deleted before the example completes. If the test has passed,
-      # we do not really care whether this invocation raises an exception about a non-existent queue since
-      # all we care about is the queue _being gone_ at the end of the example.
-      client.delete_queue(queue_url: ENV.fetch('SQS_QUEUE_URL')) rescue Aws::SQS::Errors::NonExistentQueue
-      
-      ENV.delete('SQS_QUEUE_URL')
-    else
-      example.run
-    end
-  end
-end
-

data/spec/sqewer/active_job_spec.rb

@@ -1,113 +0,0 @@
-require_relative '../spec_helper'
-require 'securerandom'
-require 'active_job'
-require 'active_record'
-require 'global_id'
-require_relative '../../lib/sqewer/extensions/active_job_adapter'
-
-class CreateFileJob < ActiveJob::Base
-  def perform(file)
-    File.open(file, 'w') {}
-  end
-end
-
-class DeleteFileJob < ActiveJob::Base
-  def perform(file)
-    File.unlink(file)
-  end
-end
-
-class ActivateUser < ActiveJob::Base
-  def perform(user)
-    user.active = true
-    user.save!
-  end
-end
-
-GlobalID.app = 'test-app'
-class User < ActiveRecord::Base
-  include GlobalID::Identification
-end
-
-describe ActiveJob::QueueAdapters::SqewerAdapter, :sqs => true do
-  let(:file) { File.join(Dir.tmpdir, "file_active_job_test_1") }
-  let(:client) { ::Aws::SQS::Client.new }
-
-  after :all do
-    # Ensure database files get killed afterwards
-    File.unlink(ActiveRecord::Base.connection_config[:database]) rescue nil
-  end
-
-  before :all do
-    ActiveJob::Base.queue_adapter = ActiveJob::QueueAdapters::SqewerAdapter
-
-    test_seed_name = SecureRandom.hex(4)
-    ActiveRecord::Base.establish_connection(adapter: 'sqlite3', database: ('master_db_%s.sqlite3' % test_seed_name))
-
-    ActiveRecord::Migration.suppress_messages do
-      ActiveRecord::Schema.define(:version => 1) do
-        create_table :users do |t|
-          t.string :email, :null => true
-          t.boolean :active, default: false
-          t.timestamps :null => false
-        end
-      end
-    end
-  end
-
-  before do
-    @queue_url_hash = { queue_url: ENV['SQS_QUEUE_URL'] }
-  end
-
-  it "sends job to the queue" do
-    CreateFileJob.perform_later(file)
-    resp = client.get_queue_attributes(@queue_url_hash.merge(attribute_names: ["ApproximateNumberOfMessages"]))
-    expect(resp.attributes["ApproximateNumberOfMessages"].to_i).to eq(1)
-  end
-
-  it "correctly serializes the job into a Sqewer job" do
-    job = CreateFileJob.perform_later(file)
-    resp = client.receive_message(@queue_url_hash)
-    serialized_job = JSON.parse(resp.messages.last.body)
-
-    expect(serialized_job["_job_class"]).to eq("ActiveJob::QueueAdapters::SqewerAdapter::Performable")
-    expect(serialized_job["_job_params"]["job"]["job_id"]).to eq(job.job_id)
-  end
-
-  it "executes job from the queue" do
-    file_delayed = "#{file}_delayed"
-    CreateFileJob.perform_later(file)
-    CreateFileJob.perform_later(file_delayed)
-    w = Sqewer::Worker.default
-    w.start
-    begin
-      wait_for { File.exist?(file) }.to eq(true)
-      File.unlink(file)
-      
-      wait_for { File.exist?(file_delayed) }.to eq(true)
-      DeleteFileJob.set(wait: 5.seconds).perform_later(file_delayed)
-      
-      sleep 6
-      
-      expect(File.exist?(file_delayed)).to eq(false)
-    ensure
-      w.stop
-    end
-  end
-
-  it "serializes and deserializes active record using GlobalID" do
-    user = User.create(email: 'test@wetransfer.com')
-    expect(user.active).to eq(false)
-    ActivateUser.perform_later(user)
-    w = Sqewer::Worker.default
-    begin
-      w.start
-      sleep 4
-      user.reload
-      expect(user.active).to eq(true)
-    ensure
-      w.stop
-    end
-  end
-
-end

data/spec/sqewer/atomic_counter_spec.rb

@@ -1,15 +0,0 @@
-require_relative '../spec_helper'
-
-describe Sqewer::AtomicCounter do
-  it 'is atomic' do
-    c = described_class.new
-    expect(c.to_i).to be_zero
-    
-    threads = (1..64).map do
-      Thread.new { sleep(rand); c.increment! }
-    end
-    threads.map(&:join)
-    
-    expect(c.to_i).to eq(threads.length)
-  end
-end

data/spec/sqewer/cli_app.rb

@@ -1,13 +0,0 @@
-require_relative '../../lib/sqewer'
-
-class MyJob
-  include Sqewer::SimpleJob
-  attr_accessor :first_name
-  attr_accessor :last_name
-  
-  def run(executor)
-    File.open("#{SecureRandom.hex(3)}-result", 'w') {|f| f << [first_name, last_name].join }
-  end
-end
-
-Sqewer::CLI.start

data/spec/sqewer/cli_spec.rb

@@ -1,83 +0,0 @@
-require_relative '../spec_helper'
-
-describe Sqewer::CLI, :sqs => true, :wait => {timeout: 120} do
-  after :each do
-    Dir.glob('*-result').each{|path| File.unlink(path) }
-  end
-  
-  describe 'with a mock Worker' do
-    it 'uses just three methods' do
-      mock_worker = Class.new do
-        def self.start; end
-        def self.stop; end
-        def self.debug_thread_information!; end
-      end
-      
-      worker_pid = fork do
-        Sqewer::CLI.start(mock_worker)
-      end
-      sleep 1
-      
-      begin
-        Process.kill('INFO', worker_pid) # Calls debug_thread_information!
-      rescue ArgumentError, Errno::ENOTSUP # on Linux
-      end
-      Process.kill('TERM', worker_pid) # Terminates the worker
-      
-      wait_for { 
-        _, status = Process.wait2(worker_pid)
-        expect(status.exitstatus).to be_zero # Must have quit cleanly
-      }
-    end
-  end
-  
-  describe 'runs the commandline app, executes jobs and then quits cleanly' do
-    it 'on a USR1 signal' do
-      submitter = Sqewer::Connection.default
-    
-      pid = fork { exec("ruby #{__dir__}/cli_app.rb") }
-
-      Thread.new do
-        20.times do
-          j = {"_job_class" => 'MyJob', "_job_params" => {first_name: 'John', last_name: 'Doe'}}
-          submitter.send_message(JSON.dump(j))
-        end
-      end
-
-      sleep 8
-      wait_for { 
-        Process.kill("USR1", pid)
-        _, status = Process.wait2(pid)
-        expect(status.exitstatus).to be_zero # Must have quit cleanly
-      }
-      
-      generated_files = Dir.glob('*-result')
-      expect(generated_files).not_to be_empty
-      generated_files.each{|path| File.unlink(path) }
-    end
-    
-    it 'on a TERM signal' do
-      submitter = Sqewer::Connection.default
-    
-      pid = fork { exec("ruby #{__dir__}/cli_app.rb") }
-
-      Thread.new do
-        20.times do
-          j = {"_job_class" => 'MyJob', "_job_params" => {first_name: 'John', last_name: 'Doe'}}
-          submitter.send_message(JSON.dump(j))
-        end
-      end
-
-      sleep 8
-      wait_for { 
-        Process.kill("TERM", pid)
-        _, status = Process.wait2(pid)
-        expect(status.exitstatus).to be_zero # Must have quit cleanly
-      }
-      
-      generated_files = Dir.glob('*-result')
-      expect(generated_files).not_to be_empty
-      generated_files.each{|path| File.unlink(path) }
-    end
-  end
-end

data/spec/sqewer/connection_spec.rb

@@ -1,145 +0,0 @@
-require_relative '../spec_helper'
-
-describe Sqewer::Connection do
-  describe '.default' do
-    it 'returns a new Connection with the SQS queue location picked from SQS_QUEUE_URL envvar' do
-      expect(ENV).to receive(:fetch).with('SQS_QUEUE_URL').and_return('https://aws-fake-queue.com')
-      default = described_class.default
-      expect(default).to be_kind_of(described_class)
-    end
-  end
-  
-  describe '#send_message' do
-    it 'sends the message to the SQS client created with the URL given to the constructor' do
-      fake_sqs_client = double('Client')
-      expect(Aws::SQS::Client).to receive(:new) { fake_sqs_client }
-      expect(fake_sqs_client).to receive(:send_message_batch).and_return(double(failed: []))
-      
-      conn = described_class.new('https://fake-queue.com')
-      expect(conn).to receive(:send_multiple_messages).and_call_original
-      conn.send_message('abcdef')
-    end
-    
-    it 'passes keyword args to Aws::SQS::Client' do
-      fake_sqs_client = double('Client')
-      expect(Aws::SQS::Client).to receive(:new) { fake_sqs_client }
-      expect(fake_sqs_client).to receive(:send_message_batch).and_return(double(failed: []))
-      
-      conn = described_class.new('https://fake-queue.com')
-      expect(conn).to receive(:send_multiple_messages).and_call_original
-      conn.send_message('abcdef', delay_seconds: 5)
-    end
-    
-    it 'retries on networking errors'
-  end
-  
-  describe '#send_multiple_messages' do
-    it 'sends 100 messages' do
-      fake_sqs_client = double('Client')
-      expect(Aws::SQS::Client).to receive(:new) { fake_sqs_client }
-      expect(fake_sqs_client).to receive(:send_message_batch).exactly(11).times {|kwargs|
-        expect(kwargs[:queue_url]).to eq("https://fake-queue.com")
-        expect(kwargs[:entries]).to be_kind_of(Array)
-        
-        entries = kwargs[:entries]
-        expect(entries.length).to be <= 10 # At most 10 messages per batch
-        entries.each do | entry |
-          expect(entry[:id]).to be_kind_of(String)
-          expect(entry[:message_body]).to be_kind_of(String)
-          expect(entry[:message_body]).to match(/Hello/)
-        end
-        double(failed: [])
-      }
-      
-      conn = described_class.new('https://fake-queue.com')
-      conn.send_multiple_messages do | b|
-        102.times { b.send_message("Hello - #{SecureRandom.uuid}") }
-      end
-    end
-    
-    it 'raises an exception if any message fails sending' do
-      fake_sqs_client = double('Client')
-      expect(Aws::SQS::Client).to receive(:new) { fake_sqs_client }
-      expect(fake_sqs_client).to receive(:send_message_batch) {|kwargs|
-        double(failed: [double(message: 'Something went wrong at AWS')])
-      }
-      
-      conn = described_class.new('https://fake-queue.com')
-      expect {
-        conn.send_multiple_messages do | b|
-          102.times { b.send_message("Hello - #{SecureRandom.uuid}") }
-        end
-      }.to raise_error(/messages failed to send/)
-    end
-    
-    it 'retries on networking errors'
-    
-  end
-  
-  describe '#delete_message' do
-    it 'deletes a single message'
-  end
-  
-  describe '#delete_multiple_messages' do
-    it 'deletes 100 messages' do
-      fake_sqs_client = double('Client')
-      expect(Aws::SQS::Client).to receive(:new) { fake_sqs_client }
-      expect(fake_sqs_client).to receive(:delete_message_batch).exactly(11).times {|kwargs|
-        expect(kwargs[:queue_url]).to eq("https://fake-queue.com")
-        expect(kwargs[:entries]).to be_kind_of(Array)
-        
-        entries = kwargs[:entries]
-        expect(entries.length).to be <= 10 # At most 10 messages per batch
-        entries.each do | entry |
-          expect(entry[:id]).to be_kind_of(String)
-          expect(entry[:receipt_handle]).to be_kind_of(String)
-        end
-        double(failed: [])
-      }
-      
-      conn = described_class.new('https://fake-queue.com')
-      conn.delete_multiple_messages do | b|
-        102.times { b.delete_message(SecureRandom.uuid) }
-      end
-    end
-    
-    it 'raises an exception if any message fails sending' do
-      fake_sqs_client = double('Client')
-      expect(Aws::SQS::Client).to receive(:new) { fake_sqs_client }
-      expect(fake_sqs_client).to receive(:delete_message_batch) {|kwargs|
-        double(failed: [double(message: 'Something went wrong at AWS')])
-      }
-      
-      conn = described_class.new('https://fake-queue.com')
-      expect {
-        conn.delete_multiple_messages do | b|
-          102.times { b.delete_message(SecureRandom.uuid) }
-        end
-      }.to raise_error(/messages failed to delete/)
-    end
-    
-    it 'retries on networking errors'
-  end
-  
-  describe '#receive_messages' do
-    it 'uses the batched receive feature' do
-      s = described_class.new('https://fake-queue')
-      
-      fake_sqs_client = double('Client')
-      expect(Aws::SQS::Client).to receive(:new) { fake_sqs_client }
-      
-      fake_messages = (1..5).map {
-        double(receipt_handle: SecureRandom.hex(4), body: SecureRandom.random_bytes(128))
-      }
-      fake_response = double(messages: fake_messages)
-      
-      expect(fake_sqs_client).to receive(:receive_message).with({:queue_url=>"https://fake-queue", :wait_time_seconds=>5, 
-          :max_number_of_messages=>10}).and_return(fake_response)
-      
-      messages = s.receive_messages
-      expect(messages.length).to eq(5)
-    end
-    
-    it 'retries on networking errors'
-  end
-end

data/spec/sqewer/execution_context_spec.rb

@@ -1,43 +0,0 @@
-require_relative '../spec_helper'
-
-describe Sqewer::ExecutionContext do
-  it 'offers a submit! that goes through the given Submitter argument' do
-    fake_submitter = double('Submitter')
-    expect(fake_submitter).to receive(:submit!).with(:fake_job, {})
-    
-    subject = described_class.new(fake_submitter)
-    subject.submit!(:fake_job)
-  end
-  
-  it 'offers arbitrary key/value storage' do
-    fake_submitter = double('Submitter')
-    subject = described_class.new(fake_submitter)
-    
-    subject['foo'] = 123
-    expect(subject['foo']).to eq(123)
-    expect(subject[:foo]).to eq(123)
-    expect(subject.fetch(:foo)).to eq(123)
-    
-    expect {
-      subject.fetch(:bar)
-    }.to raise_error(KeyError)
-    
-    default_value = subject.fetch(:bar) { 123 }
-    expect(default_value).to eq(123)
-  end
-  
-  it 'returns the NullLogger from #logger if no logger was passed to the constructor' do
-    fake_submitter = double('Submitter')
-    
-    subject = described_class.new(fake_submitter)
-    expect(subject.logger).to eq(Sqewer::NullLogger)
-  end
-  
-  it 'offers access to the given "logger" extra param if it was given to the constructor' do
-    fake_submitter = double('Submitter')
-    fake_logger = double('Logger')
-    
-    subject = described_class.new(fake_submitter, {'logger' => fake_logger})
-    expect(subject.logger).to eq(fake_logger)
-  end
-end