sqewer 1.0.0

data/spec/sqewer/simple_job_spec.rb

@@ -0,0 +1,69 @@
+require_relative '../spec_helper'
+
+describe Sqewer::SimpleJob do
+  it 'raises a clear error for an unknown attribute' do
+    example_class = Class.new do
+      attr_accessor :foo, :bar
+      include Sqewer::SimpleJob
+    end
+    
+    expect {
+      example_class.new(zoo: 1, bar: 2)
+    }.to raise_error(/Unknown attribute \:zoo for/)
+  end
+  
+  it 'uses defined accessors to provide decent string representation' do
+    example_class = Class.new do
+      attr_accessor :foo, :bar
+      include Sqewer::SimpleJob
+    end
+    
+    job = example_class.new(foo: 1, bar: 2)
+    expect(job.inspect).to include('Class')
+    expect(job.inspect).to include('{:foo=>1, :bar=>2}')
+  end
+  
+  it 'uses inspectable_attributes to limit the scope of .inspect' do
+    example_class = Class.new do
+      attr_accessor :foo, :bar
+      def inspectable_attributes
+        [:foo]
+      end
+      include Sqewer::SimpleJob
+    end
+    
+    job = example_class.new(foo: 1, bar: 2)
+    expect(job.inspect).to include('Class')
+    expect(job.inspect).to include('{:foo=>1}')
+    expect(job.inspect).not_to include('bar')
+  end
+  
+  it 'provides for a keyword argument constructor and a to_h method' do
+    example_class = Class.new do
+      attr_accessor :foo, :bar
+      include Sqewer::SimpleJob
+    end
+    
+    string_repr = example_class.to_s
+    
+    new_instance = example_class.new(foo: 1, bar: 2)
+    
+    expect(new_instance.foo).to eq(1)
+    expect(new_instance.bar).to eq(2)
+    
+    hash_repr = new_instance.to_h
+    expect(hash_repr).to eq({foo: 1, bar: 2})
+  end
+  
+  
+  it 'raises if arguments are forgotten' do
+    example_class = Class.new do
+      attr_accessor :foo, :bar
+      include Sqewer::SimpleJob
+    end
+    
+    expect {
+      example_class.new(foo: 1)
+    }.to raise_error('Missing job attribute :bar')
+  end
+end

data/spec/sqewer/submitter_spec.rb

@@ -0,0 +1,59 @@
+require_relative '../spec_helper'
+
+describe Sqewer::Submitter do
+  describe '.default' do
+    it 'returns a set up Submitter with the configured Connection and Serializer' do
+      expect(ENV).to receive(:fetch).with('SQS_QUEUE_URL').and_return('https://some-queue.aws.com')
+      
+      s = described_class.default
+      expect(s.connection).to respond_to(:send_message)
+      expect(s.serializer).to respond_to(:serialize)
+    end
+  end
+  
+  describe '#initialize' do
+    it 'creates a Submitter that you can submit jobs through' do
+      fake_serializer = double('Some serializer')
+      allow(fake_serializer).to receive(:serialize) {|object_to_serialize|
+        expect(object_to_serialize).not_to be_nil
+        'serialized-object-data'
+      }
+      
+      fake_connection = double('Some SQS connection')
+      expect(fake_connection).to receive(:send_message).at_least(5).times.with('serialized-object-data', {})
+      
+      subject = described_class.new(fake_connection, fake_serializer)
+      5.times { subject.submit!(:some_object) }
+    end
+    
+    it 'passes the keyword arguments to send_message on the connection' do
+      fake_serializer = double('Some serializer')
+      allow(fake_serializer).to receive(:serialize) {|object_to_serialize|
+        expect(object_to_serialize).not_to be_nil
+        'serialized-object-data'
+      }
+      
+      fake_connection = double('Some SQS connection')
+      expect(fake_connection).to receive(:send_message).with('serialized-object-data', {delay_seconds: 5})
+      
+      subject = described_class.new(fake_connection, fake_serializer)
+      subject.submit!(:some_object, delay_seconds: 5)
+    end
+    
+    
+    # TODO: should use batched sends
+    it 'unsplats the array of jobs and sends them one by one' do
+      fake_serializer = double('Some serializer')
+      expect(fake_serializer).to receive(:serialize).at_least(5).times {|object_to_serialize|
+        expect(object_to_serialize).not_to be_nil
+        'serialized-object-data'
+      }
+      
+      fake_connection = double('Some SQS connection')
+      expect(fake_connection).to receive(:send_message).at_least(5).times.with('serialized-object-data', {})
+      
+      subject = described_class.new(fake_connection, fake_serializer)
+      subject.submit!(:one, :two, :three, :four, :five)
+    end
+  end
+end

data/spec/sqewer/worker_spec.rb

@@ -0,0 +1,130 @@
+require_relative '../spec_helper'
+
+describe Sqewer::Worker, :sqs => true do
+  let(:silent_logger) { Logger.new(StringIO.new('')) }
+  
+  it 'supports .default' do
+    default_worker = described_class.default
+    expect(default_worker).to respond_to(:start)
+    expect(default_worker).to respond_to(:stop)
+  end
+    
+  it 'instantiates a Logger to STDERR by default' do
+    expect(Logger).to receive(:new).with(STDERR)
+    worker = described_class.new
+  end
+  
+  it 'can go through the full cycle of initialize, start, stop, start, stop' do
+    worker = described_class.new(logger: silent_logger)
+    worker.start
+    worker.stop
+    worker.start
+    worker.stop
+  end
+  
+  it 'raises a state exception if being stopped without being started' do
+    worker = described_class.new
+    expect {
+      worker.stop
+    }.to raise_error(/Cannot change state/)
+  end
+  
+  context 'when the job payload cannot be unserialized from JSON due to invalid syntax' do
+    it 'is able to cope with an exception when the job class is unknown (one of generic exceptions)' do
+      client = Aws::SQS::Client.new
+      client.send_message(queue_url: ENV.fetch('SQS_QUEUE_URL'), message_body: '{"foo":')
+      
+      logger_output = ''
+      logger_to_string = Logger.new(StringIO.new(logger_output))
+    
+      worker = described_class.new(logger: logger_to_string)
+      
+      worker.start
+      sleep 2
+      worker.stop
+      
+      expect(logger_output).to include('unexpected token at \'{"foo":')
+      expect(logger_output).to include('Stopping (clean shutdown)')
+    end
+  end
+  
+  context 'when the job cannot be instantiated due to an unknown class' do
+    it 'is able to cope with an exception when the job class is unknown (one of generic exceptions)' do
+      payload = JSON.dump({job_class: 'UnknownJobClass', arg1: 'some value'})
+    
+      client = Aws::SQS::Client.new
+      client.send_message(queue_url: ENV.fetch('SQS_QUEUE_URL'), message_body: payload)
+      
+      logger_output = ''
+      logger_to_string = Logger.new(StringIO.new(logger_output))
+    
+      worker = described_class.new(logger: logger_to_string)
+      
+      worker.start
+      sleep 2
+      worker.stop
+      
+      expect(logger_output).to include('uninitialized constant UnknownJobClass')
+      expect(logger_output).to include('Stopping (clean shutdown)')
+    end
+  end
+  
+  context 'with a job that spawns another job' do
+    it 'sets up the processing pipeline so that jobs can execute in sequence' do
+      class SecondaryJob
+        def run
+          File.open('secondary-job-run','w') {}
+        end
+      end
+      
+      class InitialJob
+        def run(executor)
+          File.open('initial-job-run','w') {}
+          executor.submit!(SecondaryJob.new)
+        end
+      end
+      
+      payload = JSON.dump({job_class: 'InitialJob'})
+      client = Aws::SQS::Client.new
+      client.send_message(queue_url: ENV.fetch('SQS_QUEUE_URL'), message_body: payload)
+      
+      logger_output = ''
+      logger_to_string = Logger.new(StringIO.new(logger_output))
+      worker = described_class.new(logger: logger_to_string, num_threads: 8)
+      
+      worker.start
+      
+      begin
+        poll(fail_after: 3) { File.exist?('initial-job-run') }
+        poll(fail_after: 3) { File.exist?('secondary-job-run') }
+        
+        File.unlink('initial-job-run')
+        File.unlink('secondary-job-run')
+        expect(true).to eq(true)
+      ensure
+        worker.stop
+      end
+      
+      # Run with a per-process isolator too
+      client = Aws::SQS::Client.new
+      client.send_message(queue_url: ENV.fetch('SQS_QUEUE_URL'), message_body: payload)
+      
+      logger_output = ''
+      logger_to_string = Logger.new(StringIO.new(logger_output))
+      worker = described_class.new(logger: logger_to_string, num_threads: 8, isolator: Sqewer::Isolator.process)
+      
+      worker.start
+      
+      begin
+        poll(fail_after: 3) { File.exist?('initial-job-run') }
+        poll(fail_after: 3) { File.exist?('secondary-job-run') }
+        
+        File.unlink('initial-job-run')
+        File.unlink('secondary-job-run')
+        expect(true).to eq(true)
+      ensure
+        worker.stop
+      end
+    end
+  end
+end

data/sqewer.gemspec

@@ -0,0 +1,108 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+# stub: sqewer 1.0.0 ruby lib
+
+Gem::Specification.new do |s|
+  s.name = "sqewer"
+  s.version = "1.0.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.require_paths = ["lib"]
+  s.authors = ["Julik Tarkhanov"]
+  s.date = "2016-01-18"
+  s.description = "Process jobs from SQS"
+  s.email = "me@julik.nl"
+  s.extra_rdoc_files = [
+    "README.md"
+  ]
+  s.files = [
+    ".gitlab-ci.yml",
+    ".yardopts",
+    "DETAILS.md",
+    "FAQ.md",
+    "Gemfile",
+    "README.md",
+    "Rakefile",
+    "example.env",
+    "lib/sqewer.rb",
+    "lib/sqewer/atomic_counter.rb",
+    "lib/sqewer/cli.rb",
+    "lib/sqewer/connection.rb",
+    "lib/sqewer/contrib/appsignal_wrapper.rb",
+    "lib/sqewer/contrib/performable.rb",
+    "lib/sqewer/execution_context.rb",
+    "lib/sqewer/isolator.rb",
+    "lib/sqewer/middleware_stack.rb",
+    "lib/sqewer/null_logger.rb",
+    "lib/sqewer/serializer.rb",
+    "lib/sqewer/simple_job.rb",
+    "lib/sqewer/submitter.rb",
+    "lib/sqewer/version.rb",
+    "lib/sqewer/worker.rb",
+    "spec/conveyor_belt_spec.rb",
+    "spec/spec_helper.rb",
+    "spec/sqewer/atomic_counter_spec.rb",
+    "spec/sqewer/cli_app.rb",
+    "spec/sqewer/cli_spec.rb",
+    "spec/sqewer/connection_spec.rb",
+    "spec/sqewer/execution_context_spec.rb",
+    "spec/sqewer/middleware_stack_spec.rb",
+    "spec/sqewer/serializer_spec.rb",
+    "spec/sqewer/simple_job_spec.rb",
+    "spec/sqewer/submitter_spec.rb",
+    "spec/sqewer/worker_spec.rb",
+    "sqewer.gemspec"
+  ]
+  s.homepage = "https://gitlab.wetransfer.net/julik/sqewer"
+  s.licenses = ["MIT"]
+  s.rubygems_version = "2.2.2"
+  s.summary = "A full-featured library for all them worker needs"
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 4
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<aws-sdk>, ["~> 2"])
+      s.add_runtime_dependency(%q<very_tiny_state_machine>, ["~> 1"])
+      s.add_runtime_dependency(%q<hash_tools>, [">= 0"])
+      s.add_runtime_dependency(%q<exceptional_fork>, [">= 0"])
+      s.add_development_dependency(%q<ks>, [">= 0"])
+      s.add_development_dependency(%q<dotenv>, [">= 0"])
+      s.add_development_dependency(%q<rake>, [">= 0"])
+      s.add_development_dependency(%q<rspec>, ["~> 3.2.0"])
+      s.add_development_dependency(%q<simplecov>, [">= 0"])
+      s.add_development_dependency(%q<rdoc>, ["~> 3.12"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0"])
+      s.add_development_dependency(%q<jeweler>, ["~> 2.0.1"])
+    else
+      s.add_dependency(%q<aws-sdk>, ["~> 2"])
+      s.add_dependency(%q<very_tiny_state_machine>, ["~> 1"])
+      s.add_dependency(%q<hash_tools>, [">= 0"])
+      s.add_dependency(%q<exceptional_fork>, [">= 0"])
+      s.add_dependency(%q<ks>, [">= 0"])
+      s.add_dependency(%q<dotenv>, [">= 0"])
+      s.add_dependency(%q<rake>, [">= 0"])
+      s.add_dependency(%q<rspec>, ["~> 3.2.0"])
+      s.add_dependency(%q<simplecov>, [">= 0"])
+      s.add_dependency(%q<rdoc>, ["~> 3.12"])
+      s.add_dependency(%q<bundler>, ["~> 1.0"])
+      s.add_dependency(%q<jeweler>, ["~> 2.0.1"])
+    end
+  else
+    s.add_dependency(%q<aws-sdk>, ["~> 2"])
+    s.add_dependency(%q<very_tiny_state_machine>, ["~> 1"])
+    s.add_dependency(%q<hash_tools>, [">= 0"])
+    s.add_dependency(%q<exceptional_fork>, [">= 0"])
+    s.add_dependency(%q<ks>, [">= 0"])
+    s.add_dependency(%q<dotenv>, [">= 0"])
+    s.add_dependency(%q<rake>, [">= 0"])
+    s.add_dependency(%q<rspec>, ["~> 3.2.0"])
+    s.add_dependency(%q<simplecov>, [">= 0"])
+    s.add_dependency(%q<rdoc>, ["~> 3.12"])
+    s.add_dependency(%q<bundler>, ["~> 1.0"])
+    s.add_dependency(%q<jeweler>, ["~> 2.0.1"])
+  end
+end
+

metadata

@@ -0,0 +1,248 @@
+--- !ruby/object:Gem::Specification
+name: sqewer
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Julik Tarkhanov
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-01-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: very_tiny_state_machine
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: hash_tools
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: exceptional_fork
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ks
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: dotenv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.2.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.2.0
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.12'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.0.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.0.1
+description: Process jobs from SQS
+email: me@julik.nl
+executables: []
+extensions: []
+extra_rdoc_files:
+- README.md
+files:
+- ".gitlab-ci.yml"
+- ".yardopts"
+- DETAILS.md
+- FAQ.md
+- Gemfile
+- README.md
+- Rakefile
+- example.env
+- lib/sqewer.rb
+- lib/sqewer/atomic_counter.rb
+- lib/sqewer/cli.rb
+- lib/sqewer/connection.rb
+- lib/sqewer/contrib/appsignal_wrapper.rb
+- lib/sqewer/contrib/performable.rb
+- lib/sqewer/execution_context.rb
+- lib/sqewer/isolator.rb
+- lib/sqewer/middleware_stack.rb
+- lib/sqewer/null_logger.rb
+- lib/sqewer/serializer.rb
+- lib/sqewer/simple_job.rb
+- lib/sqewer/submitter.rb
+- lib/sqewer/version.rb
+- lib/sqewer/worker.rb
+- spec/conveyor_belt_spec.rb
+- spec/spec_helper.rb
+- spec/sqewer/atomic_counter_spec.rb
+- spec/sqewer/cli_app.rb
+- spec/sqewer/cli_spec.rb
+- spec/sqewer/connection_spec.rb
+- spec/sqewer/execution_context_spec.rb
+- spec/sqewer/middleware_stack_spec.rb
+- spec/sqewer/serializer_spec.rb
+- spec/sqewer/simple_job_spec.rb
+- spec/sqewer/submitter_spec.rb
+- spec/sqewer/worker_spec.rb
+- sqewer.gemspec
+homepage: https://gitlab.wetransfer.net/julik/sqewer
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: A full-featured library for all them worker needs
+test_files: []