robot_sweatshop 0.3.3 → 0.4.0

data/robot_sweatshop.eye

@@ -1,5 +1,6 @@
 require 'yaml'
 
+# TODO: per-user temp file
 CONFIG = YAML.load_file '/tmp/.robot_sweatshop-eye-config.yaml'
 PID_PATH = CONFIG[:pidfile_path]
 LOG_PATH = CONFIG[:logfile_path]
@@ -13,46 +14,43 @@ Eye.application :robot_sweatshop do
   check :cpu, every: 10.seconds, below: 100, times: 3
   working_dir CONFIG[:working_path]
 
-  group 'input' do
-    process :http do
-      pid_file "#{PID_PATH}/input-http.pid"
-      stdall "#{LOG_PATH}/input-http.log"
-      start_command "#{__dir__}/bin/sweatshop-input-http"
+  group 'services' do
+    process :job_dictionary do
+      pid_file "#{PID_PATH}/job-dictionary.pid"
+      stdall "#{LOG_PATH}/job-dictionary.log"
+      start_command "#{__dir__}/bin/sweatshop-job-dictionary"
       daemonize true
     end
-  end
-  group 'queue' do
-    process :handler do
-      pid_file "#{PID_PATH}/queue-handler.pid"
-      stdall "#{LOG_PATH}/queue-handler.log"
-      start_command "#{__dir__}/bin/sweatshop-queue-handler"
+    process :payload_parser do
+      pid_file "#{PID_PATH}/payload-parser.pid"
+      stdall "#{LOG_PATH}/payload-parser.log"
+      start_command "#{__dir__}/bin/sweatshop-payload-parser"
       daemonize true
     end
-    process :broadcaster do
-      pid_file "#{PID_PATH}/queue-broadcaster.pid"
-      stdall "#{LOG_PATH}/queue-broadcaster.log"
-      start_command "#{__dir__}/bin/sweatshop-queue-broadcaster #{configatron.eye.broadcaster_interval}"
+    process :conveyor do
+      pid_file "#{PID_PATH}/conveyor.pid"
+      stdall "#{LOG_PATH}/conveyor.log"
+      start_command "#{__dir__}/bin/sweatshop-conveyor"
       daemonize true
     end
   end
-  group 'job' do
-    process :assembler do
-      pid_file "#{PID_PATH}/job-assembler.pid"
-      stdall "#{LOG_PATH}/job-assembler.log"
-      start_command "#{__dir__}/bin/sweatshop-job-assembler"
-      daemonize true
-    end
-    process :worker do
-      pid_file "#{PID_PATH}/job-worker.pid"
-      stdall "#{LOG_PATH}/job-worker.log"
-      start_command "#{__dir__}/bin/sweatshop-job-worker #{configatron.eye.worker_id}"
-      daemonize true
-    end
+
+  process :input do
+    pid_file "#{PID_PATH}/input.pid"
+    stdall "#{LOG_PATH}/input.log"
+    start_command "#{__dir__}/bin/sweatshop-input"
+    daemonize true
+  end
+  process :assembler do
+    pid_file "#{PID_PATH}/assembler.pid"
+    stdall "#{LOG_PATH}/assembler.log"
+    start_command "#{__dir__}/bin/sweatshop-assembler"
+    daemonize true
   end
-  process :payload_parser do
-    pid_file "#{PID_PATH}/payload_parser.pid"
-    stdall "#{LOG_PATH}/payload_parser.log"
-    start_command "#{__dir__}/bin/sweatshop-payload-parser"
+  process :worker do
+    pid_file "#{PID_PATH}/worker.pid"
+    stdall "#{LOG_PATH}/worker.log"
+    start_command "#{__dir__}/bin/sweatshop-worker"
     daemonize true
   end
 end

data/robot_sweatshop.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |gem|
   gem.name        = 'robot_sweatshop'
-  gem.version     = '0.3.3'
+  gem.version     = '0.4.0'
   gem.licenses    = 'MIT'
   gem.authors     = ['Justin Scott']
   gem.email       = 'jvscott@gmail.com'
@@ -15,20 +15,22 @@ Gem::Specification.new do |gem|
 
   gem.required_ruby_version = '>= 2.1'
 
-  gem.add_runtime_dependency 'sinatra'
-  gem.add_runtime_dependency 'ezmq'
+  gem.add_runtime_dependency 'bundler'
   gem.add_runtime_dependency 'faker'
   gem.add_runtime_dependency 'commander'
   gem.add_runtime_dependency 'eye'
   gem.add_runtime_dependency 'colorize' # TODO: replace with rainbow
   gem.add_runtime_dependency 'configatron'
-  gem.add_runtime_dependency 'moneta'
+  gem.add_runtime_dependency 'moneta' # TODO: needed with stubborn_queue?
   gem.add_runtime_dependency 'contracts'
+  gem.add_runtime_dependency 'sinatra'
+  gem.add_runtime_dependency 'ezmq'
+  gem.add_runtime_dependency 'stubborn_queue'
+  gem.add_runtime_dependency 'oj'
+  gem.add_runtime_dependency 'exponential-backoff'
 
   gem.add_development_dependency 'rake'
   gem.add_development_dependency 'kintama'
   gem.add_development_dependency 'http'
-  gem.add_development_dependency 'pry'
-  gem.add_development_dependency 'pry-byebug'
   gem.add_development_dependency 'simplecov'
 end

data/test/README.md

@@ -1 +1,3 @@
-You can run individual `_spec.rb` files with Ruby or `run_all.rb` to go through each.
+You can run individual `_spec.rb` files with Ruby or `all.rb` to go through each. There's no way to run everything together with the way processes are set up but this hasn't been a problem so far.
+
+Because I'm lazy, TestProcess::Stub uses the UNIX `lsof` to check if ports are in use. There's probably a better, more portable way to do this.

data/test/all.rb

@@ -0,0 +1,35 @@
+# Because of needing to isolate processes, I can't run all tests in one call.
+# This means that I have multiple sets of results which isn't nice, but the
+#   tests are largely for the sake of TDD anyway.
+
+def run_test(component)
+  test_path = File.expand_path __dir__
+  IO.popen("ruby #{test_path}/#{component}_spec.rb") do |data|
+    while line = data.gets
+      puts line
+    end
+  end
+  $?.exitstatus
+end
+
+def non_zero(array)
+  array.select { |number| number != 0 }
+end
+
+tests = %w(
+  input
+  conveyor
+  payload_parser
+  job_dictionary
+  assembler
+  worker
+  end-to-end
+)
+exit_statuses = []
+
+tests.each do |name|
+  exit_statuses.push run_test(name)
+end
+exit 1 unless non_zero(exit_statuses).empty?
+
+puts "Everything passed successfully" # TODO: announce gem

data/test/assembler_spec.rb

@@ -0,0 +1,89 @@
+require 'bundler/setup'
+require 'kintama'
+require 'ezmq'
+require 'timeout'
+require 'robot_sweatshop/config'
+require 'robot_sweatshop/connections'
+require_relative 'shared/scaffolding'
+require_relative 'shared/helpers'
+$stdout.sync = true
+
+Kintama.on_start do
+  Setup.empty_conveyor
+  @pids = TestProcess.start %w(assembler conveyor payload-parser job-dictionary)
+  Setup.populate_test_jobs
+end
+
+Kintama.on_finish do
+  TestProcess.stop @pids
+end
+
+describe 'the Job Assembler' do
+  include InputHelper
+  include OutputHelper
+  using ExtendedEZMQ
+
+  setup do
+    @worker = TestProcess.stub :worker
+    # @conveyor = TestProcess::Stub.new 'Server', on_port: configatron.conveyor_port
+    @client = EZMQ::Client.new port: configatron.conveyor_port
+    @client.serialize_with_json!
+  end
+
+  teardown do
+    @client.close
+  end
+
+  %w(Git JSON MinimalJob).each do |request|
+    given "#{request} requests on the Conveyor" do
+      setup do
+        @client.request conveyor_enqueue(request)
+        Timeout.timeout($a_while) { loop while @worker.output_empty? }
+        @worker_data = eval File.read(@worker.output_file) # TODO: oh god why
+      end
+
+      should 'push the parsed payload to a Worker' do
+        assert_kind_of Hash, @worker_data[:context]
+        assert_kind_of Array, @worker_data[:commands]
+        assert_kind_of String, @worker_data[:job_name]
+      end
+
+      should 'store everything in the context as strings' do
+        @worker_data[:context].each do |key, value|
+          assert_kind_of String, key
+          assert_kind_of String, value
+        end
+      end
+
+      should 'build the context with a parsed payload' do
+        if request == 'Git'
+          assert_equal 'develop', @worker_data[:context]['branch']
+        else
+          assert_equal 'value', @worker_data[:context]['test1']
+        end
+      end
+
+      should 'grab commands from the job config' do
+        assert_match /echo/, @worker_data[:commands].first
+      end
+    end
+  end
+
+  %w(IgnoredBranch UnknownJob EmptyJob NonJSON).each do |request|
+    given "#{request} requests on the Conveyor" do
+      setup do
+        @client.request conveyor_enqueue(request)
+        sleep $a_moment
+      end
+
+      should 'push nothing to Workers' do
+        assert @worker.output_empty?
+      end
+
+      should 'tell the Conveyor that the job is finished'
+      # testing needs a real conveyor to grab the jobs from
+      # so it would require much refactoring and complexity to stub a fake one
+      # in order to test that finish is actually called
+    end
+  end
+end

data/test/conveyor_spec.rb

@@ -0,0 +1,67 @@
+require 'bundler/setup'
+require 'kintama'
+require 'ezmq'
+require 'timeout'
+require 'robot_sweatshop/config'
+require 'robot_sweatshop/connections'
+require_relative 'shared/scaffolding'
+$stdout.sync = true
+
+Kintama.on_start do
+  Setup.empty_conveyor
+  @pids = TestProcess.start %w(conveyor)
+end
+
+Kintama.on_finish do
+  TestProcess.stop @pids
+end
+
+describe 'the Conveyor' do
+  using ExtendedEZMQ
+
+  setup do
+    @client = EZMQ::Client.new port: configatron.conveyor_port
+    @client.serialize_with_json!
+    @item = {test: 'item'}
+  end
+
+  teardown do
+    @client.close
+  end
+
+  should 'enqueue and dequeue items' do
+    id = Timeout.timeout($a_moment) do
+      @client.request method: 'enqueue', data: @item
+      @client.request method: 'dequeue'
+    end
+    assert_kind_of Fixnum, id
+  end
+
+  should 'lookup items by ID' do
+    item = Timeout.timeout($a_moment) do
+      @client.request method: 'enqueue', data: @item
+      id = @client.request method: 'dequeue'
+      @client.request method: 'lookup', data: id
+    end
+    assert_equal @item, item
+  end
+
+  should 'return nothing when given invalid data' do
+    response = Timeout.timeout($a_moment) do
+      @client.request 'not json'
+      @client.request method: 'invalid'
+      @client.request ''
+      @client.request 'assurance that the server is still up'
+    end
+    assert_nil response
+  end
+
+  should 'finish items by ID to prevent requeueing' do
+    response = Timeout.timeout($a_moment) do
+      @client.request method: 'enqueue', data: @item
+      id = @client.request method: 'dequeue'
+      @client.request method: 'finish', data: id
+    end
+    assert_not_equal '', response
+  end
+end

data/test/data/weird_job.yaml

@@ -0,0 +1,6 @@
+---
+what_info_is:
+- this?
+
+environment:
+  WORLD: earth

data/test/end-to-end_spec.rb

@@ -1,29 +1,35 @@
+require 'bundler/setup'
 require 'kintama'
 require 'ezmq'
 require 'http'
-require_relative 'shared/process_spawning'
+require_relative 'shared/scaffolding'
 require_relative 'shared/helpers'
 
 describe 'Robot Sweatshop' do
-  include QueueHelper
-  include InHelper
-  include JobHelper
+  include InputHelper
+  include OutputHelper
 
   setup do
-    @client = EZMQ::Client.new port: 5556
-    @test_file = reset_test_file
-    clear_all_queues
+    Setup.empty_conveyor
+    @pids = TestProcess.start %w(input conveyor payload-parser job-dictionary assembler worker)
+    Setup.populate_test_jobs
+    sleep $a_while
   end
 
-  context "POST git data to the HTTP Input" do
+  teardown do
+    TestProcess.stop @pids
+  end
+
+  context "POSTing data to the HTTP Input" do
     setup do
-      url = input_http_url for_job: 'test_job'
-      HTTP.post url, body: example_raw_payload(of_format: 'JSON')
-      sleep $for_io_calls
+      clear_worker_output
+      url = input_url for_job: 'test_job'
+      Timeout.timeout($a_while) { @response = HTTP.post url, body: example_raw_payload('JSON') }
     end
 
-    should 'run jobs with the context as environment variables' do
-      assert_equal "success\n", File.read(@test_file)
+    should 'run the appropriate job' do
+      Timeout.timeout($a_while) { loop until File.exist? worker_output }
+      assert_equal true, File.file?(worker_output)
     end
   end
 end

data/test/input_spec.rb

@@ -0,0 +1,49 @@
+require 'bundler/setup'
+require 'kintama'
+require 'ezmq'
+require 'oj'
+require 'timeout'
+require 'http'
+require 'robot_sweatshop/config'
+require_relative 'shared/scaffolding'
+require_relative 'shared/helpers'
+$stdout.sync = true
+
+Kintama.on_start do
+  @pids = TestProcess.start %w(input)
+  sleep $a_while
+end
+
+Kintama.on_finish do
+  TestProcess.stop @pids
+end
+
+given 'the HTTP Input' do
+  include InputHelper
+  include OutputHelper
+
+  %w(Bitbucket Github JSON Empty).each do |format|
+    context "POSTing #{format} data" do
+      setup do
+        @conveyor = TestProcess.stub :conveyor
+        url = input_url for_job: 'test_job'
+        Timeout.timeout($a_while) do
+          @response = HTTP.post url, body: example_raw_payload(format)
+        end
+        assert_equal 200, @response.code
+      end
+
+      should 'send to the conveyor' do
+        assert_equal true, File.exist?(@conveyor.output_file)
+      end
+
+      should 'enqueue payload details, user agent, and job name' do
+        request = eval File.read(@conveyor.output_file) # please don't hack me thx
+        assert_equal 'enqueue', request[:method]
+        assert_not_nil request[:data][:payload]
+        assert_not_nil request[:data][:user_agent]
+        assert_not_nil request[:data][:job_name]
+      end
+    end
+  end
+end

data/test/job_dictionary_spec.rb

@@ -0,0 +1,70 @@
+require 'bundler/setup'
+require 'kintama'
+require 'ezmq'
+require 'timeout'
+require 'robot_sweatshop/config'
+require 'robot_sweatshop/connections'
+require_relative 'shared/scaffolding'
+require_relative 'shared/helpers' # do I need this?
+$stdout.sync = true
+
+Kintama.on_start do
+  @pids = TestProcess.start %w(job-dictionary)
+  Setup.populate_test_jobs
+end
+
+Kintama.on_finish do
+  TestProcess.stop @pids
+end
+
+describe 'the Job Dictionary' do
+  include InputHelper
+  using ExtendedEZMQ
+
+  setup do
+    @client = EZMQ::Client.new port: configatron.job_dictionary_port
+    @client.serialize_with_json!
+  end
+
+  teardown do
+    @client.close
+  end
+
+  %w(git_job minimal_job test_job).each do |job_name|
+    given "a #{job_name.gsub '_', ' '}" do
+      setup do
+        @response = Timeout.timeout($a_while) do
+          @client.request job_name
+        end
+      end
+
+      should 'return no error' do
+        assert_equal true, @response[:error].empty?
+      end
+
+      should 'return a defined job object' do
+        assert_kind_of Hash, @response[:data]
+        no_environment = job_name =~ /empty|minimal/
+        has_whitelist = job_name =~ /git/
+        assert_not_nil @response[:data]['branch_whitelist'] if has_whitelist
+        assert_not_nil @response[:data]['commands']
+        assert_not_nil @response[:data]['environment'] unless no_environment
+      end
+    end
+  end
+
+  %w(empty_job undefined_job weird_job).each do |job_name|
+    given "a #{job_name.gsub '_', ' '}" do
+      setup do
+        @response = Timeout.timeout($a_while) do
+          @client.request job_name
+        end
+      end
+
+      should 'return an error object' do
+        assert_kind_of String, @response[:error]
+        assert_equal true, @response[:data].empty?
+      end
+    end
+  end
+end

data/test/payload_parser_spec.rb

@@ -1,42 +1,54 @@
+require 'bundler/setup'
 require 'kintama'
 require 'ezmq'
-require 'json'
 require 'timeout'
-require_relative 'shared/process_spawning'
+require 'robot_sweatshop/config'
+require 'robot_sweatshop/connections'
+require_relative 'shared/scaffolding'
 require_relative 'shared/helpers'
+$stdout.sync = true
+
+Kintama.on_start do
+  @pids = TestProcess.start %w(payload-parser)
+end
+
+Kintama.on_finish do
+  TestProcess.stop @pids
+end
 
 describe 'the Payload Parser' do
-  include QueueHelper
-  include InHelper
+  include InputHelper
+  using ExtendedEZMQ
 
   setup do
     @client = EZMQ::Client.new port: configatron.payload_parser_port
+    @client.serialize_with_json!
+  end
+
+  teardown do
+    @client.close
   end
 
   %w(Bitbucket Github JSON Empty).each do |format|
     given "valid #{format} payloads" do
       setup do
-        payload = example_payload_request of_format: format
-        @response = Timeout.timeout($for_a_moment) do
-          @client.request JSON.dump(payload)
+        payload = payload_parser_request format
+        @response = Timeout.timeout($a_while) do
+          @client.request payload
         end
       end
 
+      should 'return no error' do
+        assert_equal true, @response[:error].empty?
+      end
+
       should 'return a parsed payload object' do
-        @response = JSON.parse @response
-        assert_equal true, @response['error'].empty?
-        assert_kind_of Hash, @response['payload']
-
-        keys = case format
-        when 'JSON'
-          %w(test1 test2)
-        when 'Empty'
-          []
-        else
-          Payload.hash_keys
-        end
+        assert_kind_of Hash, @response[:data]
+        keys = Payload.hash_keys
+        keys = %w(test1 test2) if format == 'JSON'
+        keys = [] if format == 'Empty'
         keys.each do |key|
-          payload = @response['payload']
+          payload = @response[:data]
           assert_not_nil payload[key]
           assert_not_equal key, payload[key] # catches "string"[key]
         end
@@ -47,16 +59,15 @@ describe 'the Payload Parser' do
   %w(NonJSON).each do |format|
     given "#{format} payloads" do
       setup do
-        payload = example_raw_payload of_format: format
-        @response = Timeout.timeout($for_a_moment) do
-          @client.request "#{payload}"
+        payload = payload_parser_request format
+        @response = Timeout.timeout($a_while) do
+          @client.request payload
         end
       end
 
       should 'return an error object' do
-        @response = JSON.load(@response)
-        assert_kind_of String, @response['error']
-        assert_equal true, @response['payload'].empty?
+        assert_kind_of String, @response[:error]
+        assert_equal true, @response[:data].empty?
       end
     end
   end

data/test/shared/helpers/input.rb

@@ -0,0 +1,73 @@
+require 'bundler/setup'
+require 'yaml'
+require 'oj'
+require 'robot_sweatshop/payload'
+require 'robot_sweatshop/config'
+
+module InputHelper
+  def example_raw_payload(format)
+    payload_strings = YAML.load_file "#{__dir__}/../../data/payload_data.yaml"
+    payload_strings[format.downcase]
+  end
+
+  def input_url(for_job: 'test_job')
+    "http://localhost:#{configatron.http_port}/run/#{for_job}"
+  end
+
+  def conveyor_enqueue(type)
+    format, job = payload_configuration type
+    {
+      method: 'enqueue',
+      data: {
+        payload: example_raw_payload(format),
+        user_agent: user_agent_for(format),
+        job_name: job
+      }
+    }
+  end
+
+  def payload_parser_request(format)
+    {
+      payload: example_raw_payload(format),
+      user_agent: user_agent_for(format)
+    }
+  end
+
+  def worker_push
+    {
+      commands: [
+        'echo $custom',
+        'echo $custom > test.txt'
+      ],
+      context: {'custom' => 'hello world'},
+      job_name: 'test_job',
+      job_id: 1
+    }
+  end
+
+  def payload_configuration(type)
+    format = 'JSON'
+    format = 'Bitbucket' if type == 'Git' # develop branch
+    format = 'Github' if type == 'IgnoredBranch' # master branch
+    format = 'NonJSON' if type == 'NonJSON'
+
+    job = 'test_job'
+    job = 'git_job' if type == 'Git' || type == 'IgnoredBranch'
+    job = 'minimal_job' if type == 'MinimalJob'
+    job = 'unknown_job' if type == 'UnknownJob' # TODO: why does commenting this out still pass payload parser?!
+    job = 'empty_job' if type == 'EmptyJob'
+
+    [format, job]
+  end
+
+  def user_agent_for(format)
+    case format
+    when 'Bitbucket'
+      'Bitbucket.org'
+    when 'Github'
+      'Github-Hookshot'
+    else
+      'SomeUserAgent'
+    end
+  end
+end

data/test/shared/helpers/output.rb

@@ -0,0 +1,14 @@
+require 'bundler/setup'
+require 'fileutils'
+require 'robot_sweatshop/payload'
+require 'robot_sweatshop/config'
+
+module OutputHelper
+  def worker_output
+    "#{configatron.workspace_path}/test_job-testingid/test.txt"
+  end
+
+  def clear_worker_output
+    FileUtils.rm_rf worker_output
+  end
+end