robot_sweatshop 0.1.2

data/kintama/job_assembler_spec.rb

@@ -0,0 +1,72 @@
+require 'kintama'
+require 'ezmq'
+require 'json'
+require_relative 'shared/process_spawning'
+require_relative 'shared/helpers'
+
+describe 'the Job Assembler' do
+  include QueueHelper
+  include PayloadHelper
+  include JobHelper
+
+  setup do
+    @client = EZMQ::Client.new port: 5556
+    @parsed_payloads_queue = 'parsed-payload'
+    @jobs_queue = 'jobs'
+    clear_all_queues
+  end
+
+  given 'valid parsed payload data in \'parsed-payload\'' do
+    setup do
+      payload = example_parsed_payload(for_branch: 'develop')
+      @client.request "#{@parsed_payloads_queue} #{payload}"
+      sleep $for_a_while
+    end
+
+    should 'remove it from \'parsed-payload\'' do
+      response = @client.request @parsed_payloads_queue
+      assert_equal '', response
+    end
+
+    should 'enqueue commands, context, and job name to \'jobs\'' do
+      response = @client.request "mirror-#{@jobs_queue}"
+      response = JSON.parse response
+      assert_kind_of Hash, response['context']
+      assert_kind_of Array, response['commands']
+      assert_kind_of String, response['job_name']
+    end
+
+    should 'only enqueue string objects to context' do
+      response = @client.request "mirror-#{@jobs_queue}"
+      response = JSON.parse response
+      response['context'].each do |_key, value|
+        assert_kind_of String, value
+      end
+    end
+  end
+
+  given 'invalid job data in \'parsed-payload\'' do
+    setup do
+      invalid_data = {
+        ignored_branch: example_parsed_payload(for_branch: 'not_on_whitelist'),
+        bad_payload:    example_parsed_payload(with_payload: 'not hash'),
+        bad_job:        example_parsed_payload(for_job: 'asdf'),
+        not_json:       'not_json'
+      }
+      invalid_data.each do |_type, datum|
+        @client.request "#{@parsed_payloads_queue} #{datum}"
+      end
+      sleep $for_a_while
+    end
+
+    should 'remove all of it from \'parsed-payload\'' do
+      response = @client.request @parsed_payloads_queue
+      assert_equal '', response
+    end
+
+    should 'not queue anything to \'jobs\'' do
+      response = @client.request @jobs_queue
+      assert_equal '', response
+    end
+  end
+end

data/kintama/job_worker_spec.rb

@@ -0,0 +1,65 @@
+require 'kintama'
+require 'ezmq'
+require_relative 'shared/process_spawning'
+require_relative 'shared/helpers'
+
+describe 'the Worker' do
+  include QueueHelper
+  include JobHelper
+
+  setup do
+    @client = EZMQ::Client.new port: 5556
+    @jobs_queue = 'jobs'
+    @test_file = reset_test_file
+    clear_all_queues
+  end
+
+  given 'valid job data in \'jobs\'' do
+    setup do
+      job = example_job in_context: {custom: 'Hello world!'},
+                        with_commands: ['echo $custom','echo $custom > test.txt']
+      @client.request "#{@jobs_queue} #{job}"
+    end
+
+    should 'remove it from \'jobs\'' do
+      sleep $for_a_moment
+      response = @client.request @jobs_queue
+      assert_equal '', response
+    end
+
+    should 'run the dequeued job' do
+      sleep $for_io_calls
+      assert_equal true, File.file?(@test_file)
+    end
+
+    should 'run jobs with the context as environment variables' do
+      sleep $for_io_calls
+      assert_equal "Hello world!\n", File.read(@test_file)
+    end
+  end
+
+  given 'invalid job data in \'jobs\'' do
+    setup do
+      invalid_data = {
+        bad_context:  example_job(in_context: 'not hash'),
+        bad_commands: example_job(with_commands: 'echo 1'),
+        not_json:     'not_json'
+      }
+      invalid_data.each do |_type, datum|
+        @client.request "#{@jobs_queue} #{datum}"
+      end
+    end
+
+    should 'remove all of it from \'jobs\'' do
+      sleep $for_a_moment
+      response = @client.request @jobs_queue
+      assert_equal '', response
+    end
+
+    should 'not run anything' do
+      sleep $for_io_calls
+      response = @client.request @jobs_queue
+      assert_equal false, File.file?(@test_file)
+    end
+  end
+end

data/kintama/moneta-queue_spec.rb

@@ -0,0 +1,48 @@
+require 'kintama'
+require_relative 'shared/helpers'
+require_relative '../lib/queue/lib/moneta-queue'
+
+given 'the Moneta Queue class' do
+  include QueueHelper
+
+  setup do
+    clear_all_queues
+  end
+
+  should 'return a list of actively watched queues' do
+    assert_kind_of Array, MonetaQueue.watched_queues
+  end
+
+  context 'an instance' do
+    setup do
+      @file_queue = MonetaQueue.new 'testing'
+      @file_queue.clear
+    end
+
+    should 'return size' do
+      assert_equal @file_queue.size, 0
+    end
+
+    should 'enqueue items' do
+      @file_queue.enqueue 'item'
+      assert_equal @file_queue.size, 1
+    end
+
+    should 'dequeue items' do
+      @file_queue.enqueue 'item1'
+      @file_queue.enqueue 'item2'
+      assert_equal @file_queue.size, 2
+      assert_equal @file_queue.dequeue, 'item1'
+    end
+
+    should 'clear items' do
+      @file_queue.enqueue 'item'
+      @file_queue.clear
+      assert_equal @file_queue.size, 0
+    end
+
+    should 'return the queue on inspect' do
+      assert_equal @file_queue.inspect, '[]'
+    end
+  end
+end

data/kintama/payload_parser_spec.rb

@@ -0,0 +1,71 @@
+require 'kintama'
+require 'ezmq'
+require 'json'
+require_relative 'shared/process_spawning'
+require_relative 'shared/helpers'
+
+describe 'the Payload Parser' do
+  include QueueHelper
+  include InHelper
+  include PayloadHelper
+
+  setup do
+    @client = EZMQ::Client.new port: 5556
+    @raw_queue = 'raw-payload'
+    @parsed_queue = 'parsed-payload'
+    clear_all_queues
+  end
+
+  # TODO: refactor and add github
+  ['Bitbucket'].each do |format|
+    given "valid #{format} data in 'raw-payload'" do
+      setup do
+        payload = example_raw_payload(with_format: format)
+        @client.request "#{@raw_queue} #{payload}"
+        sleep $for_a_while
+      end
+
+      should 'remove it from \'raw-payload\'' do
+        response = @client.request @raw_queue
+        assert_equal '', response
+      end
+
+      should 'enqueue parsed payload data and job name to \'parsed-payload\'' do
+        response = @client.request "mirror-#{@parsed_queue}"
+        response = JSON.parse response
+
+        assert_kind_of Hash, response['payload']
+        Payload.hash_keys.each do |key|
+          assert_not_nil response['payload'][key]
+          assert_not_equal key, response['payload'][key] # important for how Ruby interprets "string"['key']
+        end
+        assert_kind_of String, response['job_name']
+      end
+    end
+  end
+
+  given 'invalid payload data in \'raw-payload\'' do
+    setup do
+      invalid_data = {
+        malformed_payload:  example_raw_payload(with_format: 'malformed'),
+        unsupported_format: example_raw_payload(with_format: 'asdf'),
+        not_json:           'not_json'
+      }
+      invalid_data.each do |_type, datum|
+        @client.request "#{@raw_queue} #{datum}"
+      end
+      sleep $for_a_while
+      # TODO: should not crash the payload parser
+    end
+
+    should 'remove all of it from \'raw-payload\'' do
+      response = @client.request @raw_queue
+      assert_equal '', response
+    end
+
+    should 'not queue anything to \'parsed-payload\'' do
+      response = @client.request @parsed_queue
+      assert_equal '', response
+    end
+  end
+end

data/kintama/queue_broadcaster_spec.rb

@@ -0,0 +1,39 @@
+require 'kintama'
+require 'ezmq'
+require 'timeout'
+require_relative 'shared/process_spawning'
+require_relative 'shared/helpers'
+
+given 'the Queue Broadcaster' do
+  include QueueHelper
+
+  setup do
+    @subscriber = EZMQ::Subscriber.new port: 5557, topic: 'busy-queues'
+    @item = 'item'
+    @queue = 'testing'
+    clear_all_queues
+  end
+  
+  context 'a non-empty queue' do
+    setup { enqueue @queue, @item }
+
+    should 'have their named published to \'busy-queues\'' do
+      Timeout.timeout($for_a_while) do
+        @subscriber.listen do |message|
+          assert_equal @queue, message
+          break
+        end
+      end
+    end
+  end
+
+  context 'an empty queue' do
+    should 'not have their name published' do
+      assert_raises Timeout::Error do
+        Timeout.timeout($for_a_moment) do
+          @subscriber.listen {}
+        end
+      end
+    end
+  end
+end

data/kintama/queue_handler_spec.rb

@@ -0,0 +1,54 @@
+require 'kintama'
+require 'ezmq'
+require_relative 'shared/process_spawning'
+require_relative 'shared/helpers'
+
+given 'the Queue Handler' do
+  include QueueHelper
+
+  setup do
+    @client = EZMQ::Client.new port: 5556
+    @item = 'item'
+    @queue = 'testing'
+    clear_all_queues
+  end
+  
+  context 'dequeuing' do
+    setup { @request = "#{@queue}" }
+
+    should 'return the next queued item' do
+      enqueue @queue, @item
+      response = @client.request @request
+      assert_equal @item, response
+    end
+
+    should 'return \'\' for an empty queue' do
+      response = @client.request @request
+      assert_equal '', response
+    end
+  end
+
+  context 'enqueuing' do
+    setup { @request = "#{@queue} #{@item}" }
+
+    should 'return the queue new size' do
+      response = @client.request @request
+      assert_equal '1', response
+    end
+  end
+
+  context 'queue mirroring' do
+    should 'mirror queue enqueuing' do
+      enqueue @queue, @item
+      response = @client.request "mirror-#{@queue}"
+      assert_equal @item, response
+    end
+
+    should 'mirror queue clearing' do
+      enqueue @queue, @item
+      clear_all_queues
+      response = @client.request "mirror-#{@queue}"
+      assert_equal '', response
+    end
+  end
+end

data/kintama/run_all.rb

@@ -0,0 +1,3 @@
+Dir.glob("#{__dir__}/*_spec.rb").each do |spec_file|
+  require_relative spec_file
+end

data/kintama/shared/helpers.rb

@@ -0,0 +1,55 @@
+require 'yaml'
+require 'json'
+require_relative '../../lib/queue/lib/moneta-queue'
+require_relative '../../lib/payload/lib/payload'
+require_relative '../../config'
+
+module QueueHelper
+  def clear_all_queues
+    MonetaQueue.watched_queues.each do |queue|
+      queue = MonetaQueue.new queue
+      queue.clear
+    end
+  end
+  def enqueue(queue_name, item)
+    queue = MonetaQueue.new queue_name
+    queue.enqueue item
+  end
+end
+
+module InHelper
+  def load_payload(of_format)
+    payload_strings = YAML.load_file "#{__dir__}/../data/payload_data.yaml"
+    payload_strings[of_format.downcase]
+  end
+  def example_raw_payload(with_format:)
+    payload = load_payload with_format
+    JSON.generate payload: payload,
+                  format: with_format,
+                  job_name: 'example'
+  end
+  def input_http_url(for_job: 'example', in_format: 'bitbucket')
+    "http://localhost:#{configatron.input.http.port}/#{in_format}/payload-for/#{for_job}"
+  end
+end
+
+module PayloadHelper
+  def example_parsed_payload(with_payload: nil, for_branch: 'develop', for_job: 'example')
+    payload = with_payload || { branch: for_branch }
+    JSON.generate payload: payload,
+                  job_name: for_job
+  end
+end
+
+module JobHelper
+  def example_job(in_context: {}, with_commands: [])
+    JSON.generate context: in_context,
+                  commands: with_commands,
+                  job_name: 'example'
+  end
+  def reset_test_file
+    test_file = File.expand_path "#{__dir__}/../../workspaces/example-testingid/test.txt"
+    FileUtils.rm_rf test_file
+    test_file
+  end
+end

data/kintama/shared/process_spawning.rb

@@ -0,0 +1,13 @@
+$for_a_moment = 0.1
+$for_a_while = 0.5
+$for_io_calls = 1
+$for_everything = 3
+
+Kintama.on_start do
+  puts `#{__dir__}/../../bin/sweatshop start --testing`
+  sleep $for_everything
+end
+
+Kintama.on_finish do
+  puts `#{__dir__}/../../bin/sweatshop stop`
+end

data/lib/README.md

@@ -0,0 +1,12 @@
+Job lifecycle:
+
+```
+in-* -> raw-payloads { payload:, format:, job_name: } ->
+payload-parser -> parsed-payloads { payload:, job_name: } ->
+job-assembler -> jobs { context:, commands:, job_name: } ->
+job-worker
+```
+
+All passing done via the moneta core in queue/*
+
+Also queue/watcher for debugging