robot_sweatshop 0.3.3 → 0.4.0

data/bin/sweatshop-queue-watcher

@@ -1,18 +0,0 @@
-#!/usr/bin/env ruby
-require_relative 'lib/moneta-queue'
-
-@queues = []
-[ARGV[0], "mirror-#{ARGV[0]}"].each do |queue_name|
-  @queues.push(name: queue_name, queue: MonetaQueue.new(queue_name))
-end
-
-loop do
-  system 'clear'
-  @queues.each do |q|
-    puts "Queue: #{q[:name]}"
-    puts "Size: #{q[:queue].size}", "#{'|' * q[:queue].size}"
-    puts q[:queue].inspect
-    puts
-  end
-  sleep 0.1
-end

data/lib/robot_sweatshop/moneta-queue.rb

@@ -1,49 +0,0 @@
-#!/usr/bin/env ruby
-require 'moneta'
-require_relative 'config'
-
-# A class to handle queueing through Moneta's key-value storage
-class MonetaQueue
-  attr_reader :watched_queues
-
-  @@store = Moneta.new :File, dir: File.expand_path(configatron.moneta_path)
-
-  def initialize(name)
-    @name = name
-    @mirror_name = "mirror-#{name}"
-    @@store[@name] ||= []
-    @@store[@mirror_name] ||= []
-  end
-
-  def self.watched_queues
-    %w(payload parsed-payload jobs testing)
-  end
-
-  def enqueue(item)
-    @@store[@name] = @@store[@name].push item
-    @@store[@mirror_name] = @@store[@mirror_name].push item
-  end
-
-  def dequeue
-    return '' if @@store[@name].empty?
-    item = @@store[@name].first
-    @@store[@name] = @@store[@name][1..-1]
-    item
-  end
-
-  def size
-    loop do # Moneta can return nil sometimes, so we give it time to catch up
-      queue = @@store[@name]
-      return queue.size unless queue.nil?
-    end
-  end
-
-  def clear
-    @@store[@mirror_name] = []
-    @@store[@name] = []
-  end
-
-  def inspect
-    @@store[@name].inspect
-  end
-end

data/lib/robot_sweatshop/queue-helper.rb

@@ -1,32 +0,0 @@
-#!/usr/bin/env ruby
-require 'ezmq'
-require 'json'
-
-# A collection of common methods for queue interactions with EZMQ
-module QueueHelper
-  @@client = EZMQ::Client.new port: 5556
-
-  def self.dequeue(queue_name = 'default')
-    data = @@client.request queue_name
-    begin
-      JSON.parse data
-    rescue JSON::ParserError
-      nil
-    end
-  end
-
-  def self.enqueue(object = {}, to: 'default')
-    @@client.request "#{to} #{JSON.generate object}"
-  end
-
-  def self.wait_for(queue_name = 'default')
-    puts "Waiting for messages on #{queue_name}"
-    subscriber = EZMQ::Subscriber.new port: 5557, topic: 'busy-queues'
-    subscriber.listen do |message|
-      if message == queue_name
-        data = dequeue queue_name
-        yield data unless data.nil?
-      end
-    end
-  end
-end

data/robot_sweatshop.production.eye

@@ -1,7 +0,0 @@
-require 'configatron'
-
-configatron.temp do
-  configatron.eye.broadcaster_interval = ''
-  configatron.eye.worker_id = ''
-  Eye.load('robot_sweatshop.eye')
-end

data/robot_sweatshop.testing.eye

@@ -1,7 +0,0 @@
-require 'configatron'
-
-configatron.temp do
-  configatron.eye.broadcaster_interval = 0.0
-  configatron.eye.worker_id = 'testingid'
-  Eye.load('robot_sweatshop.eye')
-end

data/test/input_http_spec.rb

@@ -1,49 +0,0 @@
-require 'kintama'
-require 'ezmq'
-require 'json'
-require 'timeout'
-require 'http'
-require_relative 'shared/process_spawning'
-require_relative 'shared/helpers'
-
-given 'the HTTP Input' do
-  include QueueHelper
-  include InHelper
-
-  setup do
-    @subscriber = EZMQ::Subscriber.new port: 5557, topic: 'busy-queues'
-    @client = EZMQ::Client.new port: 5556
-    @payload_queue = 'payload'
-    clear_all_queues
-  end
-
-  %w(Bitbucket Github JSON Empty).each do |format|
-    context "POSTing #{format} data" do
-      setup do
-        url = input_http_url for_job: 'test_job'
-        HTTP.post url, body: example_raw_payload(of_format: format)
-      end
-
-      should 'enqueue to \'payload\'' do
-        Timeout.timeout($for_a_moment) do
-          @subscriber.listen do |message, topic|
-            break if message == @payload_queue
-          end
-        end
-      end
-
-      should 'enqueue payload details and format' do
-        response = @client.request "mirror-#{@payload_queue}"
-        data = JSON.parse response
-        assert_kind_of String, data['payload']
-        assert_kind_of String, data['user_agent']
-      end
-
-      should 'enqueue job name' do
-        response = @client.request "mirror-#{@payload_queue}"
-        data = JSON.parse response
-        assert_kind_of String, data['job_name']
-      end
-    end
-  end
-end

data/test/job_assembler_spec.rb

@@ -1,80 +0,0 @@
-require 'kintama'
-require 'ezmq'
-require 'json'
-require_relative 'shared/process_spawning'
-require_relative 'shared/helpers'
-
-describe 'the Job Assembler' do
-  include QueueHelper
-  include InHelper
-  include JobHelper
-
-  setup do
-    @client = EZMQ::Client.new port: 5556
-    @payloads_queue = 'payload'
-    @jobs_queue = 'jobs'
-    clear_all_queues
-  end
-
-  %w(Git JSON MinimalJob).each do |request|
-    given "#{request} requests in \'payload\'" do
-      setup do
-        payload = example_job_request of_type: request
-        @client.request "#{@payloads_queue} #{payload}"
-        sleep $for_a_moment
-      end
-
-      should 'remove the request from \'payload\'' do
-        response = @client.request @payloads_queue
-        assert_equal '', response
-      end
-
-      should 'enqueue commands, context, and job name to \'jobs\'' do
-        response = @client.request "mirror-#{@jobs_queue}"
-        response = JSON.load response
-        assert_kind_of Hash, response['context']
-        assert_kind_of Array, response['commands']
-        assert_kind_of String, response['job_name']
-      end
-
-      should 'store everything in the context as strings' do
-        response = @client.request "mirror-#{@jobs_queue}"
-        response = JSON.load response
-        response['context'].each { |_key, value| assert_kind_of String, value }
-      end
-
-      unless request == 'EmptyPayload'
-        should 'build the context with a parsed payload' do
-          response = @client.request "mirror-#{@jobs_queue}"
-          response = JSON.load response
-          assert_kind_of Hash, response['context']
-          if request == 'Git'
-            assert_equal 'develop', response['context']['branch']
-          else
-            assert_equal 'value', response['context']['test1']
-          end
-        end
-      end
-    end
-  end
-
-  %w(IgnoredBranch UnknownJob EmptyJob NonJSON).each do |request|
-    given "#{request} requests in \'payload\'" do
-      setup do
-        payload = example_job_request of_type: request
-        @client.request "#{@payloads_queue} #{payload}"
-        sleep $for_a_moment
-      end
-
-      should 'remove the request from \'payload\'' do
-        response = @client.request @payloads_queue
-        assert_equal '', response
-      end
-
-      should 'not queue anything to \'jobs\'' do
-        response = @client.request @jobs_queue
-        assert_equal '', response
-      end
-    end
-  end
-end

data/test/job_worker_spec.rb

@@ -1,65 +0,0 @@
-require 'kintama'
-require 'ezmq'
-require_relative 'shared/process_spawning'
-require_relative 'shared/helpers'
-
-describe 'the Worker' do
-  include QueueHelper
-  include JobHelper
-
-  setup do
-    @client = EZMQ::Client.new port: 5556
-    @jobs_queue = 'jobs'
-    @test_file = reset_test_file
-    clear_all_queues
-  end
-
-  given 'valid job data in \'jobs\'' do
-    setup do
-      job = example_job in_context: {custom: 'Hello world!'},
-                        with_commands: ['echo $custom','echo $custom > test.txt']
-      @client.request "#{@jobs_queue} #{job}"
-    end
-
-    should 'remove it from \'jobs\'' do
-      sleep $for_a_moment
-      response = @client.request @jobs_queue
-      assert_equal '', response
-    end
-
-    should 'run the dequeued job' do
-      sleep $for_io_calls
-      assert_equal true, File.file?(@test_file)
-    end
-
-    should 'run jobs with the context as environment variables' do
-      sleep $for_io_calls
-      assert_equal "Hello world!\n", File.read(@test_file)
-    end
-  end
-
-  given 'invalid job data in \'jobs\'' do
-    setup do
-      invalid_data = {
-        bad_context:  example_job(in_context: 'not hash'),
-        bad_commands: example_job(with_commands: 'echo 1'),
-        not_json:     'not_json'
-      }
-      invalid_data.each do |_type, datum|
-        @client.request "#{@jobs_queue} #{datum}"
-      end
-    end
-
-    should 'remove all of it from \'jobs\'' do
-      sleep $for_a_moment
-      response = @client.request @jobs_queue
-      assert_equal '', response
-    end
-
-    should 'not run anything' do
-      sleep $for_io_calls
-      response = @client.request @jobs_queue
-      assert_equal false, File.file?(@test_file)
-    end
-  end
-end

data/test/moneta-queue_spec.rb

@@ -1,49 +0,0 @@
-require 'bundler/setup'
-require 'kintama'
-require 'robot_sweatshop/moneta-queue'
-require_relative 'shared/helpers'
-
-given 'the Moneta Queue class' do
-  include QueueHelper
-
-  setup do
-    clear_all_queues
-  end
-
-  should 'return a list of actively watched queues' do
-    assert_kind_of Array, MonetaQueue.watched_queues
-  end
-
-  context 'an instance' do
-    setup do
-      @file_queue = MonetaQueue.new 'testing'
-      @file_queue.clear
-    end
-
-    should 'return size' do
-      assert_equal @file_queue.size, 0
-    end
-
-    should 'enqueue items' do
-      @file_queue.enqueue 'item'
-      assert_equal @file_queue.size, 1
-    end
-
-    should 'dequeue items' do
-      @file_queue.enqueue 'item1'
-      @file_queue.enqueue 'item2'
-      assert_equal @file_queue.size, 2
-      assert_equal @file_queue.dequeue, 'item1'
-    end
-
-    should 'clear items' do
-      @file_queue.enqueue 'item'
-      @file_queue.clear
-      assert_equal @file_queue.size, 0
-    end
-
-    should 'return the queue on inspect' do
-      assert_equal @file_queue.inspect, '[]'
-    end
-  end
-end

data/test/queue_broadcaster_spec.rb

@@ -1,39 +0,0 @@
-require 'kintama'
-require 'ezmq'
-require 'timeout'
-require_relative 'shared/process_spawning'
-require_relative 'shared/helpers'
-
-given 'the Queue Broadcaster' do
-  include QueueHelper
-
-  setup do
-    @subscriber = EZMQ::Subscriber.new port: 5557, topic: 'busy-queues'
-    @item = 'item'
-    @queue = 'testing'
-    clear_all_queues
-  end
-  
-  context 'a non-empty queue' do
-    setup { enqueue @queue, @item }
-
-    should 'have their named published to \'busy-queues\'' do
-      Timeout.timeout($for_a_while) do
-        @subscriber.listen do |message|
-          assert_equal @queue, message
-          break
-        end
-      end
-    end
-  end
-
-  context 'an empty queue' do
-    should 'not have their name published' do
-      assert_raises Timeout::Error do
-        Timeout.timeout($for_a_moment) do
-          @subscriber.listen {}
-        end
-      end
-    end
-  end
-end

data/test/queue_handler_spec.rb

@@ -1,54 +0,0 @@
-require 'kintama'
-require 'ezmq'
-require_relative 'shared/process_spawning'
-require_relative 'shared/helpers'
-
-given 'the Queue Handler' do
-  include QueueHelper
-
-  setup do
-    @client = EZMQ::Client.new port: 5556
-    @item = 'item'
-    @queue = 'testing'
-    clear_all_queues
-  end
-  
-  context 'dequeuing' do
-    setup { @request = "#{@queue}" }
-
-    should 'return the next queued item' do
-      enqueue @queue, @item
-      response = @client.request @request
-      assert_equal @item, response
-    end
-
-    should 'return \'\' for an empty queue' do
-      response = @client.request @request
-      assert_equal '', response
-    end
-  end
-
-  context 'enqueuing' do
-    setup { @request = "#{@queue} #{@item}" }
-
-    should 'return the queue new size' do
-      response = @client.request @request
-      assert_equal '1', response
-    end
-  end
-
-  context 'queue mirroring' do
-    should 'mirror queue enqueuing' do
-      enqueue @queue, @item
-      response = @client.request "mirror-#{@queue}"
-      assert_equal @item, response
-    end
-
-    should 'mirror queue clearing' do
-      enqueue @queue, @item
-      clear_all_queues
-      response = @client.request "mirror-#{@queue}"
-      assert_equal '', response
-    end
-  end
-end

data/test/run_all.rb

@@ -1,3 +0,0 @@
-Dir.glob("#{__dir__}/*_spec.rb").each do |spec_file|
-  require_relative spec_file
-end

data/test/shared/process_spawning.rb

@@ -1,16 +0,0 @@
-$for_a_moment = 0.2
-$for_a_while = 0.5
-$for_io_calls = 1
-$for_everything = 2
-
-Kintama.on_start do
-  puts `#{__dir__}/../../bin/sweatshop start --testing`
-  %w(test minimal git empty).each do |job_file|
-    FileUtils.cp "#{__dir__}/../data/#{job_file}_job.yaml", File.expand_path(configatron.job_path)
-  end
-  sleep $for_everything
-end
-
-Kintama.on_finish do
-  puts `#{__dir__}/../../bin/sweatshop stop`
-end