robot_sweatshop 0.3.3 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/README.md +11 -11
- data/Rakefile +1 -1
- data/bin/README.md +23 -0
- data/bin/sweatshop +1 -1
- data/bin/sweatshop-assembler +101 -0
- data/bin/sweatshop-conveyor +64 -0
- data/bin/{sweatshop-input-http → sweatshop-input} +13 -7
- data/bin/sweatshop-job-dictionary +53 -0
- data/bin/sweatshop-payload-parser +28 -28
- data/bin/sweatshop-worker +58 -0
- data/config.defaults.yaml +6 -4
- data/lib/README.md +10 -13
- data/lib/robot_sweatshop/cli/common.rb +1 -1
- data/lib/robot_sweatshop/cli/config.rb +1 -1
- data/lib/robot_sweatshop/cli/job.rb +1 -1
- data/lib/robot_sweatshop/cli/start.rb +2 -2
- data/lib/robot_sweatshop/config.rb +2 -0
- data/lib/robot_sweatshop/connections.rb +17 -0
- data/lib/robot_sweatshop/create-config-directories.rb +0 -1
- data/lib/robot_sweatshop.rb +1 -2
- data/robot_sweatshop.eye +30 -32
- data/robot_sweatshop.gemspec +8 -6
- data/test/README.md +3 -1
- data/test/all.rb +35 -0
- data/test/assembler_spec.rb +89 -0
- data/test/conveyor_spec.rb +67 -0
- data/test/data/weird_job.yaml +6 -0
- data/test/end-to-end_spec.rb +19 -13
- data/test/input_spec.rb +49 -0
- data/test/job_dictionary_spec.rb +70 -0
- data/test/payload_parser_spec.rb +37 -26
- data/test/shared/helpers/input.rb +73 -0
- data/test/shared/helpers/output.rb +14 -0
- data/test/shared/helpers.rb +4 -83
- data/test/shared/scaffolding.rb +42 -0
- data/test/shared/stub.rb +44 -0
- data/test/worker_spec.rb +59 -0
- metadata +76 -44
- data/bin/sweatshop-job-assembler +0 -82
- data/bin/sweatshop-job-worker +0 -45
- data/bin/sweatshop-queue-broadcaster +0 -20
- data/bin/sweatshop-queue-handler +0 -24
- data/bin/sweatshop-queue-watcher +0 -18
- data/lib/robot_sweatshop/moneta-queue.rb +0 -49
- data/lib/robot_sweatshop/queue-helper.rb +0 -32
- data/robot_sweatshop.production.eye +0 -7
- data/robot_sweatshop.testing.eye +0 -7
- data/test/input_http_spec.rb +0 -49
- data/test/job_assembler_spec.rb +0 -80
- data/test/job_worker_spec.rb +0 -65
- data/test/moneta-queue_spec.rb +0 -49
- data/test/queue_broadcaster_spec.rb +0 -39
- data/test/queue_handler_spec.rb +0 -54
- data/test/run_all.rb +0 -3
- data/test/shared/process_spawning.rb +0 -16
data/bin/sweatshop-queue-watcher
DELETED
@@ -1,18 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
require_relative 'lib/moneta-queue'
|
3
|
-
|
4
|
-
@queues = []
|
5
|
-
[ARGV[0], "mirror-#{ARGV[0]}"].each do |queue_name|
|
6
|
-
@queues.push(name: queue_name, queue: MonetaQueue.new(queue_name))
|
7
|
-
end
|
8
|
-
|
9
|
-
loop do
|
10
|
-
system 'clear'
|
11
|
-
@queues.each do |q|
|
12
|
-
puts "Queue: #{q[:name]}"
|
13
|
-
puts "Size: #{q[:queue].size}", "#{'|' * q[:queue].size}"
|
14
|
-
puts q[:queue].inspect
|
15
|
-
puts
|
16
|
-
end
|
17
|
-
sleep 0.1
|
18
|
-
end
|
@@ -1,49 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
require 'moneta'
|
3
|
-
require_relative 'config'
|
4
|
-
|
5
|
-
# A class to handle queueing through Moneta's key-value storage
|
6
|
-
class MonetaQueue
|
7
|
-
attr_reader :watched_queues
|
8
|
-
|
9
|
-
@@store = Moneta.new :File, dir: File.expand_path(configatron.moneta_path)
|
10
|
-
|
11
|
-
def initialize(name)
|
12
|
-
@name = name
|
13
|
-
@mirror_name = "mirror-#{name}"
|
14
|
-
@@store[@name] ||= []
|
15
|
-
@@store[@mirror_name] ||= []
|
16
|
-
end
|
17
|
-
|
18
|
-
def self.watched_queues
|
19
|
-
%w(payload parsed-payload jobs testing)
|
20
|
-
end
|
21
|
-
|
22
|
-
def enqueue(item)
|
23
|
-
@@store[@name] = @@store[@name].push item
|
24
|
-
@@store[@mirror_name] = @@store[@mirror_name].push item
|
25
|
-
end
|
26
|
-
|
27
|
-
def dequeue
|
28
|
-
return '' if @@store[@name].empty?
|
29
|
-
item = @@store[@name].first
|
30
|
-
@@store[@name] = @@store[@name][1..-1]
|
31
|
-
item
|
32
|
-
end
|
33
|
-
|
34
|
-
def size
|
35
|
-
loop do # Moneta can return nil sometimes, so we give it time to catch up
|
36
|
-
queue = @@store[@name]
|
37
|
-
return queue.size unless queue.nil?
|
38
|
-
end
|
39
|
-
end
|
40
|
-
|
41
|
-
def clear
|
42
|
-
@@store[@mirror_name] = []
|
43
|
-
@@store[@name] = []
|
44
|
-
end
|
45
|
-
|
46
|
-
def inspect
|
47
|
-
@@store[@name].inspect
|
48
|
-
end
|
49
|
-
end
|
@@ -1,32 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
require 'ezmq'
|
3
|
-
require 'json'
|
4
|
-
|
5
|
-
# A collection of common methods for queue interactions with EZMQ
|
6
|
-
module QueueHelper
|
7
|
-
@@client = EZMQ::Client.new port: 5556
|
8
|
-
|
9
|
-
def self.dequeue(queue_name = 'default')
|
10
|
-
data = @@client.request queue_name
|
11
|
-
begin
|
12
|
-
JSON.parse data
|
13
|
-
rescue JSON::ParserError
|
14
|
-
nil
|
15
|
-
end
|
16
|
-
end
|
17
|
-
|
18
|
-
def self.enqueue(object = {}, to: 'default')
|
19
|
-
@@client.request "#{to} #{JSON.generate object}"
|
20
|
-
end
|
21
|
-
|
22
|
-
def self.wait_for(queue_name = 'default')
|
23
|
-
puts "Waiting for messages on #{queue_name}"
|
24
|
-
subscriber = EZMQ::Subscriber.new port: 5557, topic: 'busy-queues'
|
25
|
-
subscriber.listen do |message|
|
26
|
-
if message == queue_name
|
27
|
-
data = dequeue queue_name
|
28
|
-
yield data unless data.nil?
|
29
|
-
end
|
30
|
-
end
|
31
|
-
end
|
32
|
-
end
|
data/robot_sweatshop.testing.eye
DELETED
data/test/input_http_spec.rb
DELETED
@@ -1,49 +0,0 @@
|
|
1
|
-
require 'kintama'
|
2
|
-
require 'ezmq'
|
3
|
-
require 'json'
|
4
|
-
require 'timeout'
|
5
|
-
require 'http'
|
6
|
-
require_relative 'shared/process_spawning'
|
7
|
-
require_relative 'shared/helpers'
|
8
|
-
|
9
|
-
given 'the HTTP Input' do
|
10
|
-
include QueueHelper
|
11
|
-
include InHelper
|
12
|
-
|
13
|
-
setup do
|
14
|
-
@subscriber = EZMQ::Subscriber.new port: 5557, topic: 'busy-queues'
|
15
|
-
@client = EZMQ::Client.new port: 5556
|
16
|
-
@payload_queue = 'payload'
|
17
|
-
clear_all_queues
|
18
|
-
end
|
19
|
-
|
20
|
-
%w(Bitbucket Github JSON Empty).each do |format|
|
21
|
-
context "POSTing #{format} data" do
|
22
|
-
setup do
|
23
|
-
url = input_http_url for_job: 'test_job'
|
24
|
-
HTTP.post url, body: example_raw_payload(of_format: format)
|
25
|
-
end
|
26
|
-
|
27
|
-
should 'enqueue to \'payload\'' do
|
28
|
-
Timeout.timeout($for_a_moment) do
|
29
|
-
@subscriber.listen do |message, topic|
|
30
|
-
break if message == @payload_queue
|
31
|
-
end
|
32
|
-
end
|
33
|
-
end
|
34
|
-
|
35
|
-
should 'enqueue payload details and format' do
|
36
|
-
response = @client.request "mirror-#{@payload_queue}"
|
37
|
-
data = JSON.parse response
|
38
|
-
assert_kind_of String, data['payload']
|
39
|
-
assert_kind_of String, data['user_agent']
|
40
|
-
end
|
41
|
-
|
42
|
-
should 'enqueue job name' do
|
43
|
-
response = @client.request "mirror-#{@payload_queue}"
|
44
|
-
data = JSON.parse response
|
45
|
-
assert_kind_of String, data['job_name']
|
46
|
-
end
|
47
|
-
end
|
48
|
-
end
|
49
|
-
end
|
data/test/job_assembler_spec.rb
DELETED
@@ -1,80 +0,0 @@
|
|
1
|
-
require 'kintama'
|
2
|
-
require 'ezmq'
|
3
|
-
require 'json'
|
4
|
-
require_relative 'shared/process_spawning'
|
5
|
-
require_relative 'shared/helpers'
|
6
|
-
|
7
|
-
describe 'the Job Assembler' do
|
8
|
-
include QueueHelper
|
9
|
-
include InHelper
|
10
|
-
include JobHelper
|
11
|
-
|
12
|
-
setup do
|
13
|
-
@client = EZMQ::Client.new port: 5556
|
14
|
-
@payloads_queue = 'payload'
|
15
|
-
@jobs_queue = 'jobs'
|
16
|
-
clear_all_queues
|
17
|
-
end
|
18
|
-
|
19
|
-
%w(Git JSON MinimalJob).each do |request|
|
20
|
-
given "#{request} requests in \'payload\'" do
|
21
|
-
setup do
|
22
|
-
payload = example_job_request of_type: request
|
23
|
-
@client.request "#{@payloads_queue} #{payload}"
|
24
|
-
sleep $for_a_moment
|
25
|
-
end
|
26
|
-
|
27
|
-
should 'remove the request from \'payload\'' do
|
28
|
-
response = @client.request @payloads_queue
|
29
|
-
assert_equal '', response
|
30
|
-
end
|
31
|
-
|
32
|
-
should 'enqueue commands, context, and job name to \'jobs\'' do
|
33
|
-
response = @client.request "mirror-#{@jobs_queue}"
|
34
|
-
response = JSON.load response
|
35
|
-
assert_kind_of Hash, response['context']
|
36
|
-
assert_kind_of Array, response['commands']
|
37
|
-
assert_kind_of String, response['job_name']
|
38
|
-
end
|
39
|
-
|
40
|
-
should 'store everything in the context as strings' do
|
41
|
-
response = @client.request "mirror-#{@jobs_queue}"
|
42
|
-
response = JSON.load response
|
43
|
-
response['context'].each { |_key, value| assert_kind_of String, value }
|
44
|
-
end
|
45
|
-
|
46
|
-
unless request == 'EmptyPayload'
|
47
|
-
should 'build the context with a parsed payload' do
|
48
|
-
response = @client.request "mirror-#{@jobs_queue}"
|
49
|
-
response = JSON.load response
|
50
|
-
assert_kind_of Hash, response['context']
|
51
|
-
if request == 'Git'
|
52
|
-
assert_equal 'develop', response['context']['branch']
|
53
|
-
else
|
54
|
-
assert_equal 'value', response['context']['test1']
|
55
|
-
end
|
56
|
-
end
|
57
|
-
end
|
58
|
-
end
|
59
|
-
end
|
60
|
-
|
61
|
-
%w(IgnoredBranch UnknownJob EmptyJob NonJSON).each do |request|
|
62
|
-
given "#{request} requests in \'payload\'" do
|
63
|
-
setup do
|
64
|
-
payload = example_job_request of_type: request
|
65
|
-
@client.request "#{@payloads_queue} #{payload}"
|
66
|
-
sleep $for_a_moment
|
67
|
-
end
|
68
|
-
|
69
|
-
should 'remove the request from \'payload\'' do
|
70
|
-
response = @client.request @payloads_queue
|
71
|
-
assert_equal '', response
|
72
|
-
end
|
73
|
-
|
74
|
-
should 'not queue anything to \'jobs\'' do
|
75
|
-
response = @client.request @jobs_queue
|
76
|
-
assert_equal '', response
|
77
|
-
end
|
78
|
-
end
|
79
|
-
end
|
80
|
-
end
|
data/test/job_worker_spec.rb
DELETED
@@ -1,65 +0,0 @@
|
|
1
|
-
require 'kintama'
|
2
|
-
require 'ezmq'
|
3
|
-
require_relative 'shared/process_spawning'
|
4
|
-
require_relative 'shared/helpers'
|
5
|
-
|
6
|
-
describe 'the Worker' do
|
7
|
-
include QueueHelper
|
8
|
-
include JobHelper
|
9
|
-
|
10
|
-
setup do
|
11
|
-
@client = EZMQ::Client.new port: 5556
|
12
|
-
@jobs_queue = 'jobs'
|
13
|
-
@test_file = reset_test_file
|
14
|
-
clear_all_queues
|
15
|
-
end
|
16
|
-
|
17
|
-
given 'valid job data in \'jobs\'' do
|
18
|
-
setup do
|
19
|
-
job = example_job in_context: {custom: 'Hello world!'},
|
20
|
-
with_commands: ['echo $custom','echo $custom > test.txt']
|
21
|
-
@client.request "#{@jobs_queue} #{job}"
|
22
|
-
end
|
23
|
-
|
24
|
-
should 'remove it from \'jobs\'' do
|
25
|
-
sleep $for_a_moment
|
26
|
-
response = @client.request @jobs_queue
|
27
|
-
assert_equal '', response
|
28
|
-
end
|
29
|
-
|
30
|
-
should 'run the dequeued job' do
|
31
|
-
sleep $for_io_calls
|
32
|
-
assert_equal true, File.file?(@test_file)
|
33
|
-
end
|
34
|
-
|
35
|
-
should 'run jobs with the context as environment variables' do
|
36
|
-
sleep $for_io_calls
|
37
|
-
assert_equal "Hello world!\n", File.read(@test_file)
|
38
|
-
end
|
39
|
-
end
|
40
|
-
|
41
|
-
given 'invalid job data in \'jobs\'' do
|
42
|
-
setup do
|
43
|
-
invalid_data = {
|
44
|
-
bad_context: example_job(in_context: 'not hash'),
|
45
|
-
bad_commands: example_job(with_commands: 'echo 1'),
|
46
|
-
not_json: 'not_json'
|
47
|
-
}
|
48
|
-
invalid_data.each do |_type, datum|
|
49
|
-
@client.request "#{@jobs_queue} #{datum}"
|
50
|
-
end
|
51
|
-
end
|
52
|
-
|
53
|
-
should 'remove all of it from \'jobs\'' do
|
54
|
-
sleep $for_a_moment
|
55
|
-
response = @client.request @jobs_queue
|
56
|
-
assert_equal '', response
|
57
|
-
end
|
58
|
-
|
59
|
-
should 'not run anything' do
|
60
|
-
sleep $for_io_calls
|
61
|
-
response = @client.request @jobs_queue
|
62
|
-
assert_equal false, File.file?(@test_file)
|
63
|
-
end
|
64
|
-
end
|
65
|
-
end
|
data/test/moneta-queue_spec.rb
DELETED
@@ -1,49 +0,0 @@
|
|
1
|
-
require 'bundler/setup'
|
2
|
-
require 'kintama'
|
3
|
-
require 'robot_sweatshop/moneta-queue'
|
4
|
-
require_relative 'shared/helpers'
|
5
|
-
|
6
|
-
given 'the Moneta Queue class' do
|
7
|
-
include QueueHelper
|
8
|
-
|
9
|
-
setup do
|
10
|
-
clear_all_queues
|
11
|
-
end
|
12
|
-
|
13
|
-
should 'return a list of actively watched queues' do
|
14
|
-
assert_kind_of Array, MonetaQueue.watched_queues
|
15
|
-
end
|
16
|
-
|
17
|
-
context 'an instance' do
|
18
|
-
setup do
|
19
|
-
@file_queue = MonetaQueue.new 'testing'
|
20
|
-
@file_queue.clear
|
21
|
-
end
|
22
|
-
|
23
|
-
should 'return size' do
|
24
|
-
assert_equal @file_queue.size, 0
|
25
|
-
end
|
26
|
-
|
27
|
-
should 'enqueue items' do
|
28
|
-
@file_queue.enqueue 'item'
|
29
|
-
assert_equal @file_queue.size, 1
|
30
|
-
end
|
31
|
-
|
32
|
-
should 'dequeue items' do
|
33
|
-
@file_queue.enqueue 'item1'
|
34
|
-
@file_queue.enqueue 'item2'
|
35
|
-
assert_equal @file_queue.size, 2
|
36
|
-
assert_equal @file_queue.dequeue, 'item1'
|
37
|
-
end
|
38
|
-
|
39
|
-
should 'clear items' do
|
40
|
-
@file_queue.enqueue 'item'
|
41
|
-
@file_queue.clear
|
42
|
-
assert_equal @file_queue.size, 0
|
43
|
-
end
|
44
|
-
|
45
|
-
should 'return the queue on inspect' do
|
46
|
-
assert_equal @file_queue.inspect, '[]'
|
47
|
-
end
|
48
|
-
end
|
49
|
-
end
|
@@ -1,39 +0,0 @@
|
|
1
|
-
require 'kintama'
|
2
|
-
require 'ezmq'
|
3
|
-
require 'timeout'
|
4
|
-
require_relative 'shared/process_spawning'
|
5
|
-
require_relative 'shared/helpers'
|
6
|
-
|
7
|
-
given 'the Queue Broadcaster' do
|
8
|
-
include QueueHelper
|
9
|
-
|
10
|
-
setup do
|
11
|
-
@subscriber = EZMQ::Subscriber.new port: 5557, topic: 'busy-queues'
|
12
|
-
@item = 'item'
|
13
|
-
@queue = 'testing'
|
14
|
-
clear_all_queues
|
15
|
-
end
|
16
|
-
|
17
|
-
context 'a non-empty queue' do
|
18
|
-
setup { enqueue @queue, @item }
|
19
|
-
|
20
|
-
should 'have their named published to \'busy-queues\'' do
|
21
|
-
Timeout.timeout($for_a_while) do
|
22
|
-
@subscriber.listen do |message|
|
23
|
-
assert_equal @queue, message
|
24
|
-
break
|
25
|
-
end
|
26
|
-
end
|
27
|
-
end
|
28
|
-
end
|
29
|
-
|
30
|
-
context 'an empty queue' do
|
31
|
-
should 'not have their name published' do
|
32
|
-
assert_raises Timeout::Error do
|
33
|
-
Timeout.timeout($for_a_moment) do
|
34
|
-
@subscriber.listen {}
|
35
|
-
end
|
36
|
-
end
|
37
|
-
end
|
38
|
-
end
|
39
|
-
end
|
data/test/queue_handler_spec.rb
DELETED
@@ -1,54 +0,0 @@
|
|
1
|
-
require 'kintama'
|
2
|
-
require 'ezmq'
|
3
|
-
require_relative 'shared/process_spawning'
|
4
|
-
require_relative 'shared/helpers'
|
5
|
-
|
6
|
-
given 'the Queue Handler' do
|
7
|
-
include QueueHelper
|
8
|
-
|
9
|
-
setup do
|
10
|
-
@client = EZMQ::Client.new port: 5556
|
11
|
-
@item = 'item'
|
12
|
-
@queue = 'testing'
|
13
|
-
clear_all_queues
|
14
|
-
end
|
15
|
-
|
16
|
-
context 'dequeuing' do
|
17
|
-
setup { @request = "#{@queue}" }
|
18
|
-
|
19
|
-
should 'return the next queued item' do
|
20
|
-
enqueue @queue, @item
|
21
|
-
response = @client.request @request
|
22
|
-
assert_equal @item, response
|
23
|
-
end
|
24
|
-
|
25
|
-
should 'return \'\' for an empty queue' do
|
26
|
-
response = @client.request @request
|
27
|
-
assert_equal '', response
|
28
|
-
end
|
29
|
-
end
|
30
|
-
|
31
|
-
context 'enqueuing' do
|
32
|
-
setup { @request = "#{@queue} #{@item}" }
|
33
|
-
|
34
|
-
should 'return the queue new size' do
|
35
|
-
response = @client.request @request
|
36
|
-
assert_equal '1', response
|
37
|
-
end
|
38
|
-
end
|
39
|
-
|
40
|
-
context 'queue mirroring' do
|
41
|
-
should 'mirror queue enqueuing' do
|
42
|
-
enqueue @queue, @item
|
43
|
-
response = @client.request "mirror-#{@queue}"
|
44
|
-
assert_equal @item, response
|
45
|
-
end
|
46
|
-
|
47
|
-
should 'mirror queue clearing' do
|
48
|
-
enqueue @queue, @item
|
49
|
-
clear_all_queues
|
50
|
-
response = @client.request "mirror-#{@queue}"
|
51
|
-
assert_equal '', response
|
52
|
-
end
|
53
|
-
end
|
54
|
-
end
|
data/test/run_all.rb
DELETED
@@ -1,16 +0,0 @@
|
|
1
|
-
$for_a_moment = 0.2
|
2
|
-
$for_a_while = 0.5
|
3
|
-
$for_io_calls = 1
|
4
|
-
$for_everything = 2
|
5
|
-
|
6
|
-
Kintama.on_start do
|
7
|
-
puts `#{__dir__}/../../bin/sweatshop start --testing`
|
8
|
-
%w(test minimal git empty).each do |job_file|
|
9
|
-
FileUtils.cp "#{__dir__}/../data/#{job_file}_job.yaml", File.expand_path(configatron.job_path)
|
10
|
-
end
|
11
|
-
sleep $for_everything
|
12
|
-
end
|
13
|
-
|
14
|
-
Kintama.on_finish do
|
15
|
-
puts `#{__dir__}/../../bin/sweatshop stop`
|
16
|
-
end
|