robot_sweatshop 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +6 -0
- data/.rubocop.yml +12 -0
- data/Gemfile +2 -0
- data/LICENSE +21 -0
- data/README.md +45 -0
- data/Rakefile +3 -0
- data/bin/lib/common.rb +49 -0
- data/bin/lib/config.rb +20 -0
- data/bin/lib/inspect.rb +42 -0
- data/bin/lib/job.rb +13 -0
- data/bin/lib/start.rb +11 -0
- data/bin/sweatshop +73 -0
- data/config.rb +24 -0
- data/config.yaml +12 -0
- data/jobs/example.yaml +10 -0
- data/kintama/README.md +3 -0
- data/kintama/data/payload_data.yaml +6 -0
- data/kintama/end-to-end_spec.rb +30 -0
- data/kintama/input_http_spec.rb +45 -0
- data/kintama/job_assembler_spec.rb +72 -0
- data/kintama/job_worker_spec.rb +65 -0
- data/kintama/moneta-queue_spec.rb +48 -0
- data/kintama/payload_parser_spec.rb +71 -0
- data/kintama/queue_broadcaster_spec.rb +39 -0
- data/kintama/queue_handler_spec.rb +54 -0
- data/kintama/run_all.rb +3 -0
- data/kintama/shared/helpers.rb +55 -0
- data/kintama/shared/process_spawning.rb +13 -0
- data/lib/README.md +12 -0
- data/lib/input/http.rb +27 -0
- data/lib/job/assembler.rb +36 -0
- data/lib/job/worker.rb +40 -0
- data/lib/payload/lib/bitbucket.rb +61 -0
- data/lib/payload/lib/github.rb +45 -0
- data/lib/payload/lib/payload.rb +12 -0
- data/lib/payload/parser.rb +23 -0
- data/lib/queue-helper.rb +32 -0
- data/lib/queue/broadcaster.rb +18 -0
- data/lib/queue/handler.rb +23 -0
- data/lib/queue/lib/moneta-queue.rb +49 -0
- data/lib/queue/watcher.rb +18 -0
- data/robot_sweatshop.eye +58 -0
- data/robot_sweatshop.gemspec +26 -0
- data/robot_sweatshop.production.eye +8 -0
- data/robot_sweatshop.testing.eye +8 -0
- data/workspaces/.keep +0 -0
- metadata +233 -0
@@ -0,0 +1,72 @@
|
|
1
|
+
require 'kintama'
|
2
|
+
require 'ezmq'
|
3
|
+
require 'json'
|
4
|
+
require_relative 'shared/process_spawning'
|
5
|
+
require_relative 'shared/helpers'
|
6
|
+
|
7
|
+
describe 'the Job Assembler' do
|
8
|
+
include QueueHelper
|
9
|
+
include PayloadHelper
|
10
|
+
include JobHelper
|
11
|
+
|
12
|
+
setup do
|
13
|
+
@client = EZMQ::Client.new port: 5556
|
14
|
+
@parsed_payloads_queue = 'parsed-payload'
|
15
|
+
@jobs_queue = 'jobs'
|
16
|
+
clear_all_queues
|
17
|
+
end
|
18
|
+
|
19
|
+
given 'valid parsed payload data in \'parsed-payload\'' do
|
20
|
+
setup do
|
21
|
+
payload = example_parsed_payload(for_branch: 'develop')
|
22
|
+
@client.request "#{@parsed_payloads_queue} #{payload}"
|
23
|
+
sleep $for_a_while
|
24
|
+
end
|
25
|
+
|
26
|
+
should 'remove it from \'parsed-payload\'' do
|
27
|
+
response = @client.request @parsed_payloads_queue
|
28
|
+
assert_equal '', response
|
29
|
+
end
|
30
|
+
|
31
|
+
should 'enqueue commands, context, and job name to \'jobs\'' do
|
32
|
+
response = @client.request "mirror-#{@jobs_queue}"
|
33
|
+
response = JSON.parse response
|
34
|
+
assert_kind_of Hash, response['context']
|
35
|
+
assert_kind_of Array, response['commands']
|
36
|
+
assert_kind_of String, response['job_name']
|
37
|
+
end
|
38
|
+
|
39
|
+
should 'only enqueue string objects to context' do
|
40
|
+
response = @client.request "mirror-#{@jobs_queue}"
|
41
|
+
response = JSON.parse response
|
42
|
+
response['context'].each do |_key, value|
|
43
|
+
assert_kind_of String, value
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
given 'invalid job data in \'parsed-payload\'' do
|
49
|
+
setup do
|
50
|
+
invalid_data = {
|
51
|
+
ignored_branch: example_parsed_payload(for_branch: 'not_on_whitelist'),
|
52
|
+
bad_payload: example_parsed_payload(with_payload: 'not hash'),
|
53
|
+
bad_job: example_parsed_payload(for_job: 'asdf'),
|
54
|
+
not_json: 'not_json'
|
55
|
+
}
|
56
|
+
invalid_data.each do |_type, datum|
|
57
|
+
@client.request "#{@parsed_payloads_queue} #{datum}"
|
58
|
+
end
|
59
|
+
sleep $for_a_while
|
60
|
+
end
|
61
|
+
|
62
|
+
should 'remove all of it from \'parsed-payload\'' do
|
63
|
+
response = @client.request @parsed_payloads_queue
|
64
|
+
assert_equal '', response
|
65
|
+
end
|
66
|
+
|
67
|
+
should 'not queue anything to \'jobs\'' do
|
68
|
+
response = @client.request @jobs_queue
|
69
|
+
assert_equal '', response
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
require 'kintama'
|
2
|
+
require 'ezmq'
|
3
|
+
require_relative 'shared/process_spawning'
|
4
|
+
require_relative 'shared/helpers'
|
5
|
+
|
6
|
+
describe 'the Worker' do
|
7
|
+
include QueueHelper
|
8
|
+
include JobHelper
|
9
|
+
|
10
|
+
setup do
|
11
|
+
@client = EZMQ::Client.new port: 5556
|
12
|
+
@jobs_queue = 'jobs'
|
13
|
+
@test_file = reset_test_file
|
14
|
+
clear_all_queues
|
15
|
+
end
|
16
|
+
|
17
|
+
given 'valid job data in \'jobs\'' do
|
18
|
+
setup do
|
19
|
+
job = example_job in_context: {custom: 'Hello world!'},
|
20
|
+
with_commands: ['echo $custom','echo $custom > test.txt']
|
21
|
+
@client.request "#{@jobs_queue} #{job}"
|
22
|
+
end
|
23
|
+
|
24
|
+
should 'remove it from \'jobs\'' do
|
25
|
+
sleep $for_a_moment
|
26
|
+
response = @client.request @jobs_queue
|
27
|
+
assert_equal '', response
|
28
|
+
end
|
29
|
+
|
30
|
+
should 'run the dequeued job' do
|
31
|
+
sleep $for_io_calls
|
32
|
+
assert_equal true, File.file?(@test_file)
|
33
|
+
end
|
34
|
+
|
35
|
+
should 'run jobs with the context as environment variables' do
|
36
|
+
sleep $for_io_calls
|
37
|
+
assert_equal "Hello world!\n", File.read(@test_file)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
given 'invalid job data in \'jobs\'' do
|
42
|
+
setup do
|
43
|
+
invalid_data = {
|
44
|
+
bad_context: example_job(in_context: 'not hash'),
|
45
|
+
bad_commands: example_job(with_commands: 'echo 1'),
|
46
|
+
not_json: 'not_json'
|
47
|
+
}
|
48
|
+
invalid_data.each do |_type, datum|
|
49
|
+
@client.request "#{@jobs_queue} #{datum}"
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
should 'remove all of it from \'jobs\'' do
|
54
|
+
sleep $for_a_moment
|
55
|
+
response = @client.request @jobs_queue
|
56
|
+
assert_equal '', response
|
57
|
+
end
|
58
|
+
|
59
|
+
should 'not run anything' do
|
60
|
+
sleep $for_io_calls
|
61
|
+
response = @client.request @jobs_queue
|
62
|
+
assert_equal false, File.file?(@test_file)
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
require 'kintama'
|
2
|
+
require_relative 'shared/helpers'
|
3
|
+
require_relative '../lib/queue/lib/moneta-queue'
|
4
|
+
|
5
|
+
given 'the Moneta Queue class' do
|
6
|
+
include QueueHelper
|
7
|
+
|
8
|
+
setup do
|
9
|
+
clear_all_queues
|
10
|
+
end
|
11
|
+
|
12
|
+
should 'return a list of actively watched queues' do
|
13
|
+
assert_kind_of Array, MonetaQueue.watched_queues
|
14
|
+
end
|
15
|
+
|
16
|
+
context 'an instance' do
|
17
|
+
setup do
|
18
|
+
@file_queue = MonetaQueue.new 'testing'
|
19
|
+
@file_queue.clear
|
20
|
+
end
|
21
|
+
|
22
|
+
should 'return size' do
|
23
|
+
assert_equal @file_queue.size, 0
|
24
|
+
end
|
25
|
+
|
26
|
+
should 'enqueue items' do
|
27
|
+
@file_queue.enqueue 'item'
|
28
|
+
assert_equal @file_queue.size, 1
|
29
|
+
end
|
30
|
+
|
31
|
+
should 'dequeue items' do
|
32
|
+
@file_queue.enqueue 'item1'
|
33
|
+
@file_queue.enqueue 'item2'
|
34
|
+
assert_equal @file_queue.size, 2
|
35
|
+
assert_equal @file_queue.dequeue, 'item1'
|
36
|
+
end
|
37
|
+
|
38
|
+
should 'clear items' do
|
39
|
+
@file_queue.enqueue 'item'
|
40
|
+
@file_queue.clear
|
41
|
+
assert_equal @file_queue.size, 0
|
42
|
+
end
|
43
|
+
|
44
|
+
should 'return the queue on inspect' do
|
45
|
+
assert_equal @file_queue.inspect, '[]'
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
require 'kintama'
|
2
|
+
require 'ezmq'
|
3
|
+
require 'json'
|
4
|
+
require_relative 'shared/process_spawning'
|
5
|
+
require_relative 'shared/helpers'
|
6
|
+
|
7
|
+
describe 'the Payload Parser' do
|
8
|
+
include QueueHelper
|
9
|
+
include InHelper
|
10
|
+
include PayloadHelper
|
11
|
+
|
12
|
+
setup do
|
13
|
+
@client = EZMQ::Client.new port: 5556
|
14
|
+
@raw_queue = 'raw-payload'
|
15
|
+
@parsed_queue = 'parsed-payload'
|
16
|
+
clear_all_queues
|
17
|
+
end
|
18
|
+
|
19
|
+
# TODO: refactor and add github
|
20
|
+
['Bitbucket'].each do |format|
|
21
|
+
given "valid #{format} data in 'raw-payload'" do
|
22
|
+
setup do
|
23
|
+
payload = example_raw_payload(with_format: format)
|
24
|
+
@client.request "#{@raw_queue} #{payload}"
|
25
|
+
sleep $for_a_while
|
26
|
+
end
|
27
|
+
|
28
|
+
should 'remove it from \'raw-payload\'' do
|
29
|
+
response = @client.request @raw_queue
|
30
|
+
assert_equal '', response
|
31
|
+
end
|
32
|
+
|
33
|
+
should 'enqueue parsed payload data and job name to \'parsed-payload\'' do
|
34
|
+
response = @client.request "mirror-#{@parsed_queue}"
|
35
|
+
response = JSON.parse response
|
36
|
+
|
37
|
+
assert_kind_of Hash, response['payload']
|
38
|
+
Payload.hash_keys.each do |key|
|
39
|
+
assert_not_nil response['payload'][key]
|
40
|
+
assert_not_equal key, response['payload'][key] # important for how Ruby interprets "string"['key']
|
41
|
+
end
|
42
|
+
assert_kind_of String, response['job_name']
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
given 'invalid payload data in \'raw-payload\'' do
|
48
|
+
setup do
|
49
|
+
invalid_data = {
|
50
|
+
malformed_payload: example_raw_payload(with_format: 'malformed'),
|
51
|
+
unsupported_format: example_raw_payload(with_format: 'asdf'),
|
52
|
+
not_json: 'not_json'
|
53
|
+
}
|
54
|
+
invalid_data.each do |_type, datum|
|
55
|
+
@client.request "#{@raw_queue} #{datum}"
|
56
|
+
end
|
57
|
+
sleep $for_a_while
|
58
|
+
# TODO: should not crash the payload parser
|
59
|
+
end
|
60
|
+
|
61
|
+
should 'remove all of it from \'raw-payload\'' do
|
62
|
+
response = @client.request @raw_queue
|
63
|
+
assert_equal '', response
|
64
|
+
end
|
65
|
+
|
66
|
+
should 'not queue anything to \'parsed-payload\'' do
|
67
|
+
response = @client.request @parsed_queue
|
68
|
+
assert_equal '', response
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
require 'kintama'
|
2
|
+
require 'ezmq'
|
3
|
+
require 'timeout'
|
4
|
+
require_relative 'shared/process_spawning'
|
5
|
+
require_relative 'shared/helpers'
|
6
|
+
|
7
|
+
given 'the Queue Broadcaster' do
|
8
|
+
include QueueHelper
|
9
|
+
|
10
|
+
setup do
|
11
|
+
@subscriber = EZMQ::Subscriber.new port: 5557, topic: 'busy-queues'
|
12
|
+
@item = 'item'
|
13
|
+
@queue = 'testing'
|
14
|
+
clear_all_queues
|
15
|
+
end
|
16
|
+
|
17
|
+
context 'a non-empty queue' do
|
18
|
+
setup { enqueue @queue, @item }
|
19
|
+
|
20
|
+
should 'have their named published to \'busy-queues\'' do
|
21
|
+
Timeout.timeout($for_a_while) do
|
22
|
+
@subscriber.listen do |message|
|
23
|
+
assert_equal @queue, message
|
24
|
+
break
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
context 'an empty queue' do
|
31
|
+
should 'not have their name published' do
|
32
|
+
assert_raises Timeout::Error do
|
33
|
+
Timeout.timeout($for_a_moment) do
|
34
|
+
@subscriber.listen {}
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
require 'kintama'
|
2
|
+
require 'ezmq'
|
3
|
+
require_relative 'shared/process_spawning'
|
4
|
+
require_relative 'shared/helpers'
|
5
|
+
|
6
|
+
given 'the Queue Handler' do
|
7
|
+
include QueueHelper
|
8
|
+
|
9
|
+
setup do
|
10
|
+
@client = EZMQ::Client.new port: 5556
|
11
|
+
@item = 'item'
|
12
|
+
@queue = 'testing'
|
13
|
+
clear_all_queues
|
14
|
+
end
|
15
|
+
|
16
|
+
context 'dequeuing' do
|
17
|
+
setup { @request = "#{@queue}" }
|
18
|
+
|
19
|
+
should 'return the next queued item' do
|
20
|
+
enqueue @queue, @item
|
21
|
+
response = @client.request @request
|
22
|
+
assert_equal @item, response
|
23
|
+
end
|
24
|
+
|
25
|
+
should 'return \'\' for an empty queue' do
|
26
|
+
response = @client.request @request
|
27
|
+
assert_equal '', response
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
context 'enqueuing' do
|
32
|
+
setup { @request = "#{@queue} #{@item}" }
|
33
|
+
|
34
|
+
should 'return the queue new size' do
|
35
|
+
response = @client.request @request
|
36
|
+
assert_equal '1', response
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
context 'queue mirroring' do
|
41
|
+
should 'mirror queue enqueuing' do
|
42
|
+
enqueue @queue, @item
|
43
|
+
response = @client.request "mirror-#{@queue}"
|
44
|
+
assert_equal @item, response
|
45
|
+
end
|
46
|
+
|
47
|
+
should 'mirror queue clearing' do
|
48
|
+
enqueue @queue, @item
|
49
|
+
clear_all_queues
|
50
|
+
response = @client.request "mirror-#{@queue}"
|
51
|
+
assert_equal '', response
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
data/kintama/run_all.rb
ADDED
@@ -0,0 +1,55 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
require 'json'
|
3
|
+
require_relative '../../lib/queue/lib/moneta-queue'
|
4
|
+
require_relative '../../lib/payload/lib/payload'
|
5
|
+
require_relative '../../config'
|
6
|
+
|
7
|
+
module QueueHelper
|
8
|
+
def clear_all_queues
|
9
|
+
MonetaQueue.watched_queues.each do |queue|
|
10
|
+
queue = MonetaQueue.new queue
|
11
|
+
queue.clear
|
12
|
+
end
|
13
|
+
end
|
14
|
+
def enqueue(queue_name, item)
|
15
|
+
queue = MonetaQueue.new queue_name
|
16
|
+
queue.enqueue item
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
module InHelper
|
21
|
+
def load_payload(of_format)
|
22
|
+
payload_strings = YAML.load_file "#{__dir__}/../data/payload_data.yaml"
|
23
|
+
payload_strings[of_format.downcase]
|
24
|
+
end
|
25
|
+
def example_raw_payload(with_format:)
|
26
|
+
payload = load_payload with_format
|
27
|
+
JSON.generate payload: payload,
|
28
|
+
format: with_format,
|
29
|
+
job_name: 'example'
|
30
|
+
end
|
31
|
+
def input_http_url(for_job: 'example', in_format: 'bitbucket')
|
32
|
+
"http://localhost:#{configatron.input.http.port}/#{in_format}/payload-for/#{for_job}"
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
module PayloadHelper
|
37
|
+
def example_parsed_payload(with_payload: nil, for_branch: 'develop', for_job: 'example')
|
38
|
+
payload = with_payload || { branch: for_branch }
|
39
|
+
JSON.generate payload: payload,
|
40
|
+
job_name: for_job
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
module JobHelper
|
45
|
+
def example_job(in_context: {}, with_commands: [])
|
46
|
+
JSON.generate context: in_context,
|
47
|
+
commands: with_commands,
|
48
|
+
job_name: 'example'
|
49
|
+
end
|
50
|
+
def reset_test_file
|
51
|
+
test_file = File.expand_path "#{__dir__}/../../workspaces/example-testingid/test.txt"
|
52
|
+
FileUtils.rm_rf test_file
|
53
|
+
test_file
|
54
|
+
end
|
55
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
$for_a_moment = 0.1
|
2
|
+
$for_a_while = 0.5
|
3
|
+
$for_io_calls = 1
|
4
|
+
$for_everything = 3
|
5
|
+
|
6
|
+
Kintama.on_start do
|
7
|
+
puts `#{__dir__}/../../bin/sweatshop start --testing`
|
8
|
+
sleep $for_everything
|
9
|
+
end
|
10
|
+
|
11
|
+
Kintama.on_finish do
|
12
|
+
puts `#{__dir__}/../../bin/sweatshop stop`
|
13
|
+
end
|
data/lib/README.md
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
Job lifecycle:
|
2
|
+
|
3
|
+
```
|
4
|
+
in-* -> raw-payloads { payload:, format:, job_name: } ->
|
5
|
+
payload-parser -> parsed-payloads { payload:, job_name: } ->
|
6
|
+
job-assembler -> jobs { context:, commands:, job_name: } ->
|
7
|
+
job-worker
|
8
|
+
```
|
9
|
+
|
10
|
+
All passing done via the moneta core in queue/*
|
11
|
+
|
12
|
+
Also queue/watcher for debugging
|