robot_sweatshop 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +6 -0
  3. data/.rubocop.yml +12 -0
  4. data/Gemfile +2 -0
  5. data/LICENSE +21 -0
  6. data/README.md +45 -0
  7. data/Rakefile +3 -0
  8. data/bin/lib/common.rb +49 -0
  9. data/bin/lib/config.rb +20 -0
  10. data/bin/lib/inspect.rb +42 -0
  11. data/bin/lib/job.rb +13 -0
  12. data/bin/lib/start.rb +11 -0
  13. data/bin/sweatshop +73 -0
  14. data/config.rb +24 -0
  15. data/config.yaml +12 -0
  16. data/jobs/example.yaml +10 -0
  17. data/kintama/README.md +3 -0
  18. data/kintama/data/payload_data.yaml +6 -0
  19. data/kintama/end-to-end_spec.rb +30 -0
  20. data/kintama/input_http_spec.rb +45 -0
  21. data/kintama/job_assembler_spec.rb +72 -0
  22. data/kintama/job_worker_spec.rb +65 -0
  23. data/kintama/moneta-queue_spec.rb +48 -0
  24. data/kintama/payload_parser_spec.rb +71 -0
  25. data/kintama/queue_broadcaster_spec.rb +39 -0
  26. data/kintama/queue_handler_spec.rb +54 -0
  27. data/kintama/run_all.rb +3 -0
  28. data/kintama/shared/helpers.rb +55 -0
  29. data/kintama/shared/process_spawning.rb +13 -0
  30. data/lib/README.md +12 -0
  31. data/lib/input/http.rb +27 -0
  32. data/lib/job/assembler.rb +36 -0
  33. data/lib/job/worker.rb +40 -0
  34. data/lib/payload/lib/bitbucket.rb +61 -0
  35. data/lib/payload/lib/github.rb +45 -0
  36. data/lib/payload/lib/payload.rb +12 -0
  37. data/lib/payload/parser.rb +23 -0
  38. data/lib/queue-helper.rb +32 -0
  39. data/lib/queue/broadcaster.rb +18 -0
  40. data/lib/queue/handler.rb +23 -0
  41. data/lib/queue/lib/moneta-queue.rb +49 -0
  42. data/lib/queue/watcher.rb +18 -0
  43. data/robot_sweatshop.eye +58 -0
  44. data/robot_sweatshop.gemspec +26 -0
  45. data/robot_sweatshop.production.eye +8 -0
  46. data/robot_sweatshop.testing.eye +8 -0
  47. data/workspaces/.keep +0 -0
  48. metadata +233 -0
@@ -0,0 +1,72 @@
1
+ require 'kintama'
2
+ require 'ezmq'
3
+ require 'json'
4
+ require_relative 'shared/process_spawning'
5
+ require_relative 'shared/helpers'
6
+
7
+ describe 'the Job Assembler' do
8
+ include QueueHelper
9
+ include PayloadHelper
10
+ include JobHelper
11
+
12
+ setup do
13
+ @client = EZMQ::Client.new port: 5556
14
+ @parsed_payloads_queue = 'parsed-payload'
15
+ @jobs_queue = 'jobs'
16
+ clear_all_queues
17
+ end
18
+
19
+ given 'valid parsed payload data in \'parsed-payload\'' do
20
+ setup do
21
+ payload = example_parsed_payload(for_branch: 'develop')
22
+ @client.request "#{@parsed_payloads_queue} #{payload}"
23
+ sleep $for_a_while
24
+ end
25
+
26
+ should 'remove it from \'parsed-payload\'' do
27
+ response = @client.request @parsed_payloads_queue
28
+ assert_equal '', response
29
+ end
30
+
31
+ should 'enqueue commands, context, and job name to \'jobs\'' do
32
+ response = @client.request "mirror-#{@jobs_queue}"
33
+ response = JSON.parse response
34
+ assert_kind_of Hash, response['context']
35
+ assert_kind_of Array, response['commands']
36
+ assert_kind_of String, response['job_name']
37
+ end
38
+
39
+ should 'only enqueue string objects to context' do
40
+ response = @client.request "mirror-#{@jobs_queue}"
41
+ response = JSON.parse response
42
+ response['context'].each do |_key, value|
43
+ assert_kind_of String, value
44
+ end
45
+ end
46
+ end
47
+
48
+ given 'invalid job data in \'parsed-payload\'' do
49
+ setup do
50
+ invalid_data = {
51
+ ignored_branch: example_parsed_payload(for_branch: 'not_on_whitelist'),
52
+ bad_payload: example_parsed_payload(with_payload: 'not hash'),
53
+ bad_job: example_parsed_payload(for_job: 'asdf'),
54
+ not_json: 'not_json'
55
+ }
56
+ invalid_data.each do |_type, datum|
57
+ @client.request "#{@parsed_payloads_queue} #{datum}"
58
+ end
59
+ sleep $for_a_while
60
+ end
61
+
62
+ should 'remove all of it from \'parsed-payload\'' do
63
+ response = @client.request @parsed_payloads_queue
64
+ assert_equal '', response
65
+ end
66
+
67
+ should 'not queue anything to \'jobs\'' do
68
+ response = @client.request @jobs_queue
69
+ assert_equal '', response
70
+ end
71
+ end
72
+ end
@@ -0,0 +1,65 @@
1
+ require 'kintama'
2
+ require 'ezmq'
3
+ require_relative 'shared/process_spawning'
4
+ require_relative 'shared/helpers'
5
+
6
+ describe 'the Worker' do
7
+ include QueueHelper
8
+ include JobHelper
9
+
10
+ setup do
11
+ @client = EZMQ::Client.new port: 5556
12
+ @jobs_queue = 'jobs'
13
+ @test_file = reset_test_file
14
+ clear_all_queues
15
+ end
16
+
17
+ given 'valid job data in \'jobs\'' do
18
+ setup do
19
+ job = example_job in_context: {custom: 'Hello world!'},
20
+ with_commands: ['echo $custom','echo $custom > test.txt']
21
+ @client.request "#{@jobs_queue} #{job}"
22
+ end
23
+
24
+ should 'remove it from \'jobs\'' do
25
+ sleep $for_a_moment
26
+ response = @client.request @jobs_queue
27
+ assert_equal '', response
28
+ end
29
+
30
+ should 'run the dequeued job' do
31
+ sleep $for_io_calls
32
+ assert_equal true, File.file?(@test_file)
33
+ end
34
+
35
+ should 'run jobs with the context as environment variables' do
36
+ sleep $for_io_calls
37
+ assert_equal "Hello world!\n", File.read(@test_file)
38
+ end
39
+ end
40
+
41
+ given 'invalid job data in \'jobs\'' do
42
+ setup do
43
+ invalid_data = {
44
+ bad_context: example_job(in_context: 'not hash'),
45
+ bad_commands: example_job(with_commands: 'echo 1'),
46
+ not_json: 'not_json'
47
+ }
48
+ invalid_data.each do |_type, datum|
49
+ @client.request "#{@jobs_queue} #{datum}"
50
+ end
51
+ end
52
+
53
+ should 'remove all of it from \'jobs\'' do
54
+ sleep $for_a_moment
55
+ response = @client.request @jobs_queue
56
+ assert_equal '', response
57
+ end
58
+
59
+ should 'not run anything' do
60
+ sleep $for_io_calls
61
+ response = @client.request @jobs_queue
62
+ assert_equal false, File.file?(@test_file)
63
+ end
64
+ end
65
+ end
@@ -0,0 +1,48 @@
1
+ require 'kintama'
2
+ require_relative 'shared/helpers'
3
+ require_relative '../lib/queue/lib/moneta-queue'
4
+
5
+ given 'the Moneta Queue class' do
6
+ include QueueHelper
7
+
8
+ setup do
9
+ clear_all_queues
10
+ end
11
+
12
+ should 'return a list of actively watched queues' do
13
+ assert_kind_of Array, MonetaQueue.watched_queues
14
+ end
15
+
16
+ context 'an instance' do
17
+ setup do
18
+ @file_queue = MonetaQueue.new 'testing'
19
+ @file_queue.clear
20
+ end
21
+
22
+ should 'return size' do
23
+ assert_equal @file_queue.size, 0
24
+ end
25
+
26
+ should 'enqueue items' do
27
+ @file_queue.enqueue 'item'
28
+ assert_equal @file_queue.size, 1
29
+ end
30
+
31
+ should 'dequeue items' do
32
+ @file_queue.enqueue 'item1'
33
+ @file_queue.enqueue 'item2'
34
+ assert_equal @file_queue.size, 2
35
+ assert_equal @file_queue.dequeue, 'item1'
36
+ end
37
+
38
+ should 'clear items' do
39
+ @file_queue.enqueue 'item'
40
+ @file_queue.clear
41
+ assert_equal @file_queue.size, 0
42
+ end
43
+
44
+ should 'return the queue on inspect' do
45
+ assert_equal @file_queue.inspect, '[]'
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,71 @@
1
+ require 'kintama'
2
+ require 'ezmq'
3
+ require 'json'
4
+ require_relative 'shared/process_spawning'
5
+ require_relative 'shared/helpers'
6
+
7
+ describe 'the Payload Parser' do
8
+ include QueueHelper
9
+ include InHelper
10
+ include PayloadHelper
11
+
12
+ setup do
13
+ @client = EZMQ::Client.new port: 5556
14
+ @raw_queue = 'raw-payload'
15
+ @parsed_queue = 'parsed-payload'
16
+ clear_all_queues
17
+ end
18
+
19
+ # TODO: refactor and add github
20
+ ['Bitbucket'].each do |format|
21
+ given "valid #{format} data in 'raw-payload'" do
22
+ setup do
23
+ payload = example_raw_payload(with_format: format)
24
+ @client.request "#{@raw_queue} #{payload}"
25
+ sleep $for_a_while
26
+ end
27
+
28
+ should 'remove it from \'raw-payload\'' do
29
+ response = @client.request @raw_queue
30
+ assert_equal '', response
31
+ end
32
+
33
+ should 'enqueue parsed payload data and job name to \'parsed-payload\'' do
34
+ response = @client.request "mirror-#{@parsed_queue}"
35
+ response = JSON.parse response
36
+
37
+ assert_kind_of Hash, response['payload']
38
+ Payload.hash_keys.each do |key|
39
+ assert_not_nil response['payload'][key]
40
+ assert_not_equal key, response['payload'][key] # important for how Ruby interprets "string"['key']
41
+ end
42
+ assert_kind_of String, response['job_name']
43
+ end
44
+ end
45
+ end
46
+
47
+ given 'invalid payload data in \'raw-payload\'' do
48
+ setup do
49
+ invalid_data = {
50
+ malformed_payload: example_raw_payload(with_format: 'malformed'),
51
+ unsupported_format: example_raw_payload(with_format: 'asdf'),
52
+ not_json: 'not_json'
53
+ }
54
+ invalid_data.each do |_type, datum|
55
+ @client.request "#{@raw_queue} #{datum}"
56
+ end
57
+ sleep $for_a_while
58
+ # TODO: should not crash the payload parser
59
+ end
60
+
61
+ should 'remove all of it from \'raw-payload\'' do
62
+ response = @client.request @raw_queue
63
+ assert_equal '', response
64
+ end
65
+
66
+ should 'not queue anything to \'parsed-payload\'' do
67
+ response = @client.request @parsed_queue
68
+ assert_equal '', response
69
+ end
70
+ end
71
+ end
@@ -0,0 +1,39 @@
1
+ require 'kintama'
2
+ require 'ezmq'
3
+ require 'timeout'
4
+ require_relative 'shared/process_spawning'
5
+ require_relative 'shared/helpers'
6
+
7
+ given 'the Queue Broadcaster' do
8
+ include QueueHelper
9
+
10
+ setup do
11
+ @subscriber = EZMQ::Subscriber.new port: 5557, topic: 'busy-queues'
12
+ @item = 'item'
13
+ @queue = 'testing'
14
+ clear_all_queues
15
+ end
16
+
17
+ context 'a non-empty queue' do
18
+ setup { enqueue @queue, @item }
19
+
20
+ should 'have their named published to \'busy-queues\'' do
21
+ Timeout.timeout($for_a_while) do
22
+ @subscriber.listen do |message|
23
+ assert_equal @queue, message
24
+ break
25
+ end
26
+ end
27
+ end
28
+ end
29
+
30
+ context 'an empty queue' do
31
+ should 'not have their name published' do
32
+ assert_raises Timeout::Error do
33
+ Timeout.timeout($for_a_moment) do
34
+ @subscriber.listen {}
35
+ end
36
+ end
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,54 @@
1
+ require 'kintama'
2
+ require 'ezmq'
3
+ require_relative 'shared/process_spawning'
4
+ require_relative 'shared/helpers'
5
+
6
+ given 'the Queue Handler' do
7
+ include QueueHelper
8
+
9
+ setup do
10
+ @client = EZMQ::Client.new port: 5556
11
+ @item = 'item'
12
+ @queue = 'testing'
13
+ clear_all_queues
14
+ end
15
+
16
+ context 'dequeuing' do
17
+ setup { @request = "#{@queue}" }
18
+
19
+ should 'return the next queued item' do
20
+ enqueue @queue, @item
21
+ response = @client.request @request
22
+ assert_equal @item, response
23
+ end
24
+
25
+ should 'return \'\' for an empty queue' do
26
+ response = @client.request @request
27
+ assert_equal '', response
28
+ end
29
+ end
30
+
31
+ context 'enqueuing' do
32
+ setup { @request = "#{@queue} #{@item}" }
33
+
34
+ should 'return the queue new size' do
35
+ response = @client.request @request
36
+ assert_equal '1', response
37
+ end
38
+ end
39
+
40
+ context 'queue mirroring' do
41
+ should 'mirror queue enqueuing' do
42
+ enqueue @queue, @item
43
+ response = @client.request "mirror-#{@queue}"
44
+ assert_equal @item, response
45
+ end
46
+
47
+ should 'mirror queue clearing' do
48
+ enqueue @queue, @item
49
+ clear_all_queues
50
+ response = @client.request "mirror-#{@queue}"
51
+ assert_equal '', response
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,3 @@
1
+ Dir.glob("#{__dir__}/*_spec.rb").each do |spec_file|
2
+ require_relative spec_file
3
+ end
@@ -0,0 +1,55 @@
1
+ require 'yaml'
2
+ require 'json'
3
+ require_relative '../../lib/queue/lib/moneta-queue'
4
+ require_relative '../../lib/payload/lib/payload'
5
+ require_relative '../../config'
6
+
7
+ module QueueHelper
8
+ def clear_all_queues
9
+ MonetaQueue.watched_queues.each do |queue|
10
+ queue = MonetaQueue.new queue
11
+ queue.clear
12
+ end
13
+ end
14
+ def enqueue(queue_name, item)
15
+ queue = MonetaQueue.new queue_name
16
+ queue.enqueue item
17
+ end
18
+ end
19
+
20
+ module InHelper
21
+ def load_payload(of_format)
22
+ payload_strings = YAML.load_file "#{__dir__}/../data/payload_data.yaml"
23
+ payload_strings[of_format.downcase]
24
+ end
25
+ def example_raw_payload(with_format:)
26
+ payload = load_payload with_format
27
+ JSON.generate payload: payload,
28
+ format: with_format,
29
+ job_name: 'example'
30
+ end
31
+ def input_http_url(for_job: 'example', in_format: 'bitbucket')
32
+ "http://localhost:#{configatron.input.http.port}/#{in_format}/payload-for/#{for_job}"
33
+ end
34
+ end
35
+
36
+ module PayloadHelper
37
+ def example_parsed_payload(with_payload: nil, for_branch: 'develop', for_job: 'example')
38
+ payload = with_payload || { branch: for_branch }
39
+ JSON.generate payload: payload,
40
+ job_name: for_job
41
+ end
42
+ end
43
+
44
+ module JobHelper
45
+ def example_job(in_context: {}, with_commands: [])
46
+ JSON.generate context: in_context,
47
+ commands: with_commands,
48
+ job_name: 'example'
49
+ end
50
+ def reset_test_file
51
+ test_file = File.expand_path "#{__dir__}/../../workspaces/example-testingid/test.txt"
52
+ FileUtils.rm_rf test_file
53
+ test_file
54
+ end
55
+ end
@@ -0,0 +1,13 @@
1
+ $for_a_moment = 0.1
2
+ $for_a_while = 0.5
3
+ $for_io_calls = 1
4
+ $for_everything = 3
5
+
6
+ Kintama.on_start do
7
+ puts `#{__dir__}/../../bin/sweatshop start --testing`
8
+ sleep $for_everything
9
+ end
10
+
11
+ Kintama.on_finish do
12
+ puts `#{__dir__}/../../bin/sweatshop stop`
13
+ end
data/lib/README.md ADDED
@@ -0,0 +1,12 @@
1
+ Job lifecycle:
2
+
3
+ ```
4
+ in-* -> raw-payloads { payload:, format:, job_name: } ->
5
+ payload-parser -> parsed-payloads { payload:, job_name: } ->
6
+ job-assembler -> jobs { context:, commands:, job_name: } ->
7
+ job-worker
8
+ ```
9
+
10
+ All passing done via the moneta core in queue/*
11
+
12
+ Also queue/watcher for debugging