robot_sweatshop 0.3.3 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -0
  3. data/README.md +11 -11
  4. data/Rakefile +1 -1
  5. data/bin/README.md +23 -0
  6. data/bin/sweatshop +1 -1
  7. data/bin/sweatshop-assembler +101 -0
  8. data/bin/sweatshop-conveyor +64 -0
  9. data/bin/{sweatshop-input-http → sweatshop-input} +13 -7
  10. data/bin/sweatshop-job-dictionary +53 -0
  11. data/bin/sweatshop-payload-parser +28 -28
  12. data/bin/sweatshop-worker +58 -0
  13. data/config.defaults.yaml +6 -4
  14. data/lib/README.md +10 -13
  15. data/lib/robot_sweatshop/cli/common.rb +1 -1
  16. data/lib/robot_sweatshop/cli/config.rb +1 -1
  17. data/lib/robot_sweatshop/cli/job.rb +1 -1
  18. data/lib/robot_sweatshop/cli/start.rb +2 -2
  19. data/lib/robot_sweatshop/config.rb +2 -0
  20. data/lib/robot_sweatshop/connections.rb +17 -0
  21. data/lib/robot_sweatshop/create-config-directories.rb +0 -1
  22. data/lib/robot_sweatshop.rb +1 -2
  23. data/robot_sweatshop.eye +30 -32
  24. data/robot_sweatshop.gemspec +8 -6
  25. data/test/README.md +3 -1
  26. data/test/all.rb +35 -0
  27. data/test/assembler_spec.rb +89 -0
  28. data/test/conveyor_spec.rb +67 -0
  29. data/test/data/weird_job.yaml +6 -0
  30. data/test/end-to-end_spec.rb +19 -13
  31. data/test/input_spec.rb +49 -0
  32. data/test/job_dictionary_spec.rb +70 -0
  33. data/test/payload_parser_spec.rb +37 -26
  34. data/test/shared/helpers/input.rb +73 -0
  35. data/test/shared/helpers/output.rb +14 -0
  36. data/test/shared/helpers.rb +4 -83
  37. data/test/shared/scaffolding.rb +42 -0
  38. data/test/shared/stub.rb +44 -0
  39. data/test/worker_spec.rb +59 -0
  40. metadata +76 -44
  41. data/bin/sweatshop-job-assembler +0 -82
  42. data/bin/sweatshop-job-worker +0 -45
  43. data/bin/sweatshop-queue-broadcaster +0 -20
  44. data/bin/sweatshop-queue-handler +0 -24
  45. data/bin/sweatshop-queue-watcher +0 -18
  46. data/lib/robot_sweatshop/moneta-queue.rb +0 -49
  47. data/lib/robot_sweatshop/queue-helper.rb +0 -32
  48. data/robot_sweatshop.production.eye +0 -7
  49. data/robot_sweatshop.testing.eye +0 -7
  50. data/test/input_http_spec.rb +0 -49
  51. data/test/job_assembler_spec.rb +0 -80
  52. data/test/job_worker_spec.rb +0 -65
  53. data/test/moneta-queue_spec.rb +0 -49
  54. data/test/queue_broadcaster_spec.rb +0 -39
  55. data/test/queue_handler_spec.rb +0 -54
  56. data/test/run_all.rb +0 -3
  57. data/test/shared/process_spawning.rb +0 -16
@@ -1,18 +0,0 @@
1
- #!/usr/bin/env ruby
2
- require_relative 'lib/moneta-queue'
3
-
4
- @queues = []
5
- [ARGV[0], "mirror-#{ARGV[0]}"].each do |queue_name|
6
- @queues.push(name: queue_name, queue: MonetaQueue.new(queue_name))
7
- end
8
-
9
- loop do
10
- system 'clear'
11
- @queues.each do |q|
12
- puts "Queue: #{q[:name]}"
13
- puts "Size: #{q[:queue].size}", "#{'|' * q[:queue].size}"
14
- puts q[:queue].inspect
15
- puts
16
- end
17
- sleep 0.1
18
- end
@@ -1,49 +0,0 @@
1
- #!/usr/bin/env ruby
2
- require 'moneta'
3
- require_relative 'config'
4
-
5
- # A class to handle queueing through Moneta's key-value storage
6
- class MonetaQueue
7
- attr_reader :watched_queues
8
-
9
- @@store = Moneta.new :File, dir: File.expand_path(configatron.moneta_path)
10
-
11
- def initialize(name)
12
- @name = name
13
- @mirror_name = "mirror-#{name}"
14
- @@store[@name] ||= []
15
- @@store[@mirror_name] ||= []
16
- end
17
-
18
- def self.watched_queues
19
- %w(payload parsed-payload jobs testing)
20
- end
21
-
22
- def enqueue(item)
23
- @@store[@name] = @@store[@name].push item
24
- @@store[@mirror_name] = @@store[@mirror_name].push item
25
- end
26
-
27
- def dequeue
28
- return '' if @@store[@name].empty?
29
- item = @@store[@name].first
30
- @@store[@name] = @@store[@name][1..-1]
31
- item
32
- end
33
-
34
- def size
35
- loop do # Moneta can return nil sometimes, so we give it time to catch up
36
- queue = @@store[@name]
37
- return queue.size unless queue.nil?
38
- end
39
- end
40
-
41
- def clear
42
- @@store[@mirror_name] = []
43
- @@store[@name] = []
44
- end
45
-
46
- def inspect
47
- @@store[@name].inspect
48
- end
49
- end
@@ -1,32 +0,0 @@
1
- #!/usr/bin/env ruby
2
- require 'ezmq'
3
- require 'json'
4
-
5
- # A collection of common methods for queue interactions with EZMQ
6
- module QueueHelper
7
- @@client = EZMQ::Client.new port: 5556
8
-
9
- def self.dequeue(queue_name = 'default')
10
- data = @@client.request queue_name
11
- begin
12
- JSON.parse data
13
- rescue JSON::ParserError
14
- nil
15
- end
16
- end
17
-
18
- def self.enqueue(object = {}, to: 'default')
19
- @@client.request "#{to} #{JSON.generate object}"
20
- end
21
-
22
- def self.wait_for(queue_name = 'default')
23
- puts "Waiting for messages on #{queue_name}"
24
- subscriber = EZMQ::Subscriber.new port: 5557, topic: 'busy-queues'
25
- subscriber.listen do |message|
26
- if message == queue_name
27
- data = dequeue queue_name
28
- yield data unless data.nil?
29
- end
30
- end
31
- end
32
- end
@@ -1,7 +0,0 @@
1
- require 'configatron'
2
-
3
- configatron.temp do
4
- configatron.eye.broadcaster_interval = ''
5
- configatron.eye.worker_id = ''
6
- Eye.load('robot_sweatshop.eye')
7
- end
@@ -1,7 +0,0 @@
1
- require 'configatron'
2
-
3
- configatron.temp do
4
- configatron.eye.broadcaster_interval = 0.0
5
- configatron.eye.worker_id = 'testingid'
6
- Eye.load('robot_sweatshop.eye')
7
- end
@@ -1,49 +0,0 @@
1
- require 'kintama'
2
- require 'ezmq'
3
- require 'json'
4
- require 'timeout'
5
- require 'http'
6
- require_relative 'shared/process_spawning'
7
- require_relative 'shared/helpers'
8
-
9
- given 'the HTTP Input' do
10
- include QueueHelper
11
- include InHelper
12
-
13
- setup do
14
- @subscriber = EZMQ::Subscriber.new port: 5557, topic: 'busy-queues'
15
- @client = EZMQ::Client.new port: 5556
16
- @payload_queue = 'payload'
17
- clear_all_queues
18
- end
19
-
20
- %w(Bitbucket Github JSON Empty).each do |format|
21
- context "POSTing #{format} data" do
22
- setup do
23
- url = input_http_url for_job: 'test_job'
24
- HTTP.post url, body: example_raw_payload(of_format: format)
25
- end
26
-
27
- should 'enqueue to \'payload\'' do
28
- Timeout.timeout($for_a_moment) do
29
- @subscriber.listen do |message, topic|
30
- break if message == @payload_queue
31
- end
32
- end
33
- end
34
-
35
- should 'enqueue payload details and format' do
36
- response = @client.request "mirror-#{@payload_queue}"
37
- data = JSON.parse response
38
- assert_kind_of String, data['payload']
39
- assert_kind_of String, data['user_agent']
40
- end
41
-
42
- should 'enqueue job name' do
43
- response = @client.request "mirror-#{@payload_queue}"
44
- data = JSON.parse response
45
- assert_kind_of String, data['job_name']
46
- end
47
- end
48
- end
49
- end
@@ -1,80 +0,0 @@
1
- require 'kintama'
2
- require 'ezmq'
3
- require 'json'
4
- require_relative 'shared/process_spawning'
5
- require_relative 'shared/helpers'
6
-
7
- describe 'the Job Assembler' do
8
- include QueueHelper
9
- include InHelper
10
- include JobHelper
11
-
12
- setup do
13
- @client = EZMQ::Client.new port: 5556
14
- @payloads_queue = 'payload'
15
- @jobs_queue = 'jobs'
16
- clear_all_queues
17
- end
18
-
19
- %w(Git JSON MinimalJob).each do |request|
20
- given "#{request} requests in \'payload\'" do
21
- setup do
22
- payload = example_job_request of_type: request
23
- @client.request "#{@payloads_queue} #{payload}"
24
- sleep $for_a_moment
25
- end
26
-
27
- should 'remove the request from \'payload\'' do
28
- response = @client.request @payloads_queue
29
- assert_equal '', response
30
- end
31
-
32
- should 'enqueue commands, context, and job name to \'jobs\'' do
33
- response = @client.request "mirror-#{@jobs_queue}"
34
- response = JSON.load response
35
- assert_kind_of Hash, response['context']
36
- assert_kind_of Array, response['commands']
37
- assert_kind_of String, response['job_name']
38
- end
39
-
40
- should 'store everything in the context as strings' do
41
- response = @client.request "mirror-#{@jobs_queue}"
42
- response = JSON.load response
43
- response['context'].each { |_key, value| assert_kind_of String, value }
44
- end
45
-
46
- unless request == 'EmptyPayload'
47
- should 'build the context with a parsed payload' do
48
- response = @client.request "mirror-#{@jobs_queue}"
49
- response = JSON.load response
50
- assert_kind_of Hash, response['context']
51
- if request == 'Git'
52
- assert_equal 'develop', response['context']['branch']
53
- else
54
- assert_equal 'value', response['context']['test1']
55
- end
56
- end
57
- end
58
- end
59
- end
60
-
61
- %w(IgnoredBranch UnknownJob EmptyJob NonJSON).each do |request|
62
- given "#{request} requests in \'payload\'" do
63
- setup do
64
- payload = example_job_request of_type: request
65
- @client.request "#{@payloads_queue} #{payload}"
66
- sleep $for_a_moment
67
- end
68
-
69
- should 'remove the request from \'payload\'' do
70
- response = @client.request @payloads_queue
71
- assert_equal '', response
72
- end
73
-
74
- should 'not queue anything to \'jobs\'' do
75
- response = @client.request @jobs_queue
76
- assert_equal '', response
77
- end
78
- end
79
- end
80
- end
@@ -1,65 +0,0 @@
1
- require 'kintama'
2
- require 'ezmq'
3
- require_relative 'shared/process_spawning'
4
- require_relative 'shared/helpers'
5
-
6
- describe 'the Worker' do
7
- include QueueHelper
8
- include JobHelper
9
-
10
- setup do
11
- @client = EZMQ::Client.new port: 5556
12
- @jobs_queue = 'jobs'
13
- @test_file = reset_test_file
14
- clear_all_queues
15
- end
16
-
17
- given 'valid job data in \'jobs\'' do
18
- setup do
19
- job = example_job in_context: {custom: 'Hello world!'},
20
- with_commands: ['echo $custom','echo $custom > test.txt']
21
- @client.request "#{@jobs_queue} #{job}"
22
- end
23
-
24
- should 'remove it from \'jobs\'' do
25
- sleep $for_a_moment
26
- response = @client.request @jobs_queue
27
- assert_equal '', response
28
- end
29
-
30
- should 'run the dequeued job' do
31
- sleep $for_io_calls
32
- assert_equal true, File.file?(@test_file)
33
- end
34
-
35
- should 'run jobs with the context as environment variables' do
36
- sleep $for_io_calls
37
- assert_equal "Hello world!\n", File.read(@test_file)
38
- end
39
- end
40
-
41
- given 'invalid job data in \'jobs\'' do
42
- setup do
43
- invalid_data = {
44
- bad_context: example_job(in_context: 'not hash'),
45
- bad_commands: example_job(with_commands: 'echo 1'),
46
- not_json: 'not_json'
47
- }
48
- invalid_data.each do |_type, datum|
49
- @client.request "#{@jobs_queue} #{datum}"
50
- end
51
- end
52
-
53
- should 'remove all of it from \'jobs\'' do
54
- sleep $for_a_moment
55
- response = @client.request @jobs_queue
56
- assert_equal '', response
57
- end
58
-
59
- should 'not run anything' do
60
- sleep $for_io_calls
61
- response = @client.request @jobs_queue
62
- assert_equal false, File.file?(@test_file)
63
- end
64
- end
65
- end
@@ -1,49 +0,0 @@
1
- require 'bundler/setup'
2
- require 'kintama'
3
- require 'robot_sweatshop/moneta-queue'
4
- require_relative 'shared/helpers'
5
-
6
- given 'the Moneta Queue class' do
7
- include QueueHelper
8
-
9
- setup do
10
- clear_all_queues
11
- end
12
-
13
- should 'return a list of actively watched queues' do
14
- assert_kind_of Array, MonetaQueue.watched_queues
15
- end
16
-
17
- context 'an instance' do
18
- setup do
19
- @file_queue = MonetaQueue.new 'testing'
20
- @file_queue.clear
21
- end
22
-
23
- should 'return size' do
24
- assert_equal @file_queue.size, 0
25
- end
26
-
27
- should 'enqueue items' do
28
- @file_queue.enqueue 'item'
29
- assert_equal @file_queue.size, 1
30
- end
31
-
32
- should 'dequeue items' do
33
- @file_queue.enqueue 'item1'
34
- @file_queue.enqueue 'item2'
35
- assert_equal @file_queue.size, 2
36
- assert_equal @file_queue.dequeue, 'item1'
37
- end
38
-
39
- should 'clear items' do
40
- @file_queue.enqueue 'item'
41
- @file_queue.clear
42
- assert_equal @file_queue.size, 0
43
- end
44
-
45
- should 'return the queue on inspect' do
46
- assert_equal @file_queue.inspect, '[]'
47
- end
48
- end
49
- end
@@ -1,39 +0,0 @@
1
- require 'kintama'
2
- require 'ezmq'
3
- require 'timeout'
4
- require_relative 'shared/process_spawning'
5
- require_relative 'shared/helpers'
6
-
7
- given 'the Queue Broadcaster' do
8
- include QueueHelper
9
-
10
- setup do
11
- @subscriber = EZMQ::Subscriber.new port: 5557, topic: 'busy-queues'
12
- @item = 'item'
13
- @queue = 'testing'
14
- clear_all_queues
15
- end
16
-
17
- context 'a non-empty queue' do
18
- setup { enqueue @queue, @item }
19
-
20
- should 'have their named published to \'busy-queues\'' do
21
- Timeout.timeout($for_a_while) do
22
- @subscriber.listen do |message|
23
- assert_equal @queue, message
24
- break
25
- end
26
- end
27
- end
28
- end
29
-
30
- context 'an empty queue' do
31
- should 'not have their name published' do
32
- assert_raises Timeout::Error do
33
- Timeout.timeout($for_a_moment) do
34
- @subscriber.listen {}
35
- end
36
- end
37
- end
38
- end
39
- end
@@ -1,54 +0,0 @@
1
- require 'kintama'
2
- require 'ezmq'
3
- require_relative 'shared/process_spawning'
4
- require_relative 'shared/helpers'
5
-
6
- given 'the Queue Handler' do
7
- include QueueHelper
8
-
9
- setup do
10
- @client = EZMQ::Client.new port: 5556
11
- @item = 'item'
12
- @queue = 'testing'
13
- clear_all_queues
14
- end
15
-
16
- context 'dequeuing' do
17
- setup { @request = "#{@queue}" }
18
-
19
- should 'return the next queued item' do
20
- enqueue @queue, @item
21
- response = @client.request @request
22
- assert_equal @item, response
23
- end
24
-
25
- should 'return \'\' for an empty queue' do
26
- response = @client.request @request
27
- assert_equal '', response
28
- end
29
- end
30
-
31
- context 'enqueuing' do
32
- setup { @request = "#{@queue} #{@item}" }
33
-
34
- should 'return the queue new size' do
35
- response = @client.request @request
36
- assert_equal '1', response
37
- end
38
- end
39
-
40
- context 'queue mirroring' do
41
- should 'mirror queue enqueuing' do
42
- enqueue @queue, @item
43
- response = @client.request "mirror-#{@queue}"
44
- assert_equal @item, response
45
- end
46
-
47
- should 'mirror queue clearing' do
48
- enqueue @queue, @item
49
- clear_all_queues
50
- response = @client.request "mirror-#{@queue}"
51
- assert_equal '', response
52
- end
53
- end
54
- end
data/test/run_all.rb DELETED
@@ -1,3 +0,0 @@
1
- Dir.glob("#{__dir__}/*_spec.rb").each do |spec_file|
2
- require_relative spec_file
3
- end
@@ -1,16 +0,0 @@
1
- $for_a_moment = 0.2
2
- $for_a_while = 0.5
3
- $for_io_calls = 1
4
- $for_everything = 2
5
-
6
- Kintama.on_start do
7
- puts `#{__dir__}/../../bin/sweatshop start --testing`
8
- %w(test minimal git empty).each do |job_file|
9
- FileUtils.cp "#{__dir__}/../data/#{job_file}_job.yaml", File.expand_path(configatron.job_path)
10
- end
11
- sleep $for_everything
12
- end
13
-
14
- Kintama.on_finish do
15
- puts `#{__dir__}/../../bin/sweatshop stop`
16
- end