robot_sweatshop 0.2.1 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +14 -8
- data/bin/sweatshop +1 -1
- data/bin/sweatshop-input-http +8 -7
- data/bin/sweatshop-job-assembler +78 -41
- data/bin/sweatshop-job-worker +1 -1
- data/bin/sweatshop-payload-parser +67 -18
- data/config.defaults.yaml +3 -0
- data/kintama/data/git_job.yaml +9 -0
- data/kintama/data/payload_data.yaml +3 -2
- data/kintama/data/test_job.yaml +2 -4
- data/kintama/end-to-end_spec.rb +3 -6
- data/kintama/input_http_spec.rb +18 -14
- data/kintama/job_assembler_spec.rb +48 -48
- data/kintama/payload_parser_spec.rb +34 -43
- data/kintama/shared/helpers.rb +43 -18
- data/kintama/shared/process_spawning.rb +2 -2
- data/lib/README.md +11 -2
- data/lib/sweatshop/cli/job.rb +1 -1
- data/lib/sweatshop/cli.rb +4 -0
- data/lib/sweatshop/moneta-queue.rb +1 -1
- data/lib/sweatshop/payload/json.rb +13 -0
- data/lib/sweatshop/payload.rb +4 -0
- data/lib/sweatshop.rb +2 -13
- data/robot_sweatshop.gemspec +3 -2
- metadata +21 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: cbc73c13d57e32651f63077a89b03317769850f3
|
4
|
+
data.tar.gz: 834bb91075c655c1edc0ac7f4cc445c2048094db
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 16784f92f5abe86c20dd6aebe73d79d3d9d18296576ba50ed0018467b205038fb67517a18931a539b4309839f311427b079c5b3b0e4708f7120d7e0baf570365
|
7
|
+
data.tar.gz: bc6eca3c50fddb92091e7693b7f4aba3232ab96b857a3b495d7a20e557bff3f695cda3149842b718d5c5905011d04bdb4d6d9decaf3cca8a7dcd3c062b8a2e1e
|
data/README.md
CHANGED
@@ -8,11 +8,11 @@ Robot Sweatshop is a single-purpose CI server that runs collections of arbitrary
|
|
8
8
|
|
9
9
|
# Quick start
|
10
10
|
|
11
|
-
- install [ZMQ as described in the EZMQ gem](https://github.com/colstrom/ezmq)
|
11
|
+
- install [ZMQ as described in the EZMQ gem](https://github.com/colstrom/ezmq#operating-system-notes)
|
12
12
|
- `gem install robot_sweatshop`
|
13
13
|
- `sweatshop start` ([you may need sudo on OSX](https://github.com/JScott/robot_sweatshop/wiki))
|
14
14
|
- `sweatshop job example --auto`
|
15
|
-
-
|
15
|
+
- `curl --data '{"your": "json"}' http://localhost:8080/payload-for/example`
|
16
16
|
- `cat .robot_sweatshop/log/job-worker.log`
|
17
17
|
|
18
18
|
# Usage
|
@@ -29,6 +29,7 @@ By default, Robot Sweatshop looks in your current working directory to configure
|
|
29
29
|
|
30
30
|
- Github (application/json format only)
|
31
31
|
- Bitbucket
|
32
|
+
- JSON
|
32
33
|
|
33
34
|
# Security
|
34
35
|
|
@@ -36,12 +37,17 @@ You probably don't want to run Robot Sweatshop as a sudo user. Create a testing
|
|
36
37
|
|
37
38
|
# Roadmap
|
38
39
|
|
39
|
-
|
40
|
-
|
41
|
-
- Common scrips such as git repo syncing and creating a job run ID
|
42
|
-
- Support for multiple workers
|
43
|
-
- Better logging for the processes
|
44
|
-
- Use [eye-http](https://github.com/kostya/eye-http) for the '/' route?
|
40
|
+
1.0
|
41
|
+
|
45
42
|
- Improved architecture:
|
46
43
|
|
47
44
|
![Improved architecture diagram](http://40.media.tumblr.com/8a5b6ca59c0d93c4ce6fc6b733932a5f/tumblr_nko478zp9N1qh941oo1_1280.jpg)
|
45
|
+
|
46
|
+
- Support for multiple workers
|
47
|
+
|
48
|
+
Beyond
|
49
|
+
|
50
|
+
- Better logging for the processes
|
51
|
+
- CLI configuration via chomp and/or flags
|
52
|
+
- Common scrips such as git repo syncing and creating a job run ID
|
53
|
+
- Use [eye-http](https://github.com/kostya/eye-http) for the '/' route?
|
data/bin/sweatshop
CHANGED
@@ -10,7 +10,7 @@ require_relative '../lib/sweatshop/config'
|
|
10
10
|
require_relative '../lib/sweatshop/create-config-directories'
|
11
11
|
|
12
12
|
program :name, 'Robot Sweatshop'
|
13
|
-
program :version, '0.
|
13
|
+
program :version, '0.3.0'
|
14
14
|
program :description, 'A lightweight, unopinionated CI server'
|
15
15
|
program :help, 'Author', 'Justin Scott <jvscott@gmail.com>'
|
16
16
|
|
data/bin/sweatshop-input-http
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
|
+
require 'bundler/setup'
|
2
3
|
require 'sinatra'
|
3
4
|
require 'ezmq'
|
4
5
|
require 'json'
|
5
|
-
|
6
|
+
require 'sweatshop/config'
|
6
7
|
|
7
8
|
configure do
|
8
9
|
set :port, configatron.http_port
|
9
10
|
set :bind, configatron.http_bind
|
10
|
-
set :output_queue, 'raw-payload'
|
11
11
|
set :run, true
|
12
12
|
end
|
13
13
|
|
@@ -15,14 +15,15 @@ get '/' do
|
|
15
15
|
'Everything\'s on schedule!'
|
16
16
|
end
|
17
17
|
|
18
|
-
post '
|
19
|
-
puts "Received
|
18
|
+
post '/payload-for/:job_name' do
|
19
|
+
puts "Received payload for #{params['job_name']}"
|
20
20
|
request.body.rewind
|
21
|
+
payload = request.body.read
|
21
22
|
hash = {
|
22
|
-
payload:
|
23
|
-
|
23
|
+
payload: payload,
|
24
|
+
user_agent: request.env['HTTP_USER_AGENT'],
|
24
25
|
job_name: params['job_name']
|
25
26
|
}
|
26
27
|
client = EZMQ::Client.new port: 5556
|
27
|
-
client.request "
|
28
|
+
client.request "payload #{JSON.generate hash}"
|
28
29
|
end
|
data/bin/sweatshop-job-assembler
CHANGED
@@ -1,45 +1,82 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
|
+
require 'bundler/setup'
|
2
3
|
require 'yaml'
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
def
|
19
|
-
|
20
|
-
end
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
return
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
puts "
|
41
|
-
|
4
|
+
require 'json'
|
5
|
+
require 'contracts'
|
6
|
+
require 'sweatshop/queue-helper'
|
7
|
+
require 'sweatshop/config'
|
8
|
+
|
9
|
+
$stdout.sync = true
|
10
|
+
include Contracts
|
11
|
+
|
12
|
+
Contract None => Hash
|
13
|
+
def empty_config
|
14
|
+
puts "Job configuration not found"
|
15
|
+
{}
|
16
|
+
end
|
17
|
+
|
18
|
+
Contract None => String
|
19
|
+
def job_directory
|
20
|
+
File.expand_path configatron.job_directory
|
21
|
+
end
|
22
|
+
|
23
|
+
Contract Or[String, nil], Hash => Bool
|
24
|
+
def whitelisted?(branch, in_config:)
|
25
|
+
return true if in_config['branch_whitelist'].nil?
|
26
|
+
in_config['branch_whitelist'].include? branch
|
27
|
+
end
|
28
|
+
|
29
|
+
Contract Hash => Hash
|
30
|
+
def sanitize(data)
|
31
|
+
data.each_pair { |key, value| data[key] = value.to_s }
|
32
|
+
end
|
33
|
+
|
34
|
+
Contract Hash, Hash => Hash
|
35
|
+
def job_context(job_environment, context_from_payload)
|
36
|
+
sanitize job_environment.merge(context_from_payload)
|
37
|
+
end
|
38
|
+
|
39
|
+
Contract String => Or[Hash, nil]
|
40
|
+
def load_if_exists(config)
|
41
|
+
puts "Reading job configuration from #{config}"
|
42
|
+
YAML.load_file config if File.exists? config
|
43
|
+
end
|
44
|
+
|
45
|
+
Contract String => Hash
|
46
|
+
def load_config_for(job_name)
|
47
|
+
load_if_exists("#{job_directory}/#{job_name}.yaml") || empty_config
|
48
|
+
end
|
49
|
+
|
50
|
+
Contract Hash => [Hash, String]
|
51
|
+
def parse_payload(request)
|
52
|
+
client = EZMQ::Client.new port: configatron.payload_parser_port
|
53
|
+
response = JSON.load client.request(JSON.dump request)
|
54
|
+
puts response['error'] unless response['error'].empty?
|
55
|
+
[response['payload'], response['error']]
|
56
|
+
end
|
57
|
+
|
58
|
+
Contract Hash, Hash => Bool
|
59
|
+
def can_work_with?(job_config, payload)
|
60
|
+
return false if job_config.empty?
|
61
|
+
return false unless whitelisted? payload['branch'], in_config: job_config
|
62
|
+
true
|
63
|
+
end
|
64
|
+
|
65
|
+
Contract Hash => Or[Hash, false]
|
66
|
+
def assemble_job(request)
|
67
|
+
job_config = load_config_for request['job_name']
|
68
|
+
payload, error = parse_payload request
|
69
|
+
return false unless error.empty? && can_work_with?(job_config, payload)
|
70
|
+
{
|
71
|
+
commands: job_config['commands'],
|
72
|
+
context: job_context(job_config['environment'], payload),
|
73
|
+
job_name: request['job_name']
|
74
|
+
}
|
75
|
+
end
|
76
|
+
|
77
|
+
puts 'Started'
|
78
|
+
QueueHelper.wait_for('payload') do |request|
|
79
|
+
puts "Assembling: #{request}"
|
80
|
+
assembled_job = assemble_job request
|
42
81
|
QueueHelper.enqueue assembled_job, to: 'jobs' if assembled_job
|
43
|
-
$stdout.flush
|
44
|
-
$stderr.flush
|
45
82
|
end
|
data/bin/sweatshop-job-worker
CHANGED
@@ -21,7 +21,7 @@ end
|
|
21
21
|
def execute(context = {}, command)
|
22
22
|
puts "Executing '#{command}'..."
|
23
23
|
# TODO: path.split(' ') to bypass the shell when we're not using env vars
|
24
|
-
|
24
|
+
|
25
25
|
# Run the command with the context in environment,
|
26
26
|
# printing the output as it's generated
|
27
27
|
IO.popen(context, command) do |io_stream|
|
@@ -1,23 +1,72 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
2
|
+
require 'bundler/setup'
|
3
|
+
require 'json'
|
4
|
+
require 'ezmq'
|
5
|
+
require 'contracts'
|
6
|
+
require 'sweatshop/config'
|
7
|
+
require 'sweatshop/payload'
|
8
|
+
|
9
|
+
$stdout.sync = true
|
10
|
+
include Contracts
|
11
|
+
|
12
|
+
Contract String => Bool
|
13
|
+
def json?(string)
|
14
|
+
begin
|
15
|
+
JSON.load string
|
16
|
+
true
|
17
|
+
rescue JSON::ParserError => e
|
18
|
+
false
|
13
19
|
end
|
14
20
|
end
|
15
21
|
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
22
|
+
Contract String => Bool
|
23
|
+
def from_github?(user_agent)
|
24
|
+
user_agent.start_with? 'Github-Hookshot'
|
25
|
+
end
|
26
|
+
|
27
|
+
Contract String => Bool
|
28
|
+
def from_bitbucket?(user_agent)
|
29
|
+
user_agent.start_with? 'Bitbucket.org'
|
30
|
+
end
|
31
|
+
|
32
|
+
Contract Hash => String
|
33
|
+
def detect_format_of(request)
|
34
|
+
return 'empty' if request['payload'].empty?
|
35
|
+
return 'Github' if from_github? request['user_agent']
|
36
|
+
return 'Bitbucket' if from_bitbucket? request['user_agent']
|
37
|
+
return 'JSON' if json? request['payload']
|
38
|
+
'unsupported'
|
39
|
+
end
|
40
|
+
|
41
|
+
Contract String, String => Or[Hash, nil]
|
42
|
+
def payload_hash_from(payload, format)
|
43
|
+
lib_file = "#{__dir__}/../lib/sweatshop/payload/#{format}.rb"
|
44
|
+
return nil unless File.file?(lib_file)
|
45
|
+
require_relative lib_file
|
46
|
+
Object.const_get("#{format}Payload").new(payload).to_hash
|
47
|
+
end
|
48
|
+
|
49
|
+
Contract Hash => Hash
|
50
|
+
def parse(request)
|
51
|
+
format = detect_format_of request
|
52
|
+
payload = payload_hash_from request['payload'], format
|
53
|
+
return {payload: {}, error: "Can't parse #{format} payload"} if payload.nil?
|
54
|
+
{payload: payload, error: ''}
|
55
|
+
end
|
56
|
+
|
57
|
+
Contract Any => Hash
|
58
|
+
def validate(request)
|
59
|
+
return {payload: {}, error: "Invalid JSON request"} unless json? request
|
60
|
+
request = JSON.load request
|
61
|
+
return {payload: {}, error: "Empty request"} if request.nil?
|
62
|
+
request
|
63
|
+
end
|
64
|
+
|
65
|
+
puts 'Started'
|
66
|
+
server = EZMQ::Server.new port: configatron.payload_parser_port#, encode: -> m { JSON.dump m }
|
67
|
+
server.listen do |request|
|
68
|
+
puts "Parsing: #{request}"
|
69
|
+
request = validate request
|
70
|
+
request = parse request unless request[:error]
|
71
|
+
JSON.dump request
|
23
72
|
end
|
data/config.defaults.yaml
CHANGED
@@ -1,7 +1,8 @@
|
|
1
1
|
---
|
2
2
|
# http://www.url-encode-decode.com/
|
3
|
-
|
4
|
-
|
3
|
+
empty: ''
|
4
|
+
nonjson: not json
|
5
|
+
json: '{"test1": "value", "test2": "value"}'
|
5
6
|
bitbucket: |
|
6
7
|
payload=%7B%22repository%22%3A+%7B%22website%22%3A+%22%22%2C+%22fork%22%3A+false%2C+%22name%22%3A+%22Project+X%22%2C+%22scm%22%3A+%22git%22%2C+%22owner%22%3A+%22marcus%22%2C+%22absolute_url%22%3A+%22%2Fmarcus%2Fproject-x%2F%22%2C+%22slug%22%3A+%22project-x%22%2C+%22is_private%22%3A+true%7D%2C+%22truncated%22%3A+false%2C+%22commits%22%3A+%5B%7B%22node%22%3A+%22620ade18607a%22%2C+%22files%22%3A+%5B%7B%22type%22%3A+%22modified%22%2C+%22file%22%3A+%22README%22%7D%5D%2C+%22raw_author%22%3A+%22Marcus+Bertrand+%3Cmarcus%40somedomain.com%3E%22%2C+%22utctimestamp%22%3A+%222014-08-08+00%3A04%3A07%2B00%3A00%22%2C+%22author%22%3A+%22marcus%22%2C+%22timestamp%22%3A+%222014-08-08+02%3A04%3A07%22%2C+%22raw_node%22%3A+%22620ade18607ac42d872b568bb92acaa9a28620e9%22%2C+%22parents%22%3A+%5B%22702c70160afc%22%5D%2C+%22branch%22%3A+%22develop%22%2C+%22message%22%3A+%22Added+some+more+things+to+somefile.py%5Cn%22%2C+%22revision%22%3A+null%2C+%22size%22%3A+-1%7D%5D%2C+%22canon_url%22%3A+%22https%3A%2F%2Fbitbucket.org%22%2C+%22user%22%3A+%22marcus%22%7D
|
7
8
|
github: |
|
data/kintama/data/test_job.yaml
CHANGED
data/kintama/end-to-end_spec.rb
CHANGED
@@ -11,18 +11,15 @@ describe 'Robot Sweatshop' do
|
|
11
11
|
|
12
12
|
setup do
|
13
13
|
@client = EZMQ::Client.new port: 5556
|
14
|
-
@job_name = 'test_job'
|
15
14
|
@test_file = reset_test_file
|
16
15
|
clear_all_queues
|
17
16
|
end
|
18
17
|
|
19
18
|
context "POST git data to the HTTP Input" do
|
20
19
|
setup do
|
21
|
-
url = input_http_url for_job:
|
22
|
-
|
23
|
-
|
24
|
-
puts "post successful"
|
25
|
-
sleep $for_everything
|
20
|
+
url = input_http_url for_job: 'test_job'
|
21
|
+
HTTP.post url, body: example_raw_payload(of_format: 'JSON')
|
22
|
+
sleep $for_io_calls
|
26
23
|
end
|
27
24
|
|
28
25
|
should 'run jobs with the context as environment variables' do
|
data/kintama/input_http_spec.rb
CHANGED
@@ -13,32 +13,36 @@ given 'the HTTP Input' do
|
|
13
13
|
setup do
|
14
14
|
@subscriber = EZMQ::Subscriber.new port: 5557, topic: 'busy-queues'
|
15
15
|
@client = EZMQ::Client.new port: 5556
|
16
|
-
@
|
17
|
-
@raw_payload_queue = 'raw-payload'
|
16
|
+
@payload_queue = 'payload'
|
18
17
|
clear_all_queues
|
19
18
|
end
|
20
|
-
|
21
|
-
%w(Bitbucket).each do |
|
22
|
-
context "
|
19
|
+
|
20
|
+
%w(Bitbucket Github JSON).each do |format|
|
21
|
+
context "POSTing #{format} data" do
|
23
22
|
setup do
|
24
|
-
url = input_http_url for_job:
|
25
|
-
HTTP.post url, body:
|
23
|
+
url = input_http_url for_job: 'test_job'
|
24
|
+
HTTP.post url, body: example_raw_payload(of_format: format)
|
26
25
|
end
|
27
26
|
|
28
|
-
should 'enqueue to \'
|
27
|
+
should 'enqueue to \'payload\'' do
|
29
28
|
Timeout.timeout($for_a_moment) do
|
30
29
|
@subscriber.listen do |message, topic|
|
31
|
-
break if message == @
|
30
|
+
break if message == @payload_queue
|
32
31
|
end
|
33
32
|
end
|
34
33
|
end
|
35
34
|
|
36
|
-
should 'enqueue payload details' do
|
37
|
-
response = @client.request "mirror-#{@
|
35
|
+
should 'enqueue payload details and format' do
|
36
|
+
response = @client.request "mirror-#{@payload_queue}"
|
38
37
|
data = JSON.parse response
|
39
|
-
|
40
|
-
|
41
|
-
|
38
|
+
assert_kind_of String, data['payload']
|
39
|
+
assert_kind_of String, data['user_agent']
|
40
|
+
end
|
41
|
+
|
42
|
+
should 'enqueue job name' do
|
43
|
+
response = @client.request "mirror-#{@payload_queue}"
|
44
|
+
data = JSON.parse response
|
45
|
+
assert_kind_of String, data['job_name']
|
42
46
|
end
|
43
47
|
end
|
44
48
|
end
|
@@ -6,73 +6,73 @@ require_relative 'shared/helpers'
|
|
6
6
|
|
7
7
|
describe 'the Job Assembler' do
|
8
8
|
include QueueHelper
|
9
|
-
include
|
9
|
+
include InHelper
|
10
10
|
include JobHelper
|
11
11
|
|
12
12
|
setup do
|
13
13
|
@client = EZMQ::Client.new port: 5556
|
14
|
-
@
|
14
|
+
@payloads_queue = 'payload'
|
15
15
|
@jobs_queue = 'jobs'
|
16
16
|
clear_all_queues
|
17
17
|
end
|
18
18
|
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
19
|
+
%w(Git JSON).each do |request|
|
20
|
+
given "#{request} requests in \'payload\'" do
|
21
|
+
setup do
|
22
|
+
payload = example_job_request of_type: request
|
23
|
+
@client.request "#{@payloads_queue} #{payload}"
|
24
|
+
sleep $for_a_moment
|
25
|
+
end
|
25
26
|
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
27
|
+
should 'remove the request from \'payload\'' do
|
28
|
+
response = @client.request @payloads_queue
|
29
|
+
assert_equal '', response
|
30
|
+
end
|
30
31
|
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
32
|
+
should 'enqueue commands, context, and job name to \'jobs\'' do
|
33
|
+
response = @client.request "mirror-#{@jobs_queue}"
|
34
|
+
response = JSON.load response
|
35
|
+
assert_kind_of Hash, response['context']
|
36
|
+
assert_kind_of Array, response['commands']
|
37
|
+
assert_kind_of String, response['job_name']
|
38
|
+
end
|
38
39
|
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
40
|
+
should 'store everything in the context as strings' do
|
41
|
+
response = @client.request "mirror-#{@jobs_queue}"
|
42
|
+
response = JSON.load response
|
43
|
+
response['context'].each { |_key, value| assert_kind_of String, value }
|
44
|
+
end
|
44
45
|
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
46
|
+
should 'build the context with a parsed payload' do
|
47
|
+
response = @client.request "mirror-#{@jobs_queue}"
|
48
|
+
response = JSON.load response
|
49
|
+
assert_kind_of Hash, response['context']
|
50
|
+
if request == 'Git'
|
51
|
+
assert_equal 'develop', response['context']['branch']
|
52
|
+
else
|
53
|
+
assert_equal 'value', response['context']['test1']
|
54
|
+
end
|
50
55
|
end
|
51
56
|
end
|
52
57
|
end
|
53
58
|
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
not_json: 'not_json'
|
61
|
-
}
|
62
|
-
invalid_data.each do |_type, datum|
|
63
|
-
@client.request "#{@parsed_payloads_queue} #{datum}"
|
59
|
+
%w(IgnoredBranch UnknownJob NonJSON).each do |request|
|
60
|
+
given "#{request} requests in \'payload\'" do
|
61
|
+
setup do
|
62
|
+
payload = example_job_request of_type: request
|
63
|
+
@client.request "#{@payloads_queue} #{payload}"
|
64
|
+
sleep $for_a_moment
|
64
65
|
end
|
65
|
-
sleep $for_a_while
|
66
|
-
end
|
67
66
|
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
67
|
+
should 'remove the request from \'payload\'' do
|
68
|
+
response = @client.request @payloads_queue
|
69
|
+
assert_equal '', response
|
70
|
+
end
|
72
71
|
|
73
|
-
|
74
|
-
|
75
|
-
|
72
|
+
should 'not queue anything to \'jobs\'' do
|
73
|
+
response = @client.request @jobs_queue
|
74
|
+
assert_equal '', response
|
75
|
+
end
|
76
76
|
end
|
77
77
|
end
|
78
78
|
end
|
@@ -1,70 +1,61 @@
|
|
1
1
|
require 'kintama'
|
2
2
|
require 'ezmq'
|
3
3
|
require 'json'
|
4
|
+
require 'timeout'
|
4
5
|
require_relative 'shared/process_spawning'
|
5
6
|
require_relative 'shared/helpers'
|
6
7
|
|
7
8
|
describe 'the Payload Parser' do
|
8
9
|
include QueueHelper
|
9
10
|
include InHelper
|
10
|
-
include PayloadHelper
|
11
11
|
|
12
12
|
setup do
|
13
|
-
@client = EZMQ::Client.new port:
|
14
|
-
@raw_queue = 'raw-payload'
|
15
|
-
@parsed_queue = 'parsed-payload'
|
16
|
-
clear_all_queues
|
13
|
+
@client = EZMQ::Client.new port: configatron.payload_parser_port
|
17
14
|
end
|
18
15
|
|
19
|
-
%w(Bitbucket Github).each do |format|
|
20
|
-
given "valid #{format}
|
16
|
+
%w(Bitbucket Github JSON).each do |format|
|
17
|
+
given "valid #{format} payloads" do
|
21
18
|
setup do
|
22
|
-
payload =
|
23
|
-
@
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
should 'remove it from \'raw-payload\'' do
|
28
|
-
response = @client.request @raw_queue
|
29
|
-
assert_equal '', response
|
19
|
+
payload = example_payload_request of_format: format
|
20
|
+
@response = Timeout.timeout($for_a_while) do
|
21
|
+
@client.request JSON.dump(payload)
|
22
|
+
end
|
30
23
|
end
|
31
24
|
|
32
|
-
should '
|
33
|
-
response =
|
34
|
-
|
25
|
+
should 'return a parsed payload object' do
|
26
|
+
@response = JSON.parse @response
|
27
|
+
assert_equal true, @response['error'].empty?
|
28
|
+
assert_kind_of Hash, @response['payload']
|
35
29
|
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
30
|
+
keys = case format
|
31
|
+
when 'JSON'
|
32
|
+
%w(test1 test2)
|
33
|
+
else
|
34
|
+
Payload.hash_keys
|
35
|
+
end
|
36
|
+
keys.each do |key|
|
37
|
+
payload = @response['payload']
|
38
|
+
assert_not_nil payload[key]
|
39
|
+
assert_not_equal key, payload[key] # catches "string"[key]
|
40
40
|
end
|
41
|
-
assert_kind_of String, response['job_name']
|
42
41
|
end
|
43
42
|
end
|
44
43
|
end
|
45
44
|
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
invalid_data.each do |_type, datum|
|
54
|
-
@client.request "#{@raw_queue} #{datum}"
|
45
|
+
%w(Empty NonJSON).each do |format|
|
46
|
+
given "#{format} payloads" do
|
47
|
+
setup do
|
48
|
+
payload = example_raw_payload of_format: format
|
49
|
+
@response = Timeout.timeout($for_a_while) do
|
50
|
+
@client.request "#{payload}"
|
51
|
+
end
|
55
52
|
end
|
56
|
-
sleep $for_a_while
|
57
|
-
# TODO: should not crash the payload parser
|
58
|
-
end
|
59
53
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
should 'not queue anything to \'parsed-payload\'' do
|
66
|
-
response = @client.request @parsed_queue
|
67
|
-
assert_equal '', response
|
54
|
+
should 'return an error object' do
|
55
|
+
@response = JSON.load(@response)
|
56
|
+
assert_kind_of String, @response['error']
|
57
|
+
assert_equal true, @response['payload'].empty?
|
58
|
+
end
|
68
59
|
end
|
69
60
|
end
|
70
61
|
end
|
data/kintama/shared/helpers.rb
CHANGED
@@ -1,8 +1,9 @@
|
|
1
|
+
require 'bundler/setup'
|
1
2
|
require 'yaml'
|
2
3
|
require 'json'
|
3
|
-
|
4
|
-
|
5
|
-
|
4
|
+
require 'sweatshop/moneta-queue'
|
5
|
+
require 'sweatshop/payload'
|
6
|
+
require 'sweatshop/config'
|
6
7
|
|
7
8
|
module QueueHelper
|
8
9
|
def clear_all_queues
|
@@ -18,26 +19,50 @@ module QueueHelper
|
|
18
19
|
end
|
19
20
|
|
20
21
|
module InHelper
|
21
|
-
def
|
22
|
+
def example_raw_payload(of_format:)
|
22
23
|
payload_strings = YAML.load_file "#{__dir__}/../data/payload_data.yaml"
|
23
24
|
payload_strings[of_format.downcase]
|
24
25
|
end
|
25
|
-
def
|
26
|
-
payload
|
27
|
-
JSON.generate payload: payload,
|
28
|
-
format: with_format,
|
29
|
-
job_name: 'test_job'
|
26
|
+
def input_http_url(for_job: 'test_job')
|
27
|
+
"http://localhost:#{configatron.http_port}/payload-for/#{for_job}"
|
30
28
|
end
|
31
|
-
def
|
32
|
-
|
29
|
+
def user_agent_for(format)
|
30
|
+
case format
|
31
|
+
when 'Bitbucket'
|
32
|
+
'Bitbucket.org'
|
33
|
+
when 'Github'
|
34
|
+
'Github-Hookshot'
|
35
|
+
else
|
36
|
+
'SomeRandomUserAgent'
|
37
|
+
end
|
33
38
|
end
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
39
|
+
def example_payload_request(of_format:)
|
40
|
+
{
|
41
|
+
payload: example_raw_payload(of_format: of_format),
|
42
|
+
user_agent: user_agent_for(of_format)
|
43
|
+
}
|
44
|
+
end
|
45
|
+
def job_request_data(type)
|
46
|
+
case type
|
47
|
+
when 'Git'
|
48
|
+
['Bitbucket', 'git_job'] # develop branch
|
49
|
+
when 'JSON'
|
50
|
+
['JSON', 'test_job']
|
51
|
+
when 'IgnoredBranch'
|
52
|
+
['Github', 'git_job'] # master branch
|
53
|
+
when 'UnknownJob'
|
54
|
+
['Bitbucket', 'unknown_job']
|
55
|
+
when 'NonJSON'
|
56
|
+
['JSON', 'git_job', 'not json']
|
57
|
+
else
|
58
|
+
['I', 'AM', 'ERROR']
|
59
|
+
end
|
60
|
+
end
|
61
|
+
def example_job_request(of_type:)
|
62
|
+
format, job_name, payload = job_request_data of_type
|
63
|
+
JSON.generate payload: payload || example_raw_payload(of_format: format),
|
64
|
+
job_name: job_name,
|
65
|
+
user_agent: user_agent_for(format)
|
41
66
|
end
|
42
67
|
end
|
43
68
|
|
@@ -1,15 +1,15 @@
|
|
1
1
|
$for_a_moment = 0.1
|
2
2
|
$for_a_while = 0.5
|
3
3
|
$for_io_calls = 1
|
4
|
-
$for_everything =
|
4
|
+
$for_everything = 2
|
5
5
|
|
6
6
|
Kintama.on_start do
|
7
7
|
puts `#{__dir__}/../../bin/sweatshop start --testing`
|
8
8
|
FileUtils.cp "#{__dir__}/../data/test_job.yaml", File.expand_path(configatron.job_directory)
|
9
|
+
FileUtils.cp "#{__dir__}/../data/git_job.yaml", File.expand_path(configatron.job_directory)
|
9
10
|
sleep $for_everything
|
10
11
|
end
|
11
12
|
|
12
13
|
Kintama.on_finish do
|
13
14
|
puts `#{__dir__}/../../bin/sweatshop stop`
|
14
|
-
# FileUtils.rm "#{configatron.job_directory}/test_job.yaml"
|
15
15
|
end
|
data/lib/README.md
CHANGED
@@ -1,8 +1,7 @@
|
|
1
1
|
Job lifecycle:
|
2
2
|
|
3
3
|
```
|
4
|
-
in-* ->
|
5
|
-
payload-parser -> parsed-payloads { payload:, job_name: } ->
|
4
|
+
in-* -> payloads { payload:, user_agent:, job_name: } ->
|
6
5
|
job-assembler -> jobs { context:, commands:, job_name: } ->
|
7
6
|
job-worker
|
8
7
|
```
|
@@ -10,3 +9,13 @@ job-worker
|
|
10
9
|
All passing done via the moneta core in queue/*
|
11
10
|
|
12
11
|
Also queue/watcher for debugging
|
12
|
+
|
13
|
+
---
|
14
|
+
|
15
|
+
The Job Assembler also uses the Payload Parser service:
|
16
|
+
|
17
|
+
```
|
18
|
+
{ payload:, user_agent: } -> payload-parser -> { payload:, error: }
|
19
|
+
```
|
20
|
+
|
21
|
+
Error is an empty string when it's successful. Otherwise it details what went wrong.
|
data/lib/sweatshop/cli/job.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
require_relative '../config'
|
2
2
|
|
3
3
|
def default_job
|
4
|
-
"---\
|
4
|
+
"---\n# branch_whitelist:\n# - master\n\ncommands:\n- echo \"Hello $WORLD!\"\n\nenvironment:\n WORLD: Earth\n"
|
5
5
|
end
|
6
6
|
|
7
7
|
def get_job_path(for_job: nil)
|
data/lib/sweatshop.rb
CHANGED
@@ -1,16 +1,5 @@
|
|
1
1
|
require_relative 'sweatshop/moneta-queue'
|
2
2
|
require_relative 'sweatshop/queue-helper'
|
3
3
|
require_relative 'sweatshop/config'
|
4
|
-
|
5
|
-
require_relative 'sweatshop/
|
6
|
-
require_relative 'sweatshop/payload/bitbucket'
|
7
|
-
require_relative 'sweatshop/payload/github'
|
8
|
-
|
9
|
-
require_relative 'sweatshop/cli/common'
|
10
|
-
require_relative 'sweatshop/cli/config'
|
11
|
-
require_relative 'sweatshop/cli/inspect'
|
12
|
-
require_relative 'sweatshop/cli/job'
|
13
|
-
require_relative 'sweatshop/cli/setup'
|
14
|
-
require_relative 'sweatshop/cli/start'
|
15
|
-
|
16
|
-
|
4
|
+
require_relative 'sweatshop/payload'
|
5
|
+
require_relative 'sweatshop/cli'
|
data/robot_sweatshop.gemspec
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |gem|
|
2
2
|
gem.name = 'robot_sweatshop'
|
3
|
-
gem.version = '0.
|
3
|
+
gem.version = '0.3.0'
|
4
4
|
gem.licenses = 'MIT'
|
5
5
|
gem.authors = ['Justin Scott']
|
6
6
|
gem.email = 'jvscott@gmail.com'
|
@@ -20,9 +20,10 @@ Gem::Specification.new do |gem|
|
|
20
20
|
gem.add_runtime_dependency 'faker'
|
21
21
|
gem.add_runtime_dependency 'commander'
|
22
22
|
gem.add_runtime_dependency 'eye'
|
23
|
-
gem.add_runtime_dependency 'colorize'
|
23
|
+
gem.add_runtime_dependency 'colorize' # TODO: replace with rainbow
|
24
24
|
gem.add_runtime_dependency 'configatron'
|
25
25
|
gem.add_runtime_dependency 'moneta'
|
26
|
+
gem.add_runtime_dependency 'contracts'
|
26
27
|
|
27
28
|
gem.add_development_dependency 'rake'
|
28
29
|
gem.add_development_dependency 'kintama'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: robot_sweatshop
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Justin Scott
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-04-
|
11
|
+
date: 2015-04-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: sinatra
|
@@ -122,6 +122,20 @@ dependencies:
|
|
122
122
|
- - ">="
|
123
123
|
- !ruby/object:Gem::Version
|
124
124
|
version: '0'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: contracts
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - ">="
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: '0'
|
132
|
+
type: :runtime
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - ">="
|
137
|
+
- !ruby/object:Gem::Version
|
138
|
+
version: '0'
|
125
139
|
- !ruby/object:Gem::Dependency
|
126
140
|
name: rake
|
127
141
|
requirement: !ruby/object:Gem::Requirement
|
@@ -236,6 +250,7 @@ files:
|
|
236
250
|
- bin/sweatshop-queue-watcher
|
237
251
|
- config.defaults.yaml
|
238
252
|
- kintama/README.md
|
253
|
+
- kintama/data/git_job.yaml
|
239
254
|
- kintama/data/payload_data.yaml
|
240
255
|
- kintama/data/test_job.yaml
|
241
256
|
- kintama/end-to-end_spec.rb
|
@@ -251,6 +266,7 @@ files:
|
|
251
266
|
- kintama/shared/process_spawning.rb
|
252
267
|
- lib/README.md
|
253
268
|
- lib/sweatshop.rb
|
269
|
+
- lib/sweatshop/cli.rb
|
254
270
|
- lib/sweatshop/cli/common.rb
|
255
271
|
- lib/sweatshop/cli/config.rb
|
256
272
|
- lib/sweatshop/cli/job.rb
|
@@ -258,8 +274,10 @@ files:
|
|
258
274
|
- lib/sweatshop/config.rb
|
259
275
|
- lib/sweatshop/create-config-directories.rb
|
260
276
|
- lib/sweatshop/moneta-queue.rb
|
277
|
+
- lib/sweatshop/payload.rb
|
261
278
|
- lib/sweatshop/payload/bitbucket.rb
|
262
279
|
- lib/sweatshop/payload/github.rb
|
280
|
+
- lib/sweatshop/payload/json.rb
|
263
281
|
- lib/sweatshop/payload/payload.rb
|
264
282
|
- lib/sweatshop/queue-helper.rb
|
265
283
|
- robot_sweatshop.eye
|
@@ -291,6 +309,7 @@ signing_key:
|
|
291
309
|
specification_version: 4
|
292
310
|
summary: Robot Sweatshop
|
293
311
|
test_files:
|
312
|
+
- kintama/data/git_job.yaml
|
294
313
|
- kintama/data/payload_data.yaml
|
295
314
|
- kintama/data/test_job.yaml
|
296
315
|
- kintama/shared/helpers.rb
|