robot_sweatshop 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 94213cf7a5098291271e0d6b1b053e822da20269
-  data.tar.gz: c6e7f4877c894243ccb386f339bbef0ed4fc822d
+  metadata.gz: cbc73c13d57e32651f63077a89b03317769850f3
+  data.tar.gz: 834bb91075c655c1edc0ac7f4cc445c2048094db
 SHA512:
-  metadata.gz: 42394c288d96179fd7dfc979e23eb44a5f9a75ea6bbdc43ea2c92c4180357df6c9b84ab0c09a0cf1632cbb59bcd1d3ac1092aea61450ce7e69867090f1fa33b9
-  data.tar.gz: aa468e0ca4eafb867b68ee9c68c3ee423156394921c85a95429b20ad5499ba42a345e645169a563950ea17c3bb994e67b6124b2acf2847f15550272771240d72
+  metadata.gz: 16784f92f5abe86c20dd6aebe73d79d3d9d18296576ba50ed0018467b205038fb67517a18931a539b4309839f311427b079c5b3b0e4708f7120d7e0baf570365
+  data.tar.gz: bc6eca3c50fddb92091e7693b7f4aba3232ab96b857a3b495d7a20e557bff3f695cda3149842b718d5c5905011d04bdb4d6d9decaf3cca8a7dcd3c062b8a2e1e

data/README.md

@@ -8,11 +8,11 @@ Robot Sweatshop is a single-purpose CI server that runs collections of arbitrary
 
 # Quick start
 
-- install [ZMQ as described in the EZMQ gem](https://github.com/colstrom/ezmq)
+- install [ZMQ as described in the EZMQ gem](https://github.com/colstrom/ezmq#operating-system-notes)
 - `gem install robot_sweatshop`
 - `sweatshop start` ([you may need sudo on OSX](https://github.com/JScott/robot_sweatshop/wiki))
 - `sweatshop job example --auto`
-- POST a Github payload to `yourserver.com:8080/github/payload-for/example`
+- `curl --data '{"your": "json"}' http://localhost:8080/payload-for/example`
 - `cat .robot_sweatshop/log/job-worker.log`
 
 # Usage
@@ -29,6 +29,7 @@ By default, Robot Sweatshop looks in your current working directory to configure
 
 - Github (application/json format only)
 - Bitbucket
+- JSON
 
 # Security
 
@@ -36,12 +37,17 @@ You probably don't want to run Robot Sweatshop as a sudo user. Create a testing
 
 # Roadmap
 
-- CLI job running
-- custom/empty payloads
-- Common scrips such as git repo syncing and creating a job run ID
-- Support for multiple workers
-- Better logging for the processes
-- Use [eye-http](https://github.com/kostya/eye-http) for the '/' route?
+1.0
+
 - Improved architecture:
 
 ![Improved architecture diagram](http://40.media.tumblr.com/8a5b6ca59c0d93c4ce6fc6b733932a5f/tumblr_nko478zp9N1qh941oo1_1280.jpg)
+
+- Support for multiple workers
+
+Beyond
+
+- Better logging for the processes
+- CLI configuration via chomp and/or flags
+- Common scrips such as git repo syncing and creating a job run ID
+- Use [eye-http](https://github.com/kostya/eye-http) for the '/' route?

data/bin/sweatshop

@@ -10,7 +10,7 @@ require_relative '../lib/sweatshop/config'
 require_relative '../lib/sweatshop/create-config-directories'
 
 program :name, 'Robot Sweatshop'
-program :version, '0.2.1'
+program :version, '0.3.0'
 program :description, 'A lightweight, unopinionated CI server'
 program :help, 'Author', 'Justin Scott <jvscott@gmail.com>'
 

data/bin/sweatshop-input-http

@@ -1,13 +1,13 @@
 #!/usr/bin/env ruby
+require 'bundler/setup'
 require 'sinatra'
 require 'ezmq'
 require 'json'
-require_relative '../lib/sweatshop/config'
+require 'sweatshop/config'
 
 configure do
   set :port, configatron.http_port
   set :bind, configatron.http_bind
-  set :output_queue, 'raw-payload'
   set :run, true
 end
 
@@ -15,14 +15,15 @@ get '/' do
   'Everything\'s on schedule!'
 end
 
-post '/:format/payload-for/:job_name' do
-  puts "Received #{params['format']} payload for #{params['job_name']}"
+post '/payload-for/:job_name' do
+  puts "Received payload for #{params['job_name']}"
   request.body.rewind
+  payload = request.body.read
   hash = {
-    payload: request.body.read,
-    format: params['format'],
+    payload: payload,
+    user_agent: request.env['HTTP_USER_AGENT'],
     job_name: params['job_name']
   }
   client = EZMQ::Client.new port: 5556
-  client.request "#{settings.output_queue} #{JSON.generate hash}"
+  client.request "payload #{JSON.generate hash}"
 end

data/bin/sweatshop-job-assembler

@@ -1,45 +1,82 @@
 #!/usr/bin/env ruby
+require 'bundler/setup'
 require 'yaml'
-require_relative '../lib/sweatshop/queue-helper'
-require_relative '../lib/sweatshop/config'
-
-def load_config(for_job:)
-  job_directory = File.expand_path configatron.job_directory
-  file_path = "#{job_directory}/#{for_job}.yaml"
-  puts file_path
-  if File.file?(file_path)
-    YAML.load_file file_path
-  else
-    puts "No config found for job '#{for_job}'"
-    nil
-  end
-end
-
-def serialize(value)
-  value.is_a?(Hash) ? JSON.generate(value) : value.to_s
-end
-
-def assemble_job(data)
-  job_config = load_config for_job: data['job_name']
-  return nil if job_config.nil?
-  if job_config['branch_whitelist'].include? data['payload']['branch']
-    context = job_config['environment'].merge(data['payload'])
-    context.each { |key, value| context[key] = serialize value }
-    {
-      commands: job_config['commands'],
-      context: context,
-      job_name: data['job_name']
-    }
-  else
-    puts "Branch '#{data['payload']['branch']}' is not whitelisted"
-    nil
-  end
-end
-
-QueueHelper.wait_for('parsed-payload') do |data|
-  puts "Assembling: #{data}"
-  assembled_job = assemble_job data
+require 'json'
+require 'contracts'
+require 'sweatshop/queue-helper'
+require 'sweatshop/config'
+
+$stdout.sync = true
+include Contracts
+
+Contract None => Hash
+def empty_config
+  puts "Job configuration not found"
+  {}
+end
+
+Contract None => String
+def job_directory
+  File.expand_path configatron.job_directory
+end
+
+Contract Or[String, nil], Hash => Bool
+def whitelisted?(branch, in_config:)
+  return true if in_config['branch_whitelist'].nil?
+  in_config['branch_whitelist'].include? branch
+end
+
+Contract Hash => Hash
+def sanitize(data)
+  data.each_pair { |key, value| data[key] = value.to_s }
+end
+
+Contract Hash, Hash => Hash
+def job_context(job_environment, context_from_payload)
+  sanitize job_environment.merge(context_from_payload)
+end
+
+Contract String => Or[Hash, nil]
+def load_if_exists(config)
+  puts "Reading job configuration from #{config}"
+  YAML.load_file config if File.exists? config
+end
+
+Contract String => Hash
+def load_config_for(job_name)
+  load_if_exists("#{job_directory}/#{job_name}.yaml") || empty_config
+end
+
+Contract Hash => [Hash, String]
+def parse_payload(request)
+  client = EZMQ::Client.new port: configatron.payload_parser_port
+  response = JSON.load client.request(JSON.dump request)
+  puts response['error'] unless response['error'].empty?
+  [response['payload'], response['error']]
+end
+
+Contract Hash, Hash => Bool
+def can_work_with?(job_config, payload)
+  return false if job_config.empty?
+  return false unless whitelisted? payload['branch'], in_config: job_config
+  true
+end
+
+Contract Hash => Or[Hash, false]
+def assemble_job(request)
+  job_config = load_config_for request['job_name']
+  payload, error = parse_payload request
+  return false unless error.empty? && can_work_with?(job_config, payload)
+  {
+    commands: job_config['commands'],
+    context: job_context(job_config['environment'], payload),
+    job_name: request['job_name']
+  }
+end
+
+puts 'Started'
+QueueHelper.wait_for('payload') do |request|
+  puts "Assembling: #{request}"
+  assembled_job = assemble_job request
   QueueHelper.enqueue assembled_job, to: 'jobs' if assembled_job
-  $stdout.flush
-  $stderr.flush
 end

data/bin/sweatshop-job-worker

@@ -21,7 +21,7 @@ end
 def execute(context = {}, command)
   puts "Executing '#{command}'..."
   # TODO: path.split(' ') to bypass the shell when we're not using env vars
-  
+
   # Run the command with the context in environment,
   # printing the output as it's generated
   IO.popen(context, command) do |io_stream|

data/bin/sweatshop-payload-parser

@@ -1,23 +1,72 @@
 #!/usr/bin/env ruby
-require_relative '../lib/sweatshop/queue-helper'
-
-def parse(payload = '', of_format:)
-  of_format = of_format.downcase
-  lib_file = "#{__dir__}/../lib/sweatshop/payload/#{of_format}.rb"
-  if of_format != 'payload' && File.file?(lib_file)
-    require_relative lib_file
-    Object.const_get("#{of_format.capitalize}Payload").new payload
-  else
-    puts "Dropping bad format: #{of_format}"
-    nil
+require 'bundler/setup'
+require 'json'
+require 'ezmq'
+require 'contracts'
+require 'sweatshop/config'
+require 'sweatshop/payload'
+
+$stdout.sync = true
+include Contracts
+
+Contract String => Bool
+def json?(string)
+  begin
+    JSON.load string
+    true
+  rescue JSON::ParserError => e
+    false
   end
 end
 
-QueueHelper.wait_for('raw-payload') do |data|
-  puts "Parsing: #{data}"
-  payload = parse data['payload'], of_format: data['format']
-  if payload
-    hash = { payload: payload.to_hash, job_name: data['job_name'] }
-    QueueHelper.enqueue hash, to: 'parsed-payload'
-  end
+Contract String => Bool
+def from_github?(user_agent)
+  user_agent.start_with? 'Github-Hookshot'
+end
+
+Contract String => Bool
+def from_bitbucket?(user_agent)
+  user_agent.start_with? 'Bitbucket.org'
+end
+
+Contract Hash => String
+def detect_format_of(request)
+  return 'empty' if request['payload'].empty?
+  return 'Github' if from_github? request['user_agent']
+  return 'Bitbucket' if from_bitbucket? request['user_agent']
+  return 'JSON' if json? request['payload']
+  'unsupported'
+end
+
+Contract String, String => Or[Hash, nil]
+def payload_hash_from(payload, format)
+  lib_file = "#{__dir__}/../lib/sweatshop/payload/#{format}.rb"
+  return nil unless File.file?(lib_file)
+  require_relative lib_file
+  Object.const_get("#{format}Payload").new(payload).to_hash
+end
+
+Contract Hash => Hash
+def parse(request)
+  format = detect_format_of request
+  payload = payload_hash_from request['payload'], format
+  return {payload: {}, error: "Can't parse #{format} payload"} if payload.nil?
+  {payload: payload, error: ''}
+end
+
+Contract Any => Hash
+def validate(request)
+  return {payload: {}, error: "Invalid JSON request"} unless json? request
+  request = JSON.load request
+  return {payload: {}, error: "Empty request"} if request.nil?
+  request
+end
+
+puts 'Started'
+server = EZMQ::Server.new port: configatron.payload_parser_port#, encode: -> m { JSON.dump m }
+server.listen do |request|
+  puts "Parsing: #{request}"
+  request = validate request
+  request = parse request unless request[:error]
+  JSON.dump request
 end

data/config.defaults.yaml

@@ -8,3 +8,6 @@ moneta_directory: .robot_sweatshop/db
 # group: nogroup
 http_port: 8080
 http_bind: 0.0.0.0
+# queue_handler_port: 5556
+# queue_broadcaster_port: 5557
+payload_parser_port: 5558

data/kintama/data/git_job.yaml

@@ -0,0 +1,9 @@
+---
+branch_whitelist:
+- develop
+
+commands:
+- echo $custom_env > test.txt
+
+environment:
+  custom_env: success

data/kintama/data/payload_data.yaml

@@ -1,7 +1,8 @@
 ---
 # http://www.url-encode-decode.com/
-malformed: |
-  payload=%7B%22nothing%22%3A%22here%22%7D
+empty: ''
+nonjson: not json
+json: '{"test1": "value", "test2": "value"}'
 bitbucket: |
   payload=%7B%22repository%22%3A+%7B%22website%22%3A+%22%22%2C+%22fork%22%3A+false%2C+%22name%22%3A+%22Project+X%22%2C+%22scm%22%3A+%22git%22%2C+%22owner%22%3A+%22marcus%22%2C+%22absolute_url%22%3A+%22%2Fmarcus%2Fproject-x%2F%22%2C+%22slug%22%3A+%22project-x%22%2C+%22is_private%22%3A+true%7D%2C+%22truncated%22%3A+false%2C+%22commits%22%3A+%5B%7B%22node%22%3A+%22620ade18607a%22%2C+%22files%22%3A+%5B%7B%22type%22%3A+%22modified%22%2C+%22file%22%3A+%22README%22%7D%5D%2C+%22raw_author%22%3A+%22Marcus+Bertrand+%3Cmarcus%40somedomain.com%3E%22%2C+%22utctimestamp%22%3A+%222014-08-08+00%3A04%3A07%2B00%3A00%22%2C+%22author%22%3A+%22marcus%22%2C+%22timestamp%22%3A+%222014-08-08+02%3A04%3A07%22%2C+%22raw_node%22%3A+%22620ade18607ac42d872b568bb92acaa9a28620e9%22%2C+%22parents%22%3A+%5B%22702c70160afc%22%5D%2C+%22branch%22%3A+%22develop%22%2C+%22message%22%3A+%22Added+some+more+things+to+somefile.py%5Cn%22%2C+%22revision%22%3A+null%2C+%22size%22%3A+-1%7D%5D%2C+%22canon_url%22%3A+%22https%3A%2F%2Fbitbucket.org%22%2C+%22user%22%3A+%22marcus%22%7D
 github: |

data/kintama/data/test_job.yaml

@@ -1,10 +1,8 @@
 ---
-branch_whitelist:
-- develop
-
 commands:
 - echo $custom_env > test.txt
 
 environment:
   custom_env: success
-  converts_to_string: { string: only }
+  some_hash: { string: only }
+  some_array: ['string', 'only']

data/kintama/end-to-end_spec.rb

@@ -11,18 +11,15 @@ describe 'Robot Sweatshop' do
 
   setup do
     @client = EZMQ::Client.new port: 5556
-    @job_name = 'test_job'
     @test_file = reset_test_file
     clear_all_queues
   end
 
   context "POST git data to the HTTP Input" do
     setup do
-      url = input_http_url for_job: @job_name
-puts "posting..."
-      HTTP.post url, body: load_payload('bitbucket')
-puts "post successful"
-      sleep $for_everything
+      url = input_http_url for_job: 'test_job'
+      HTTP.post url, body: example_raw_payload(of_format: 'JSON')
+      sleep $for_io_calls
     end
 
     should 'run jobs with the context as environment variables' do

data/kintama/input_http_spec.rb

@@ -13,32 +13,36 @@ given 'the HTTP Input' do
   setup do
     @subscriber = EZMQ::Subscriber.new port: 5557, topic: 'busy-queues'
     @client = EZMQ::Client.new port: 5556
-    @job_name = 'test_job'
-    @raw_payload_queue = 'raw-payload'
+    @payload_queue = 'payload'
     clear_all_queues
   end
-  
-  %w(Bitbucket).each do |git_host|
-    context "POST data from #{git_host}" do
+
+  %w(Bitbucket Github JSON).each do |format|
+    context "POSTing #{format} data" do
       setup do
-        url = input_http_url for_job: @job_name, in_format: git_host
-        HTTP.post url, body: load_payload(git_host)
+        url = input_http_url for_job: 'test_job'
+        HTTP.post url, body: example_raw_payload(of_format: format)
       end
 
-      should 'enqueue to \'raw-payload\'' do
+      should 'enqueue to \'payload\'' do
         Timeout.timeout($for_a_moment) do
           @subscriber.listen do |message, topic|
-            break if message == @raw_payload_queue
+            break if message == @payload_queue
           end
         end
       end
 
-      should 'enqueue payload details' do
-        response = @client.request "mirror-#{@raw_payload_queue}"
+      should 'enqueue payload details and format' do
+        response = @client.request "mirror-#{@payload_queue}"
         data = JSON.parse response
-        %w(payload format job_name).each do |type|
-          assert_kind_of String, data[type]
-        end
+        assert_kind_of String, data['payload']
+        assert_kind_of String, data['user_agent']
+      end
+
+      should 'enqueue job name' do
+        response = @client.request "mirror-#{@payload_queue}"
+        data = JSON.parse response
+        assert_kind_of String, data['job_name']
       end
     end
   end

data/kintama/job_assembler_spec.rb

@@ -6,73 +6,73 @@ require_relative 'shared/helpers'
 
 describe 'the Job Assembler' do
   include QueueHelper
-  include PayloadHelper
+  include InHelper
   include JobHelper
 
   setup do
     @client = EZMQ::Client.new port: 5556
-    @parsed_payloads_queue = 'parsed-payload'
+    @payloads_queue = 'payload'
     @jobs_queue = 'jobs'
     clear_all_queues
   end
 
-  given 'valid parsed payload data in \'parsed-payload\'' do
-    setup do
-      payload = example_parsed_payload(for_branch: 'develop')
-      @client.request "#{@parsed_payloads_queue} #{payload}"
-      sleep $for_a_while
-    end
+  %w(Git JSON).each do |request|
+    given "#{request} requests in \'payload\'" do
+      setup do
+        payload = example_job_request of_type: request
+        @client.request "#{@payloads_queue} #{payload}"
+        sleep $for_a_moment
+      end
 
-    should 'remove it from \'parsed-payload\'' do
-      response = @client.request @parsed_payloads_queue
-      assert_equal '', response
-    end
+      should 'remove the request from \'payload\'' do
+        response = @client.request @payloads_queue
+        assert_equal '', response
+      end
 
-    should 'enqueue commands, context, and job name to \'jobs\'' do
-      response = @client.request "mirror-#{@jobs_queue}"
-      response = JSON.parse response
-      assert_kind_of Hash, response['context']
-      assert_kind_of Array, response['commands']
-      assert_kind_of String, response['job_name']
-    end
+      should 'enqueue commands, context, and job name to \'jobs\'' do
+        response = @client.request "mirror-#{@jobs_queue}"
+        response = JSON.load response
+        assert_kind_of Hash, response['context']
+        assert_kind_of Array, response['commands']
+        assert_kind_of String, response['job_name']
+      end
 
-    should 'convert objects into JSON strings' do
-      response = @client.request "mirror-#{@jobs_queue}"
-      response = JSON.parse response
-      assert_kind_of Hash, JSON.parse(response['context']['converts_to_string'])
-    end
+      should 'store everything in the context as strings' do
+        response = @client.request "mirror-#{@jobs_queue}"
+        response = JSON.load response
+        response['context'].each { |_key, value| assert_kind_of String, value }
+      end
 
-    should 'only enqueue string objects to context' do
-      response = @client.request "mirror-#{@jobs_queue}"
-      response = JSON.parse response
-      response['context'].each do |_key, value|
-        assert_kind_of String, value
+      should 'build the context with a parsed payload' do
+        response = @client.request "mirror-#{@jobs_queue}"
+        response = JSON.load response
+        assert_kind_of Hash, response['context']
+        if request == 'Git'
+          assert_equal 'develop', response['context']['branch']
+        else
+          assert_equal 'value', response['context']['test1']
+        end
       end
     end
   end
 
-  given 'invalid job data in \'parsed-payload\'' do
-    setup do
-      invalid_data = {
-        ignored_branch: example_parsed_payload(for_branch: 'not_on_whitelist'),
-        bad_payload:    example_parsed_payload(with_payload: 'not hash'),
-        bad_job:        example_parsed_payload(for_job: 'asdf'),
-        not_json:       'not_json'
-      }
-      invalid_data.each do |_type, datum|
-        @client.request "#{@parsed_payloads_queue} #{datum}"
+  %w(IgnoredBranch UnknownJob NonJSON).each do |request|
+    given "#{request} requests in \'payload\'" do
+      setup do
+        payload = example_job_request of_type: request
+        @client.request "#{@payloads_queue} #{payload}"
+        sleep $for_a_moment
       end
-      sleep $for_a_while
-    end
 
-    should 'remove all of it from \'parsed-payload\'' do
-      response = @client.request @parsed_payloads_queue
-      assert_equal '', response
-    end
+      should 'remove the request from \'payload\'' do
+        response = @client.request @payloads_queue
+        assert_equal '', response
+      end
 
-    should 'not queue anything to \'jobs\'' do
-      response = @client.request @jobs_queue
-      assert_equal '', response
+      should 'not queue anything to \'jobs\'' do
+        response = @client.request @jobs_queue
+        assert_equal '', response
+      end
     end
   end
 end

data/kintama/payload_parser_spec.rb

@@ -1,70 +1,61 @@
 require 'kintama'
 require 'ezmq'
 require 'json'
+require 'timeout'
 require_relative 'shared/process_spawning'
 require_relative 'shared/helpers'
 
 describe 'the Payload Parser' do
   include QueueHelper
   include InHelper
-  include PayloadHelper
 
   setup do
-    @client = EZMQ::Client.new port: 5556
-    @raw_queue = 'raw-payload'
-    @parsed_queue = 'parsed-payload'
-    clear_all_queues
+    @client = EZMQ::Client.new port: configatron.payload_parser_port
   end
 
-  %w(Bitbucket Github).each do |format|
-    given "valid #{format} data in 'raw-payload'" do
+  %w(Bitbucket Github JSON).each do |format|
+    given "valid #{format} payloads" do
       setup do
-        payload = example_raw_payload(with_format: format)
-        @client.request "#{@raw_queue} #{payload}"
-        sleep $for_a_while
-      end
-
-      should 'remove it from \'raw-payload\'' do
-        response = @client.request @raw_queue
-        assert_equal '', response
+        payload = example_payload_request of_format: format
+        @response = Timeout.timeout($for_a_while) do
+          @client.request JSON.dump(payload)
+        end
       end
 
-      should 'enqueue parsed payload data and job name to \'parsed-payload\'' do
-        response = @client.request "mirror-#{@parsed_queue}"
-        response = JSON.parse response
+      should 'return a parsed payload object' do
+        @response = JSON.parse @response
+        assert_equal true, @response['error'].empty?
+        assert_kind_of Hash, @response['payload']
 
-        assert_kind_of Hash, response['payload']
-        Payload.hash_keys.each do |key|
-          assert_not_nil response['payload'][key]
-          assert_not_equal key, response['payload'][key] # important for how Ruby interprets "string"['key']
+        keys = case format
+        when 'JSON'
+          %w(test1 test2)
+        else
+          Payload.hash_keys
+        end
+        keys.each do |key|
+          payload = @response['payload']
+          assert_not_nil payload[key]
+          assert_not_equal key, payload[key] # catches "string"[key]
         end
-        assert_kind_of String, response['job_name']
       end
     end
   end
 
-  given 'invalid payload data in \'raw-payload\'' do
-    setup do
-      invalid_data = {
-        malformed_payload:  example_raw_payload(with_format: 'malformed'),
-        unsupported_format: example_raw_payload(with_format: 'asdf'),
-        not_json:           'not_json'
-      }
-      invalid_data.each do |_type, datum|
-        @client.request "#{@raw_queue} #{datum}"
+  %w(Empty NonJSON).each do |format|
+    given "#{format} payloads" do
+      setup do
+        payload = example_raw_payload of_format: format
+        @response = Timeout.timeout($for_a_while) do
+          @client.request "#{payload}"
+        end
       end
-      sleep $for_a_while
-      # TODO: should not crash the payload parser
-    end
 
-    should 'remove all of it from \'raw-payload\'' do
-      response = @client.request @raw_queue
-      assert_equal '', response
-    end
-
-    should 'not queue anything to \'parsed-payload\'' do
-      response = @client.request @parsed_queue
-      assert_equal '', response
+      should 'return an error object' do
+        @response = JSON.load(@response)
+        assert_kind_of String, @response['error']
+        assert_equal true, @response['payload'].empty?
+      end
     end
   end
 end

data/kintama/shared/helpers.rb

@@ -1,8 +1,9 @@
+require 'bundler/setup'
 require 'yaml'
 require 'json'
-require_relative '../../lib/sweatshop/moneta-queue'
-require_relative '../../lib/sweatshop/payload/payload'
-require_relative '../../lib/sweatshop/config'
+require 'sweatshop/moneta-queue'
+require 'sweatshop/payload'
+require 'sweatshop/config'
 
 module QueueHelper
   def clear_all_queues
@@ -18,26 +19,50 @@ module QueueHelper
 end
 
 module InHelper
-  def load_payload(of_format)
+  def example_raw_payload(of_format:)
     payload_strings = YAML.load_file "#{__dir__}/../data/payload_data.yaml"
     payload_strings[of_format.downcase]
   end
-  def example_raw_payload(with_format:)
-    payload = load_payload with_format
-    JSON.generate payload: payload,
-                  format: with_format,
-                  job_name: 'test_job'
+  def input_http_url(for_job: 'test_job')
+    "http://localhost:#{configatron.http_port}/payload-for/#{for_job}"
   end
-  def input_http_url(for_job: 'test_job', in_format: 'bitbucket')
-    "http://localhost:#{configatron.http_port}/#{in_format}/payload-for/#{for_job}"
+  def user_agent_for(format)
+    case format
+    when 'Bitbucket'
+      'Bitbucket.org'
+    when 'Github'
+      'Github-Hookshot'
+    else
+      'SomeRandomUserAgent'
+    end
   end
-end
-
-module PayloadHelper
-  def example_parsed_payload(with_payload: nil, for_branch: 'develop', for_job: 'test_job')
-    payload = with_payload || { branch: for_branch }
-    JSON.generate payload: payload,
-                  job_name: for_job
+  def example_payload_request(of_format:)
+    {
+      payload: example_raw_payload(of_format: of_format),
+      user_agent: user_agent_for(of_format)
+    }
+  end
+  def job_request_data(type)
+    case type
+    when 'Git'
+      ['Bitbucket', 'git_job'] # develop branch
+    when 'JSON'
+      ['JSON', 'test_job']
+    when 'IgnoredBranch'
+      ['Github', 'git_job'] # master branch
+    when 'UnknownJob'
+      ['Bitbucket', 'unknown_job']
+    when 'NonJSON'
+      ['JSON', 'git_job', 'not json']
+    else
+      ['I', 'AM', 'ERROR']
+    end
+  end
+  def example_job_request(of_type:)
+    format, job_name, payload = job_request_data of_type
+    JSON.generate payload: payload || example_raw_payload(of_format: format),
+                  job_name: job_name,
+                  user_agent: user_agent_for(format)
   end
 end
 

data/kintama/shared/process_spawning.rb

@@ -1,15 +1,15 @@
 $for_a_moment = 0.1
 $for_a_while = 0.5
 $for_io_calls = 1
-$for_everything = 3
+$for_everything = 2
 
 Kintama.on_start do
   puts `#{__dir__}/../../bin/sweatshop start --testing`
   FileUtils.cp "#{__dir__}/../data/test_job.yaml", File.expand_path(configatron.job_directory)
+  FileUtils.cp "#{__dir__}/../data/git_job.yaml", File.expand_path(configatron.job_directory)
   sleep $for_everything
 end
 
 Kintama.on_finish do
   puts `#{__dir__}/../../bin/sweatshop stop`
-  # FileUtils.rm "#{configatron.job_directory}/test_job.yaml"
 end

data/lib/README.md

@@ -1,8 +1,7 @@
 Job lifecycle:
 
 ```
-in-* -> raw-payloads { payload:, format:, job_name: } ->
-payload-parser -> parsed-payloads { payload:, job_name: } ->
+in-* -> payloads { payload:, user_agent:, job_name: } ->
 job-assembler -> jobs { context:, commands:, job_name: } ->
 job-worker
 ```
@@ -10,3 +9,13 @@ job-worker
 All passing done via the moneta core in queue/*
 
 Also queue/watcher for debugging
+
+---
+
+The Job Assembler also uses the Payload Parser service:
+
+```
+{ payload:, user_agent: } -> payload-parser -> { payload:, error: }
+```
+
+Error is an empty string when it's successful. Otherwise it details what went wrong.

data/lib/sweatshop/cli/job.rb

@@ -1,7 +1,7 @@
 require_relative '../config'
 
 def default_job
-  "---\nbranch_whitelist:\n- master\n\ncommands:\n- echo 'Hello $WORLD!'\n\nenvironment:\n  WORLD: Earth\n"
+  "---\n# branch_whitelist:\n# - master\n\ncommands:\n- echo \"Hello $WORLD!\"\n\nenvironment:\n  WORLD: Earth\n"
 end
 
 def get_job_path(for_job: nil)

data/lib/sweatshop/cli.rb

@@ -0,0 +1,4 @@
+require_relative 'cli/common'
+require_relative 'cli/config'
+require_relative 'cli/job'
+require_relative 'cli/start'

data/lib/sweatshop/moneta-queue.rb

@@ -16,7 +16,7 @@ class MonetaQueue
   end
 
   def self.watched_queues
-    %w(raw-payload parsed-payload jobs testing)
+    %w(payload parsed-payload jobs testing)
   end
 
   def enqueue(item)

data/lib/sweatshop/payload/json.rb

@@ -0,0 +1,13 @@
+require 'json'
+require_relative 'payload'
+
+# A parser for arbitrary data
+class JSONPayload < Payload
+  def initialize(payload)
+    @payload = JSON.parse payload || {}
+  end
+
+  def to_hash
+    @payload
+  end
+end

data/lib/sweatshop/payload.rb

@@ -0,0 +1,4 @@
+require_relative 'payload/payload'
+require_relative 'payload/bitbucket'
+require_relative 'payload/github'
+require_relative 'payload/json'

data/lib/sweatshop.rb

@@ -1,16 +1,5 @@
 require_relative 'sweatshop/moneta-queue'
 require_relative 'sweatshop/queue-helper'
 require_relative 'sweatshop/config'
-
-require_relative 'sweatshop/payload/payload'
-require_relative 'sweatshop/payload/bitbucket'
-require_relative 'sweatshop/payload/github'
-
-require_relative 'sweatshop/cli/common'
-require_relative 'sweatshop/cli/config'
-require_relative 'sweatshop/cli/inspect'
-require_relative 'sweatshop/cli/job'
-require_relative 'sweatshop/cli/setup'
-require_relative 'sweatshop/cli/start'
-
-
+require_relative 'sweatshop/payload'
+require_relative 'sweatshop/cli'

data/robot_sweatshop.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |gem|
   gem.name        = 'robot_sweatshop'
-  gem.version     = '0.2.1'
+  gem.version     = '0.3.0'
   gem.licenses    = 'MIT'
   gem.authors     = ['Justin Scott']
   gem.email       = 'jvscott@gmail.com'
@@ -20,9 +20,10 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency 'faker'
   gem.add_runtime_dependency 'commander'
   gem.add_runtime_dependency 'eye'
-  gem.add_runtime_dependency 'colorize'
+  gem.add_runtime_dependency 'colorize' # TODO: replace with rainbow
   gem.add_runtime_dependency 'configatron'
   gem.add_runtime_dependency 'moneta'
+  gem.add_runtime_dependency 'contracts'
 
   gem.add_development_dependency 'rake'
   gem.add_development_dependency 'kintama'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: robot_sweatshop
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Justin Scott
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-24 00:00:00.000000000 Z
+date: 2015-04-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sinatra
@@ -122,6 +122,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: contracts
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -236,6 +250,7 @@ files:
 - bin/sweatshop-queue-watcher
 - config.defaults.yaml
 - kintama/README.md
+- kintama/data/git_job.yaml
 - kintama/data/payload_data.yaml
 - kintama/data/test_job.yaml
 - kintama/end-to-end_spec.rb
@@ -251,6 +266,7 @@ files:
 - kintama/shared/process_spawning.rb
 - lib/README.md
 - lib/sweatshop.rb
+- lib/sweatshop/cli.rb
 - lib/sweatshop/cli/common.rb
 - lib/sweatshop/cli/config.rb
 - lib/sweatshop/cli/job.rb
@@ -258,8 +274,10 @@ files:
 - lib/sweatshop/config.rb
 - lib/sweatshop/create-config-directories.rb
 - lib/sweatshop/moneta-queue.rb
+- lib/sweatshop/payload.rb
 - lib/sweatshop/payload/bitbucket.rb
 - lib/sweatshop/payload/github.rb
+- lib/sweatshop/payload/json.rb
 - lib/sweatshop/payload/payload.rb
 - lib/sweatshop/queue-helper.rb
 - robot_sweatshop.eye
@@ -291,6 +309,7 @@ signing_key:
 specification_version: 4
 summary: Robot Sweatshop
 test_files:
+- kintama/data/git_job.yaml
 - kintama/data/payload_data.yaml
 - kintama/data/test_job.yaml
 - kintama/shared/helpers.rb