robot_sweatshop 0.3.3 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 04bbac597bb3b25f0ed7fdf9e44f5232fa6a5f46
-  data.tar.gz: 1b8e0bef439759c90312b49580440624ca998f26
+  metadata.gz: 0549401a67fbd6e2dc3b5b22017d42700958f9c9
+  data.tar.gz: 5f741330b8b0999cb078cf0508e9440f595e02e6
 SHA512:
-  metadata.gz: daae1e558e2276e84f1bd10692b673fee401a6fb09bfa25e5f944f058f26d9831a3fabaa86d9177df2cedc66f36b5d3e3ea392b81730297f946553a4c229b8ad
-  data.tar.gz: cdae51f479d280542d028628bf6985c1c513acb597ca1ec567436e5d9c80106a7ecf2e5f222d9773c3aa78c9505f770b5302a82c2d7afef2fb5b1d6af365cbfd
+  metadata.gz: e1c7523aea2b3006952303d6e3dfb9f13886cd304479ecec7c5841cfaba5c1455aed8a6ff13815b18ae7ec6af7904dee785333d30ad1a4a4af52badebef6ca5a
+  data.tar.gz: 83b86d2de23a74dc9a4623fb23417e37d822271b42e96b05cb3c7cc6f62eb764924d11beb8b1b0417199ac5cdefaa2b09918f670f930b84d24c20558a09993ac

data/.gitignore

@@ -1,3 +1,4 @@
+.test.*
 *.gem
 jobs/*.yaml
 .robot_sweatshop

data/README.md

@@ -10,20 +10,20 @@ Robot Sweatshop is a single-purpose CI server that runs collections of arbitrary
 
 - install [ZMQ as described in the EZMQ gem](https://github.com/colstrom/ezmq#operating-system-notes)
 - `gem install robot_sweatshop`
-- `sweatshop start` ([you may need sudo on OSX](https://github.com/JScott/robot_sweatshop/wiki))
+- `sweatshop start`
 - `sweatshop job example --auto`
-- `curl -X POST http://localhost:8080/payload-for/example`
+- `curl -d '' localhost:8080/run/example`
 - `cat .robot_sweatshop/log/job-worker.log`
 
 # Usage
 
-Drop the `--auto` flag to create the job interactively. You can specify which branches will trigger the job, which commands will be run, and any environment variables you might need.
+Drop the `--auto` flag to create the job interactively. You can specify which branches will trigger the job, which commands will be run, and any environment variables you might need. See [the wiki](https://github.com/JScott/robot_sweatshop/wiki/Job-configuration) for more details.
 
 Robot Sweatshop uses [Eye](https://github.com/kostya/eye) to handle its processes so you can use its commandline tool to monitor their status.
 
 # Configuration
 
-By default, Robot Sweatshop looks in your current working path to configure and run. You can supply a custom configuration with `sweatshop config [local|user|system]`. Read [the wiki](https://github.com/JScott/robot_sweatshop/wiki) for more information.
+By default, Robot Sweatshop looks in your current working path to configure and run. You can supply a custom configuration with `sweatshop config [local|user|system]`. See [the wiki](https://github.com/JScott/robot_sweatshop/wiki) for more information.
 
 # Supported payload formats
 
@@ -34,21 +34,21 @@ By default, Robot Sweatshop looks in your current working path to configure and
 
 # Security
 
-You probably don't want to run Robot Sweatshop as a sudo user. Create a testing user and group and point to them in the configuration file to run Robot Sweatshop processes with those permissions.
+You probably don't want to run Robot Sweatshop as a sudo user. Create a testing user and group and run `sweatshop start` as them.
 
 # Roadmap
 
 1.0
 
-- Improved architecture:
-
-![Improved architecture diagram](http://40.media.tumblr.com/8a5b6ca59c0d93c4ce6fc6b733932a5f/tumblr_nko478zp9N1qh941oo1_1280.jpg)
-
-- Support for multiple workers
+- Docs on architecture
+- Easier way to run multiple workers
+- Mascot
 
 Beyond
 
+- Jenkins-to-Sweatshop job converting script
+- Take a look at beefcake for data versioning/serialization
 - Better logging for the processes
 - CLI configuration via chomp and/or flags
-- Common scrips such as git repo syncing and creating a job run ID
+- Common scrips such as git repo syncing and creating a job run ID (see: [sweatshop-tears](https://github.com/JScott/sweatshop-tears))
 - Use [eye-http](https://github.com/kostya/eye-http) for the '/' route?

data/Rakefile

@@ -7,5 +7,5 @@ task :test do
     add_filter "/vendor/"
     add_filter "/test/"
   end
-  require_relative 'test/run_all'
+  require_relative 'test/all'
 end

data/bin/README.md

@@ -0,0 +1,23 @@
+# Services
+
+sweatshop-conveyor
+```
+{ method:, data: }
+
+id = enqueue(item)
+id = dequeue
+item = lookup(id)
+something = finish(id)
+```
+
+sweatshop-payload-parser
+```
+req: { payload:, user_agent: }
+rep: { data:, error: }
+```
+
+sweatshop-job-dictionary
+```
+req: { job_name: }
+rep: { data:, error: }
+```

data/bin/sweatshop

@@ -8,7 +8,7 @@ require 'robot_sweatshop/config'
 require 'robot_sweatshop/create-config-directories'
 
 program :name, 'Robot Sweatshop'
-program :version, '0.3.3'
+program :version, '0.4.0'
 program :description, 'A lightweight, unopinionated CI server'
 program :help, 'Author', 'Justin Scott <jvscott@gmail.com>'
 

data/bin/sweatshop-assembler

@@ -0,0 +1,101 @@
+#!/usr/bin/env ruby
+require 'bundler/setup'
+require 'yaml'
+require 'contracts'
+require 'exponential_backoff'
+require 'ezmq'
+require 'robot_sweatshop/config'
+require 'robot_sweatshop/connections'
+$stdout.sync = true
+include Contracts
+using ExtendedEZMQ
+
+Contract Hash => Hash
+def sanitize(data)
+  data = data.map { |key, value| {key => value.to_s} }
+  data.reduce(:merge)
+end
+
+Contract Hash, Hash => Hash
+def job_context(job_environment, context_from_payload)
+  sanitize job_environment.merge(context_from_payload)
+end
+
+Contract Hash, Hash, Hash => Hash
+def assemble(job, payload, definition)
+  {
+    commands: definition['commands'],
+    context: job_context(definition['environment'] || {}, payload),
+    job_name: job[:job_name],
+    job_id: job[:job_id]
+  }
+end
+
+Contract None => Or[Hash,nil]
+def request_job
+  job_id = @sockets[:conveyor].request({method: 'dequeue'}, {})
+  return nil if job_id.nil?
+  raw_job = @sockets[:conveyor].request({method: 'lookup', data: job_id}, {})
+  puts "Assembling: '#{raw_job}'"
+  raw_job.merge job_id: job_id
+end
+
+Contract EZMQ::Socket, Any => Hash
+def request(socket, data)
+  response = socket.request data, {}
+  raise response[:error] unless response[:error].empty?
+  response[:data]
+end
+
+Contract Hash => [Hash, Hash]
+def request_data_for(job)
+  payload = request @sockets[:parser], job
+  definition = request @sockets[:dictionary], job[:job_name]
+  [payload, definition]
+end
+
+Contract Or[Array,nil], Or[String, nil] => nil
+def check_whitelist(whitelist, branch)
+  return if whitelist.nil?
+  raise 'Branch not whitelisted' unless whitelist.include? branch
+end
+
+Contract Fixnum => Bool
+def finish(job_id)
+  @sockets[:conveyor].request({method: 'finish', data: job_id}, {})
+end
+
+job_search = Fiber.new do
+  # TODO: profiler to get a better idea of how long we should wait based on historical information
+  timer = ExponentialBackoff.new 0.1, 3
+  loop do
+    sleep timer.next_interval
+    job = request_job
+    next if job.nil?
+    timer.clear
+    Fiber.yield job
+  end
+end
+
+puts 'Started'
+@sockets = {
+  conveyor: EZMQ::Client.new(port: configatron.conveyor_port),
+  worker: EZMQ::Pusher.new(:bind, port: configatron.worker_port),
+  parser: EZMQ::Client.new(port: configatron.payload_parser_port),
+  dictionary: EZMQ::Client.new(port: configatron.job_dictionary_port)
+}
+@sockets.each { |_key, socket| socket.serialize_with_json! }
+
+loop do
+  job = job_search.resume
+  begin
+    payload, definition = request_data_for job
+    check_whitelist definition['branch_whitelist'], payload['branch']
+    assembled_job = assemble job, payload, definition
+    @sockets[:worker].send(assembled_job, {})
+  rescue RuntimeError => error
+    puts error.message
+    finish job[:job_id]
+    next
+  end
+end

data/bin/sweatshop-conveyor

@@ -0,0 +1,64 @@
+#!/usr/bin/env ruby
+require 'bundler/setup'
+require 'ezmq'
+require 'stubborn_queue'
+require 'oj'
+require 'contracts'
+require 'robot_sweatshop/config'
+require 'robot_sweatshop/connections'
+using ExtendedEZMQ
+
+queue_settings = {
+  name: 'test',
+  timeout: configatron.job_timeout_length,
+  file: "#{configatron.database_path}/conveyor.db"
+}
+@items = StubbornQueue.new queue_settings
+$stdout.sync = true
+include Contracts
+
+Contract Hash => Fixnum
+def enqueue(item)
+  puts "enqueue #{item}"
+  @items.enqueue Oj.dump(item)
+end
+
+Contract None => Or[Fixnum,nil]
+def dequeue
+  puts "dequeue"
+  @items.dequeue
+end
+
+Contract Fixnum => Hash
+def lookup(id)
+  puts "lookup #{id}"
+  Oj.load @items.lookup(id)
+end
+
+Contract Fixnum => Bool
+def finish(id)
+  puts "finish #{id}"
+  @items.finish id
+end
+
+Contract String => Bool
+def supported?(method)
+  %w(enqueue dequeue lookup finish).include? method
+end
+
+Contract Hash => Any
+def complete(request)
+  arguments = []
+  return send(request[:method], request[:data]) if request[:data]
+  send(request[:method])
+end
+
+puts 'Starting the Conveyor'
+server = EZMQ::Server.new port: configatron.conveyor_port
+server.serialize_with_json!
+server.listen do |request|
+  puts "Received: #{request.inspect}"
+  next unless request.is_a? Hash
+  next unless supported? request[:method]
+  complete request
+end

data/bin/{sweatshop-input-http → sweatshop-input}

@@ -2,28 +2,34 @@
 require 'bundler/setup'
 require 'sinatra'
 require 'ezmq'
-require 'json'
 require 'robot_sweatshop/config'
+require 'robot_sweatshop/connections'
+using ExtendedEZMQ
 
 configure do
   set :port, configatron.http_port
   set :bind, configatron.http_bind
   set :run, true
+  set :conveyor, EZMQ::Client.new(port: configatron.conveyor_port)
+  settings.conveyor.serialize_with_json!
 end
 
 get '/' do
   'Everything\'s on schedule!'
 end
 
-post '/payload-for/:job_name' do
+post '/run/:job_name' do
   puts "Received payload for #{params['job_name']}"
   request.body.rewind
   payload = request.body.read
   hash = {
-    payload: payload,
-    user_agent: request.env['HTTP_USER_AGENT'],
-    job_name: params['job_name']
+    method: 'enqueue',
+    data: {
+      payload: payload,
+      user_agent: request.env['HTTP_USER_AGENT'],
+      job_name: params['job_name']
+    }
   }
-  client = EZMQ::Client.new port: 5556
-  client.request "payload #{JSON.generate hash}"
+  settings.conveyor.request hash, {}
+  200
 end

data/bin/sweatshop-job-dictionary

@@ -0,0 +1,53 @@
+#!/usr/bin/env ruby
+require 'bundler/setup'
+require 'oj'
+require 'ezmq'
+require 'contracts'
+require 'robot_sweatshop/config'
+require 'robot_sweatshop/connections'
+$stdout.sync = true
+include Contracts
+using ExtendedEZMQ
+
+Contract None => String
+def job_path
+  File.expand_path configatron.job_path
+end
+
+Contract String => Or[Hash, nil]
+def load_if_exists(config_path)
+  puts "Reading job configuration from #{config_path}"
+  YAML.load_file config_path if File.exists? config_path
+end
+
+Contract None => Hash
+def empty_config
+  puts "Job configuration not found or empty"
+  {}
+end
+
+Contract String => Hash
+def load_config_for(job_name)
+  load_if_exists("#{job_path}/#{job_name}.yaml") || empty_config
+end
+
+Contract Hash, Hash => Hash
+def formatted(payload={}, error:'')
+  {data: payload, error: error}
+end
+
+Contract String => Hash
+def define(job_name)
+  config = load_config_for job_name
+  return formatted error: 'Job not found or empty' if config.empty?
+  return formatted error: 'No commands' unless config['commands'].is_a? Array
+  formatted config
+end
+
+puts 'Started'
+server = EZMQ::Server.new port: configatron.job_dictionary_port
+server.serialize_with_json!
+server.listen do |job_name|
+  puts "Looking up: #{job_name}"
+  define job_name
+end

data/bin/sweatshop-payload-parser

@@ -1,20 +1,21 @@
 #!/usr/bin/env ruby
 require 'bundler/setup'
-require 'json'
+require 'oj'
 require 'ezmq'
 require 'contracts'
 require 'robot_sweatshop/config'
 require 'robot_sweatshop/payload'
-
+require 'robot_sweatshop/connections'
 $stdout.sync = true
 include Contracts
+using ExtendedEZMQ
 
 Contract String => Bool
 def json?(string)
   begin
-    JSON.load string
+    Oj.load string
     true
-  rescue JSON::ParserError => e
+  rescue Oj::ParseError => e
     false
   end
 end
@@ -31,42 +32,41 @@ end
 
 Contract Hash => String
 def detect_format_of(request)
-  return 'empty' if request['payload'].empty?
-  return 'Github' if from_github? request['user_agent']
-  return 'Bitbucket' if from_bitbucket? request['user_agent']
-  return 'JSON' if json? request['payload']
+  return 'empty' if request[:payload].empty?
+  return 'Github' if from_github? request[:user_agent]
+  return 'Bitbucket' if from_bitbucket? request[:user_agent]
+  return 'JSON' if json? request[:payload]
   'unsupported'
 end
 
 Contract String, String => Or[Hash, nil]
 def payload_hash_from(payload, format)
-  # TODO: use require instead of require_relative
-  lib_file = "#{__dir__}/../lib/robot_sweatshop/payload/#{format.downcase}.rb"
-  return nil unless File.file?(lib_file)
-  require_relative lib_file
-  Object.const_get("#{format}Payload").new(payload).to_hash
+  begin
+    Object.const_get("#{format}Payload").new(payload).to_hash
+  rescue NameError
+    nil
+  end
 end
 
-Contract Hash => Hash
-def parse(request)
-  format = detect_format_of request
-  return {payload: {}, error: ''} if format == 'empty'
-  payload = payload_hash_from request['payload'], format
-  return {payload: {}, error: "Can't parse #{format} payload"} if payload.nil?
-  {payload: payload, error: ''}
+Contract Hash, Hash => Hash
+def formatted(payload={}, error:'')
+  {data: payload, error: error}
 end
 
-Contract Any => Hash
-def validate(request)
-  return {payload: {}, error: "Invalid JSON request"} unless json? request
-  JSON.load request
+Contract String, String => Hash
+def parse(raw_payload, format)
+  return formatted if format == 'empty'
+  return formatted error: 'Unknown format' if format == 'unsupported'
+  payload = payload_hash_from raw_payload, format
+  return formatted error: "Invalid #{format} payload" if payload.nil?
+  formatted payload
 end
 
 puts 'Started'
-server = EZMQ::Server.new port: configatron.payload_parser_port#, encode: -> m { JSON.dump m }
+server = EZMQ::Server.new port: configatron.payload_parser_port
+server.serialize_with_json!
 server.listen do |request|
   puts "Parsing: #{request}"
-  request = validate request
-  request = parse request unless request[:error]
-  JSON.dump request
+  format = detect_format_of request
+  parse request[:payload], format
 end

data/bin/sweatshop-worker

@@ -0,0 +1,58 @@
+#!/usr/bin/env ruby
+require 'bundler/setup'
+require 'ezmq'
+require 'faker'
+require 'fileutils'
+require 'contracts'
+require 'robot_sweatshop/config'
+require 'robot_sweatshop/connections'
+using ExtendedEZMQ
+$stdout.sync = true
+$stderr.sync = true
+include Contracts
+
+# TODO: check existing worker ids. it'd be a problem to share a workspace
+@worker_id = ARGV[0] || "#{Faker::Name.first_name}"
+
+Contract Hash, Proc => Any
+def from_workspace(named:)
+  workspace = "#{named}-#{@worker_id}"
+  puts "Workspace: #{workspace}"
+  path = File.expand_path "#{configatron.workspace_path}/#{workspace}"
+  FileUtils.mkpath path
+  Dir.chdir(path) { yield if block_given? }
+end
+
+Contract Hash, String => nil
+def execute(context, command)
+  puts "Executing '#{command}'..."
+  # TODO: path.split(' ') to bypass the shell when we're not using env vars
+  IO.popen(context, command) do |io_stream|
+    while line = io_stream.gets
+      puts line
+    end
+  end
+  puts "Execution complete with exit status: #{$?.exitstatus}"
+end
+
+Contract Fixnum => nil
+def finish(id)
+  @sockets[:conveyor].request({method: 'finish', data: id}, {})
+  puts "Job finished.\n\n"
+end
+
+puts 'Starting'
+@sockets = {
+  conveyor: EZMQ::Client.new(port: configatron.conveyor_port),
+  puller: EZMQ::Puller.new(:connect, port: configatron.worker_port)
+}
+@sockets.each { |_key, socket| socket.serialize_with_json! }
+
+@sockets[:puller].listen do |data|
+  puts "Running: #{data}"
+  from_workspace named: data[:job_name] do
+    context = data[:context] || {}
+    data[:commands].each { |command| execute context, command }
+  end
+  finish data[:job_id]
+end

data/config.defaults.yaml

@@ -3,9 +3,11 @@ pidfile_path: .robot_sweatshop/run
 logfile_path: .robot_sweatshop/log
 job_path: .robot_sweatshop/jobs
 workspace_path: .robot_sweatshop/workspaces
-moneta_path: .robot_sweatshop/db
+database_path: .robot_sweatshop/db
 http_port: 8080
 http_bind: 0.0.0.0
-# queue_handler_port: 5556
-# queue_broadcaster_port: 5557
-payload_parser_port: 5558
+conveyor_port: 5555
+payload_parser_port: 5556
+job_dictionary_port: 5557
+worker_port: 5558
+job_timeout_length: 300 # 5 minutes

data/lib/README.md

@@ -1,21 +1,18 @@
 Job lifecycle:
 
 ```
-in-* -> payloads { payload:, user_agent:, job_name: } ->
-job-assembler -> jobs { context:, commands:, job_name: } ->
-job-worker
-```
-
-All passing done via the moneta core in queue/*
+input -> conveyor
+{ payload:, user_agent:, job_name: }
 
-Also queue/watcher for debugging
+assembler <-> conveyor
 
----
+assembler <-> payload-parser
+assembler <-> job-dictionary
+{ payload:, user_agent: } <-> { payload:, error: }
 
-The Job Assembler also uses the Payload Parser service:
-
-```
-{ payload:, user_agent: } -> payload-parser -> { payload:, error: }
+assembler -> worker
+{ context:, commands:, job_name:, job_id }
 ```
 
-Error is an empty string when it's successful. Otherwise it details what went wrong.
+context is passed around with string keys because it's user provided
+everything else is passed with symbol keys

data/lib/robot_sweatshop/cli/common.rb

@@ -1,5 +1,5 @@
 require 'fileutils'
-require_relative '../config'
+require 'robot_sweatshop/config'
 
 def notify(type = :success, string)
   color = case type

data/lib/robot_sweatshop/cli/config.rb

@@ -1,4 +1,4 @@
-require_relative '../config'
+require 'robot_sweatshop/config'
 
 def default_config
   File.read "#{__dir__}/../../../config.defaults.yaml"

data/lib/robot_sweatshop/cli/job.rb

@@ -1,4 +1,4 @@
-require_relative '../config'
+require 'robot_sweatshop/config'
 
 def default_job
   "---\n# branch_whitelist:\n# - master\n\ncommands:\n- echo \"Hello $WORLD!\"\n\nenvironment:\n  WORLD: Earth\n"

data/lib/robot_sweatshop/cli/start.rb

@@ -1,5 +1,5 @@
 require 'yaml'
-require_relative '../config'
+require 'robot_sweatshop/config'
 
 def store_config_for_eye
   config = configatron.to_h
@@ -12,7 +12,7 @@ end
 
 def start_sweatshop(for_environment:)
   store_config_for_eye
-  eye_config = File.expand_path "#{__dir__}/../../../robot_sweatshop.#{for_environment}.eye"
+  eye_config = File.expand_path "#{__dir__}/../../../robot_sweatshop.eye"
   output = `eye load #{eye_config}`
   if $?.exitstatus != 0
     notify :failure, output

data/lib/robot_sweatshop/config.rb

@@ -17,3 +17,5 @@ configurations.each do |config_path|
     configatron.configure_from_hash hash
   end
 end
+
+require 'robot_sweatshop/create-config-directories'

data/lib/robot_sweatshop/connections.rb

@@ -0,0 +1,17 @@
+require 'ezmq'
+require 'oj'
+require 'robot_sweatshop/config'
+
+module ExtendedEZMQ
+  refine EZMQ::Socket do
+    def serialize_with_json!
+      self.encode = -> message { Oj.dump message }
+      self.decode = -> message { Oj.load message }
+    end
+
+    def close
+      self.socket.close
+      self.context.terminate
+    end
+  end
+end

data/lib/robot_sweatshop/create-config-directories.rb

@@ -1,5 +1,4 @@
 require 'fileutils'
-require_relative 'config'
 
 def create_path(path)
   begin

data/lib/robot_sweatshop.rb

@@ -1,5 +1,4 @@
-require_relative 'robot_sweatshop/moneta-queue'
-require_relative 'robot_sweatshop/queue-helper'
 require_relative 'robot_sweatshop/config'
 require_relative 'robot_sweatshop/payload'
 require_relative 'robot_sweatshop/cli'
+require_relative 'robot_sweatshop/connections'