robot_sweatshop 0.1.2

data/lib/input/http.rb

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+require 'sinatra'
+require 'ezmq'
+require 'json'
+require_relative '../../config'
+
+configure do
+  set :port, configatron.input.http.port
+  set :bind, configatron.input.http.bind
+  set :output_queue, 'raw-payload'
+end
+
+get '/' do
+  'Everything\'s on schedule!'
+end
+
+post '/:format/payload-for/:job_name' do
+  puts "Received #{params['format']} payload for #{params['job_name']}"
+  request.body.rewind
+  hash = {
+    payload: request.body.read,
+    format: params['format'],
+    job_name: params['job_name']
+  }
+  client = EZMQ::Client.new port: 5556
+  client.request "#{settings.output_queue} #{JSON.generate hash}"
+end

data/lib/job/assembler.rb

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+require 'yaml'
+require_relative '../queue-helper'
+
+def get_config(for_job_name:)
+  job_directory = "#{__dir__}/../../jobs"
+  job_config_path = "#{job_directory}/#{for_job_name}.yaml"
+  unless File.file? job_config_path
+    puts "No config found for job '#{for_job_name}'"
+    return nil
+  end
+  YAML.load_file job_config_path
+end
+
+def assemble_job(data)
+  job_config = get_config for_job_name: data['job_name']
+  return nil unless job_config
+  if job_config['branch_whitelist'].include? data['payload']['branch']
+    context = job_config['environment'].merge(data['payload'])
+    context.each { |key, value| context[key] = value.to_s }
+    {
+      commands: job_config['commands'],
+      context: context,
+      job_name: data['job_name']
+    }
+  else
+    puts "Branch '#{data['payload']['branch']}' is not whitelisted"
+    nil
+  end
+end
+
+QueueHelper.wait_for('parsed-payload') do |data|
+  puts "Assembling: #{data}"
+  assembled_job = assemble_job data
+  QueueHelper.enqueue assembled_job, to: 'jobs' if assembled_job
+end

data/lib/job/worker.rb

@@ -0,0 +1,40 @@
+#!/usr/bin/env ruby
+require 'faker'
+require 'fileutils'
+require_relative '../queue-helper'
+
+# TODO: check existing worker ids. it'd be disastrous to have two sharing a workspace
+@worker_id = ARGV[0] || "#{Faker::Name.first_name}"
+
+def from_workspace(named: 'no_job_name')
+  workspace = "#{named}-#{@worker_id}"
+  puts "Workspace: #{workspace}"
+  path = File.expand_path "#{__dir__}/../../workspaces/#{workspace}"
+  FileUtils.mkpath path
+  Dir.chdir(path) { yield if block_given? }
+end
+
+def execute(context = {}, command)
+  puts "Executing '#{command}'..."
+  # TODO: path.split(' ') to bypass the shell when we're not using env vars
+  
+  # Run the command with the context in environment,
+  # printing the output as it's generated
+  IO.popen(context, command) do |io_stream|
+    while line = io_stream.gets
+      puts line
+    end
+  end
+  puts "Execution complete with exit status: #{$?.exitstatus}"
+end
+
+QueueHelper.wait_for('jobs') do |data|
+  puts "Running: #{data}"
+  if data['commands'].is_a? Array
+    from_workspace(named: data['job_name']) do
+      context = data['context'] || {}
+      data['commands'].each { |command| execute context, command }
+    end
+  end
+  puts "Job finished.\n\n"
+end

data/lib/payload/lib/bitbucket.rb

@@ -0,0 +1,61 @@
+require 'uri'
+require 'json'
+require_relative 'payload'
+
+# A parser for Bitbucket payload data
+class BitbucketPayload < Payload
+  def initialize(data)
+    data = URI.decode_www_form(data)[0][1]
+    @data = JSON.parse data || {}
+  end
+
+  def latest_commit
+    return {} if @data['commits'].nil?
+    @data['commits'].last
+  end
+
+  def repository
+    return {} if @data['repository'].nil?
+    @data['repository']
+  end
+
+  def author
+    return {} if latest_commit['raw_author'].nil?
+    name, email = latest_commit['raw_author'].split(/\s+</)
+    email.slice! '>' unless email.nil?
+    {
+      'name' => name,
+      'email' => email || '',
+      'username' => latest_commit['author']
+    }
+  end
+
+  def clone_url
+    "#{ @data['canon_url'] }#{ repository['absolute_url'] }"
+  end
+
+  def hash
+    latest_commit['raw_node']
+  end
+
+  def branch
+    latest_commit['branch']
+  end
+
+  def message
+    latest_commit['message']
+  end
+
+  def repo_slug
+    slug = repository['absolute_url']
+    slug.nil? ? nil : slug[1...-1]
+  end
+
+  def source_url
+    return '' if  @data['canon_url'].nil? ||
+                  repository.empty? ||
+                  latest_commit.empty?
+    base_url = @data['canon_url']
+    "#{base_url}/#{repo_slug}/commits/#{hash}/?at=#{branch}"
+  end
+end

data/lib/payload/lib/github.rb

@@ -0,0 +1,45 @@
+require 'json'
+require_relative 'payload'
+
+# A parser for Github payload data
+class GithubPayload < Payload
+  def initialize(payload)
+    @payload = JSON.parse payload || {}
+  end
+
+  def commit
+    @payload['head_commit'] || {}
+  end
+
+  def repository
+    @payload['repository'] || {}
+  end
+
+  def clone_url
+    repository['clone_url'] || ''
+  end
+
+  def author
+    commit['author']
+  end
+
+  def hash
+    commit['id'] || {}
+  end
+
+  def branch
+    @payload['ref'] || ''
+  end
+
+  def message
+    commit['message'] || ''
+  end
+
+  def repo_slug
+    repository['full_name'] || ''
+  end
+
+  def source_url
+    head_commit['url'] || ''
+  end
+end

data/lib/payload/lib/payload.rb

@@ -0,0 +1,12 @@
+# The base interface that the payload parser will be expecting
+class Payload
+  def self.hash_keys
+    %w(author hash branch message repo_slug source_url clone_url)
+  end
+
+  def to_hash
+    keys = Payload.hash_keys
+    values = Payload.hash_keys.map { |method| method(method.to_sym).call }
+    [keys, values].transpose.to_h
+  end
+end

data/lib/payload/parser.rb

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+require_relative '../queue-helper'
+
+def parse(payload = '', of_format:)
+  of_format = of_format.downcase
+  lib_file = "#{__dir__}/lib/#{of_format}.rb"
+  if of_format != 'payload' && File.file?(lib_file)
+    require_relative lib_file
+    Object.const_get("#{of_format.capitalize}Payload").new payload
+  else
+    puts "Dropping bad format: #{of_format}"
+    nil
+  end
+end
+
+QueueHelper.wait_for('raw-payload') do |data|
+  puts "Parsing: #{data}"
+  payload = parse data['payload'], of_format: data['format']
+  if payload
+    hash = { payload: payload.to_hash, job_name: data['job_name'] }
+    QueueHelper.enqueue hash, to: 'parsed-payload'
+  end
+end

data/lib/queue-helper.rb

@@ -0,0 +1,32 @@
+#!/usr/bin/env ruby
+require 'ezmq'
+require 'json'
+
+# A collection of common methods for queue interactions with EZMQ
+module QueueHelper
+  @@client = EZMQ::Client.new port: 5556
+
+  def self.dequeue(queue_name = 'default')
+    data = @@client.request queue_name
+    begin
+      JSON.parse data
+    rescue JSON::ParserError
+      nil
+    end
+  end
+
+  def self.enqueue(object = {}, to: 'default')
+    @@client.request "#{to} #{JSON.generate object}"
+  end
+
+  def self.wait_for(queue_name = 'default')
+    puts "Waiting for messages on #{queue_name}"
+    subscriber = EZMQ::Subscriber.new port: 5557, topic: 'busy-queues'
+    subscriber.listen do |message|
+      if message == queue_name
+        data = dequeue queue_name
+        yield data unless data.nil?
+      end
+    end
+  end
+end

data/lib/queue/broadcaster.rb

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+require_relative 'lib/moneta-queue'
+require 'ezmq'
+
+publisher = EZMQ::Publisher.new port: 5557
+queues = {}
+MonetaQueue.watched_queues.each do |queue|
+  queues[queue] = MonetaQueue.new queue
+end
+
+@wait_time = ARGV[0].to_f || 0.5
+
+loop do
+  queues.each do |queue_name, queue|
+    publisher.send queue_name, topic: 'busy-queues' if queue.size > 0
+  end
+  sleep @wait_time unless @wait_time == 0
+end

data/lib/queue/handler.rb

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+require_relative 'lib/moneta-queue'
+require 'ezmq'
+
+def enqueue(name, item)
+  puts "enqueue #{name} #{item}"
+  queue = MonetaQueue.new name
+  queue.enqueue item
+  queue.size.to_s
+end
+
+def dequeue(name)
+  puts "dequeue #{name}"
+  queue = MonetaQueue.new name
+  queue.dequeue
+end
+
+server = EZMQ::Server.new port: 5556
+server.listen do |message|
+  name, item = message.split ' ', 2
+  is_dequeue_request = item.nil?
+  is_dequeue_request ? dequeue(name) : enqueue(name, item)
+end

data/lib/queue/lib/moneta-queue.rb

@@ -0,0 +1,49 @@
+#!/usr/bin/env ruby
+require 'moneta'
+require_relative '../../../config'
+
+# A class to handle queueing through Moneta's key-value storage
+class MonetaQueue
+  attr_reader :watched_queues
+
+  @@store = Moneta.new :File, dir: configatron.queue.moneta_directory
+
+  def initialize(name)
+    @name = name
+    @mirror_name = "mirror-#{name}"
+    @@store[@name] ||= []
+    @@store[@mirror_name] ||= []
+  end
+
+  def self.watched_queues
+    %w(raw-payload parsed-payload jobs testing)
+  end
+
+  def enqueue(item)
+    @@store[@name] = @@store[@name].push item
+    @@store[@mirror_name] = @@store[@mirror_name].push item
+  end
+
+  def dequeue
+    return '' if @@store[@name].empty?
+    item = @@store[@name].first
+    @@store[@name] = @@store[@name][1..-1]
+    item
+  end
+
+  def size
+    loop do # Moneta can return nil sometimes, so we give it time to catch up
+      queue = @@store[@name]
+      return queue.size unless queue.nil?
+    end
+  end
+
+  def clear
+    @@store[@mirror_name] = []
+    @@store[@name] = []
+  end
+
+  def inspect
+    @@store[@name].inspect
+  end
+end

data/lib/queue/watcher.rb

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+require_relative 'lib/moneta-queue'
+
+@queues = []
+[ARGV[0], "mirror-#{ARGV[0]}"].each do |queue_name|
+  @queues.push(name: queue_name, queue: MonetaQueue.new(queue_name))
+end
+
+loop do
+  system 'clear'
+  @queues.each do |q|
+    puts "Queue: #{q[:name]}"
+    puts "Size: #{q[:queue].size}", "#{'|' * q[:queue].size}"
+    puts q[:queue].inspect
+    puts
+  end
+  sleep 0.1
+end

data/robot_sweatshop.eye

@@ -0,0 +1,58 @@
+#!/usr/bin/env ruby
+require_relative 'config'
+
+log_path = configatron.common.logfile_directory
+pid_path = configatron.common.pidfile_directory
+
+Eye.config do
+  logger "#{log_path}/eye.log"
+end
+
+Eye.application :robot_sweatshop do
+  trigger :flapping, times: 10, within: 1.minute, retry_in: 10.minutes
+  check :cpu, every: 10.seconds, below: 100, times: 3
+  working_dir "#{__dir__}/lib"
+
+  group 'input' do
+    process :http do
+      pid_file "#{pid_path}/input-http.pid"
+      stdall "#{log_path}/input-http.log"
+      start_command "ruby input/http.rb"
+      daemonize true
+    end
+  end
+  group 'queue' do
+    process :handler do
+      pid_file "#{pid_path}/queue-handler.pid"
+      stdall "#{log_path}/queue-handler.log"
+      start_command "ruby queue/handler.rb"
+      daemonize true
+    end
+    process :broadcaster do
+      pid_file "#{pid_path}/queue-broadcaster.pid"
+      stdall "#{log_path}/queue-broadcaster.log"
+      start_command "ruby queue/broadcaster.rb #{configatron.eye.broadcaster_interval}"
+      daemonize true
+    end
+  end
+  group 'job' do
+    process :assembler do
+      pid_file "#{pid_path}/job-assembler.pid"
+      stdall "#{log_path}/job-assembler.log"
+      start_command "ruby job/assembler.rb"
+      daemonize true
+    end
+    process :worker do
+      pid_file "#{pid_path}/job-worker.pid"
+      stdall "#{log_path}/job-worker.log"
+      start_command "ruby job/worker.rb #{configatron.eye.worker_id}"
+      daemonize true
+    end
+  end
+  process :payload_parser do
+    pid_file "#{pid_path}/payload_parser.pid"
+    stdall "#{log_path}/payload_parser.log"
+    start_command "ruby payload/parser.rb"
+    daemonize true
+  end
+end