robot_sweatshop 0.3.3 → 0.4.0

data/test/shared/helpers.rb

@@ -1,84 +1,5 @@
-require 'bundler/setup'
-require 'yaml'
-require 'json'
-require 'robot_sweatshop/moneta-queue'
-require 'robot_sweatshop/payload'
-require 'robot_sweatshop/config'
+require_relative 'helpers/input'
+require_relative 'helpers/output'
 
-module QueueHelper
-  def clear_all_queues
-    MonetaQueue.watched_queues.each do |queue|
-      queue = MonetaQueue.new queue
-      queue.clear
-    end
-  end
-  def enqueue(queue_name, item)
-    queue = MonetaQueue.new queue_name
-    queue.enqueue item
-  end
-end
-
-module InHelper
-  def example_raw_payload(of_format:)
-    payload_strings = YAML.load_file "#{__dir__}/../data/payload_data.yaml"
-    payload_strings[of_format.downcase]
-  end
-  def input_http_url(for_job: 'test_job')
-    "http://localhost:#{configatron.http_port}/payload-for/#{for_job}"
-  end
-  def user_agent_for(format)
-    case format
-    when 'Bitbucket'
-      'Bitbucket.org'
-    when 'Github'
-      'Github-Hookshot'
-    else
-      'SomeUserAgent'
-    end
-  end
-  def example_payload_request(of_format:)
-    {
-      payload: example_raw_payload(of_format: of_format),
-      user_agent: user_agent_for(of_format)
-    }
-  end
-  def job_request_data(type)
-    case type
-    when 'Git'
-      ['Bitbucket', 'git_job'] # develop branch payload
-    when 'JSON'
-      ['JSON', 'test_job']
-    when 'MinimalJob'
-      ['JSON', 'minimal_job']
-    when 'IgnoredBranch'
-      ['Github', 'git_job'] # master branch payload
-    when 'UnknownJob'
-      ['Bitbucket', 'unknown_job']
-    when 'EmptyJob'
-      ['JSON', 'empty_job']
-    when 'NonJSON'
-      ['NonJSON', 'test_job']
-    else
-      ['', '']
-    end
-  end
-  def example_job_request(of_type:)
-    format, job_name = job_request_data of_type
-    JSON.generate payload: example_raw_payload(of_format: format),
-                  job_name: job_name,
-                  user_agent: user_agent_for(format)
-  end
-end
-
-module JobHelper
-  def example_job(in_context: {}, with_commands: [])
-    JSON.generate context: in_context,
-                  commands: with_commands,
-                  job_name: 'test_job'
-  end
-  def reset_test_file
-    test_file = File.expand_path "#{configatron.workspace_path}/test_job-testingid/test.txt"
-    FileUtils.rm_rf test_file
-    test_file
-  end
-end
+$a_moment = 0.1
+$a_while = 1

data/test/shared/scaffolding.rb

@@ -0,0 +1,42 @@
+require 'bundler/setup'
+require 'fileutils'
+require 'oj'
+require 'robot_sweatshop/config'
+require 'robot_sweatshop/connections'
+require_relative 'stub'
+
+module Setup
+  def self.empty_conveyor
+    db_file = "#{configatron.database_path}/conveyor.db"
+    File.truncate db_file, 0 if File.exist? db_file
+  end
+
+  def self.populate_test_jobs
+    test_jobs = Dir.glob "#{__dir__}/../data/*_job.yaml"
+    test_jobs.each { |test_job| FileUtils.cp test_job, configatron.job_path }
+  end
+end
+
+module TestProcess
+  def self.start(name_list)
+    pids = []
+    name_list.each do |name|
+      input_script = File.expand_path "#{__dir__}/../../bin/sweatshop-#{name}"
+      input_script += ' testingid' if name == 'worker'
+      pids.push spawn(input_script, out: '/dev/null', err: '/dev/null')
+    end
+    pids
+  end
+
+  def self.stop(pid_list)
+    pid_list.each { |pid| Process.kill 'TERM', pid }
+  end
+
+  def self.stub(process_name)
+    process = {
+      conveyor: {type: 'Server', port: configatron.conveyor_port},
+      worker:   {type: 'Puller', port: configatron.worker_port}
+    }[process_name]
+    Stub.new process[:type], on_port: process[:port]
+  end
+end

data/test/shared/stub.rb

@@ -0,0 +1,44 @@
+require 'ezmq'
+using ExtendedEZMQ
+
+class Stub
+  def initialize(type, on_port:)
+    @type = type
+    @port = on_port
+    clear_output
+    @thread = Thread.new { start_listener type, on_port } unless running?
+  end
+
+  def running?
+    not `lsof -i :#{@port}`.empty? # TODO: I'm so sorry
+  end
+
+  def output_file
+    ".test.#{@type}"
+  end
+
+  def clear_output
+    FileUtils.rm output_file if File.exist? output_file
+  end
+
+  def output_empty?
+    (not File.exist? output_file) || File.read(output_file).empty?
+  end
+
+  def start_listener(type, port)
+    listener = case type
+    when 'Puller'
+      EZMQ::Puller.new :connect, port: port
+    else
+      EZMQ.const_get(type).new port: port
+    end
+    listener.serialize_with_json!
+    listener.listen { |message| write message }
+  end
+
+  def write(message)
+    file = File.new output_file, 'w'
+    file.write message
+    file.close
+  end
+end

data/test/worker_spec.rb

@@ -0,0 +1,59 @@
+require 'bundler/setup'
+require 'kintama'
+require 'ezmq'
+require 'timeout'
+require 'robot_sweatshop/config'
+require 'robot_sweatshop/connections'
+require_relative 'shared/scaffolding'
+require_relative 'shared/helpers'
+$stdout.sync = true
+
+Kintama.on_start do
+  Setup.empty_conveyor
+  @pids = TestProcess.start %w(worker)
+end
+
+Kintama.on_finish do
+  TestProcess.stop @pids
+end
+
+describe 'the Worker' do
+  include InputHelper
+  include OutputHelper
+  using ExtendedEZMQ
+
+  setup do
+    @conveyor = TestProcess.stub :conveyor
+    @pusher = EZMQ::Pusher.new :bind, port: configatron.worker_port
+    @pusher.serialize_with_json!
+    clear_worker_output
+  end
+
+  teardown do
+    @pusher.close
+  end
+
+  given 'valid job data is pushed' do
+    setup do
+      @pusher.send worker_push
+      Timeout.timeout($a_while) do
+        loop until File.exist? worker_output
+        loop while @conveyor.output_empty?
+      end
+      @worker_data = eval File.read(@conveyor.output_file)
+    end
+
+    should 'run the commands' do
+      assert_equal true, File.file?(worker_output)
+    end
+
+    should 'run with the context as environment variables' do
+      assert_equal "hello world\n", File.read(worker_output)
+    end
+
+    should 'tell the conveyor that job is complete' do
+      assert_equal 'finish', @worker_data[:method]
+      assert_kind_of Fixnum, @worker_data[:data]
+    end
+  end
+end

metadata

@@ -1,17 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: robot_sweatshop
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.4.0
 platform: ruby
 authors:
 - Justin Scott
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-11 00:00:00.000000000 Z
+date: 2015-05-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: sinatra
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -25,7 +25,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: ezmq
+  name: faker
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -39,7 +39,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: faker
+  name: commander
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -53,7 +53,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: commander
+  name: eye
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -67,7 +67,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: eye
+  name: colorize
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -81,7 +81,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: colorize
+  name: configatron
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -95,7 +95,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: configatron
+  name: moneta
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -109,7 +109,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: moneta
+  name: contracts
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -123,7 +123,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: contracts
+  name: sinatra
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -137,13 +137,13 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: ezmq
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -151,13 +151,13 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: kintama
+  name: stubborn_queue
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -165,7 +165,35 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: http
+  name: oj
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: exponential-backoff
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -179,7 +207,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: pry
+  name: kintama
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -193,7 +221,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: pry-byebug
+  name: http
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -223,14 +251,14 @@ dependencies:
 description: A lightweight, unopinionated CI server.
 email: jvscott@gmail.com
 executables:
+- README.md
 - sweatshop
-- sweatshop-input-http
-- sweatshop-job-assembler
-- sweatshop-job-worker
+- sweatshop-assembler
+- sweatshop-conveyor
+- sweatshop-input
+- sweatshop-job-dictionary
 - sweatshop-payload-parser
-- sweatshop-queue-broadcaster
-- sweatshop-queue-handler
-- sweatshop-queue-watcher
+- sweatshop-worker
 extensions: []
 extra_rdoc_files: []
 files:
@@ -240,14 +268,14 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- bin/README.md
 - bin/sweatshop
-- bin/sweatshop-input-http
-- bin/sweatshop-job-assembler
-- bin/sweatshop-job-worker
+- bin/sweatshop-assembler
+- bin/sweatshop-conveyor
+- bin/sweatshop-input
+- bin/sweatshop-job-dictionary
 - bin/sweatshop-payload-parser
-- bin/sweatshop-queue-broadcaster
-- bin/sweatshop-queue-handler
-- bin/sweatshop-queue-watcher
+- bin/sweatshop-worker
 - config.defaults.yaml
 - lib/README.md
 - lib/robot_sweatshop.rb
@@ -257,35 +285,35 @@ files:
 - lib/robot_sweatshop/cli/job.rb
 - lib/robot_sweatshop/cli/start.rb
 - lib/robot_sweatshop/config.rb
+- lib/robot_sweatshop/connections.rb
 - lib/robot_sweatshop/create-config-directories.rb
-- lib/robot_sweatshop/moneta-queue.rb
 - lib/robot_sweatshop/payload.rb
 - lib/robot_sweatshop/payload/bitbucket.rb
 - lib/robot_sweatshop/payload/github.rb
 - lib/robot_sweatshop/payload/json.rb
 - lib/robot_sweatshop/payload/payload.rb
-- lib/robot_sweatshop/queue-helper.rb
 - robot_sweatshop.eye
 - robot_sweatshop.gemspec
-- robot_sweatshop.production.eye
-- robot_sweatshop.testing.eye
 - test/README.md
+- test/all.rb
+- test/assembler_spec.rb
+- test/conveyor_spec.rb
 - test/data/empty_job.yaml
 - test/data/git_job.yaml
 - test/data/minimal_job.yaml
 - test/data/payload_data.yaml
 - test/data/test_job.yaml
+- test/data/weird_job.yaml
 - test/end-to-end_spec.rb
-- test/input_http_spec.rb
-- test/job_assembler_spec.rb
-- test/job_worker_spec.rb
-- test/moneta-queue_spec.rb
+- test/input_spec.rb
+- test/job_dictionary_spec.rb
 - test/payload_parser_spec.rb
-- test/queue_broadcaster_spec.rb
-- test/queue_handler_spec.rb
-- test/run_all.rb
 - test/shared/helpers.rb
-- test/shared/process_spawning.rb
+- test/shared/helpers/input.rb
+- test/shared/helpers/output.rb
+- test/shared/scaffolding.rb
+- test/shared/stub.rb
+- test/worker_spec.rb
 homepage: http://www.github.com/jscott/robot_sweatshop/
 licenses:
 - MIT
@@ -306,7 +334,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.3
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: Robot Sweatshop
@@ -316,6 +344,10 @@ test_files:
 - test/data/minimal_job.yaml
 - test/data/payload_data.yaml
 - test/data/test_job.yaml
+- test/data/weird_job.yaml
 - test/shared/helpers.rb
-- test/shared/process_spawning.rb
+- test/shared/helpers/input.rb
+- test/shared/helpers/output.rb
+- test/shared/scaffolding.rb
+- test/shared/stub.rb
 has_rdoc: 

data/bin/sweatshop-job-assembler

@@ -1,82 +0,0 @@
-#!/usr/bin/env ruby
-require 'bundler/setup'
-require 'yaml'
-require 'json'
-require 'contracts'
-require 'robot_sweatshop/queue-helper'
-require 'robot_sweatshop/config'
-
-$stdout.sync = true
-include Contracts
-
-Contract None => Hash
-def empty_config
-  puts "Job configuration not found or empty"
-  {}
-end
-
-Contract None => String
-def job_path
-  File.expand_path configatron.job_path
-end
-
-Contract Or[String, nil], Hash => Bool
-def whitelisted?(branch, in_config:)
-  return true if in_config['branch_whitelist'].nil?
-  in_config['branch_whitelist'].include? branch
-end
-
-Contract Hash => Hash
-def sanitize(data)
-  data.each_pair { |key, value| data[key] = value.to_s }
-end
-
-Contract Hash, Hash => Hash
-def job_context(job_environment, context_from_payload)
-  sanitize job_environment.merge(context_from_payload)
-end
-
-Contract String => Or[Hash, nil]
-def load_if_exists(config)
-  puts "Reading job configuration from #{config}"
-  YAML.load_file config if File.exists? config
-end
-
-Contract String => Hash
-def load_config_for(job_name)
-  load_if_exists("#{job_path}/#{job_name}.yaml") || empty_config
-end
-
-Contract Hash => [Hash, String]
-def parse_payload(request)
-  client = EZMQ::Client.new port: configatron.payload_parser_port
-  response = JSON.load client.request(JSON.dump request)
-  puts response['error'] unless response['error'].empty?
-  [response['payload'], response['error']]
-end
-
-Contract Hash, Hash => Bool
-def can_work_with?(job_config, payload)
-  return false if job_config.empty?
-  return false unless whitelisted? payload['branch'], in_config: job_config
-  true
-end
-
-Contract Hash => Or[Hash, false]
-def assemble_job(request)
-  job_config = load_config_for request['job_name']
-  payload, error = parse_payload request
-  return false unless error.empty? && can_work_with?(job_config, payload)
-  {
-    commands: job_config['commands'],
-    context: job_context(job_config['environment'] || {}, payload),
-    job_name: request['job_name']
-  }
-end
-
-puts 'Started'
-QueueHelper.wait_for('payload') do |request|
-  puts "Assembling: #{request}"
-  assembled_job = assemble_job request
-  QueueHelper.enqueue assembled_job, to: 'jobs' if assembled_job
-end

data/bin/sweatshop-job-worker

@@ -1,45 +0,0 @@
-#!/usr/bin/env ruby
-require 'bundler/setup'
-require 'faker'
-require 'fileutils'
-require 'robot_sweatshop/queue-helper'
-require 'robot_sweatshop/config'
-
-$stdout.sync = true
-$stderr.sync = true
-
-# TODO: check existing worker ids. it'd be disastrous to have two sharing a workspace
-@worker_id = ARGV[0] || "#{Faker::Name.first_name}"
-
-def from_workspace(named: 'no_job_name')
-  workspace = "#{named}-#{@worker_id}"
-  puts "Workspace: #{workspace}"
-  path = File.expand_path "#{configatron.workspace_path}/#{workspace}"
-  FileUtils.mkpath path
-  Dir.chdir(path) { yield if block_given? }
-end
-
-def execute(context = {}, command)
-  puts "Executing '#{command}'..."
-  # TODO: path.split(' ') to bypass the shell when we're not using env vars
-
-  # Run the command with the context in environment,
-  # printing the output as it's generated
-  IO.popen(context, command) do |io_stream|
-    while line = io_stream.gets
-      puts line
-    end
-  end
-  puts "Execution complete with exit status: #{$?.exitstatus}"
-end
-
-QueueHelper.wait_for('jobs') do |data|
-  puts "Running: #{data}"
-  if data['commands'].is_a? Array
-    from_workspace(named: data['job_name']) do
-      context = data['context'] || {}
-      data['commands'].each { |command| execute context, command }
-    end
-  end
-  puts "Job finished.\n\n"
-end

data/bin/sweatshop-queue-broadcaster

@@ -1,20 +0,0 @@
-#!/usr/bin/env ruby
-require 'bundler/setup'
-require 'ezmq'
-require 'robot_sweatshop/moneta-queue'
-
-publisher = EZMQ::Publisher.new port: 5557
-queues = {}
-MonetaQueue.watched_queues.each do |queue|
-  queues[queue] = MonetaQueue.new queue
-end
-
-@wait_time = ARGV[0].nil? ? 0.5 : ARGV[0].to_f
-
-loop do
-  queues.each do |queue_name, queue|
-    p queue_name if queue.size > 0
-    publisher.send queue_name, topic: 'busy-queues' if queue.size > 0
-  end
-  sleep @wait_time unless @wait_time == 0
-end

data/bin/sweatshop-queue-handler

@@ -1,24 +0,0 @@
-#!/usr/bin/env ruby
-require 'bundler/setup'
-require 'ezmq'
-require 'robot_sweatshop/moneta-queue'
-
-def enqueue(name, item)
-  puts "enqueue #{name} #{item}"
-  queue = MonetaQueue.new name
-  queue.enqueue item
-  queue.size.to_s
-end
-
-def dequeue(name)
-  puts "dequeue #{name}"
-  queue = MonetaQueue.new name
-  queue.dequeue
-end
-
-server = EZMQ::Server.new port: 5556
-server.listen do |message|
-  name, item = message.split ' ', 2
-  is_dequeue_request = item.nil?
-  is_dequeue_request ? dequeue(name) : enqueue(name, item)
-end