robot_sweatshop 0.4.12 → 0.4.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fffc414119fc6a821cc01c60fea6d845782a2e0e
-  data.tar.gz: bc2c3d9a589a306d6de391119613a98c673a9de5
+  metadata.gz: 08cbf83ffc230f92b481c9bd3e050e30c8b2a771
+  data.tar.gz: 3945c41f5f694828ad9651c18e6e8d33e761f0b8
 SHA512:
-  metadata.gz: 1d659543269e4645aeb12281ee5b32f7f0b59ff1caaa5d57188e3391813e86c5f98e236434ed4db611995341c597a3570324415ff9fd4da28be1583c29cf3e97
-  data.tar.gz: 84d9a03f30e543c3744d3082166952c2784777f6926b37c4bb2067d10a835057f006c384d6ac95870c8b10e0e57b322cc9cef84b1627b1d3e852719307072978
+  metadata.gz: b0c61a1c671dd53bf48c3aa723cc78b41266817492bc49312d2bdc1cb216491c2f2b597e83659e1ee9479519af92de3d3c207a60f536cd5f8dcebdaf04899195
+  data.tar.gz: 10fa29c67fc744705d541c7846bda2882529c7cf79279838e6c3028ba6795f6cecba7633d5d390efda31483a7db5dcaa5b16515bb18319a331575f871027b27b

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    robot_sweatshop (0.4.9)
+    robot_sweatshop (0.4.12)
       commander
       configatron
       contracts

data/bin/sweatshop

@@ -7,7 +7,7 @@ require 'robot_sweatshop/config'
 require 'robot_sweatshop/create-config-directories'
 
 program :name, 'Robot Sweatshop'
-program :version, '0.4.12'
+program :version, '0.4.13'
 program :description, 'A lightweight, nonopinionated CI server'
 program :help, 'Author', 'Justin Scott <jvscott@gmail.com>'
 

data/bin/sweatshop-api

@@ -10,6 +10,7 @@ configure do
   set :port, configatron.api_port
   set :bind, configatron.http_bind
   set :run, true
+  set :log, EZMQ::Logger.new('api')
   set :conveyor, EZMQ::Client.new(port: configatron.conveyor_port)
   settings.conveyor.serialize_with_json!
   enable :cross_origin if configatron.http_cross_origin
@@ -20,7 +21,7 @@ get '/' do
 end
 
 post '/run/:job_name' do
-  puts "Received payload for #{params['job_name']}"
+  settings.log.write "Received payload for #{params['job_name']}"
   request.body.rewind
   payload = request.body.read
   hash = {

data/bin/sweatshop-assembler

@@ -5,7 +5,6 @@ require 'exponential_backoff'
 require 'ezmq'
 require 'robot_sweatshop/config'
 require 'robot_sweatshop/connections'
-$stdout.sync = true
 include Contracts
 using ExtendedEZMQ
 
@@ -45,7 +44,7 @@ def request_job
   job_id = @sockets[:conveyor].request method: 'dequeue'
   return nil if job_id.nil?
   raw_job = @sockets[:conveyor].request method: 'lookup', data: job_id
-  puts "Assembling: '#{raw_job}'"
+  @sockets[:logger].write "Assembling: '#{raw_job}'"
   raw_job.merge job_id: job_id
 end
 
@@ -87,15 +86,18 @@ job_search = Fiber.new do
   end
 end
 
-puts 'Started'
 @sockets = {
   conveyor: EZMQ::Client.new(port: configatron.conveyor_port),
   worker: EZMQ::Pusher.new(:bind, port: configatron.worker_port),
   parser: EZMQ::Client.new(port: configatron.payload_parser_port),
-  dictionary: EZMQ::Client.new(port: configatron.job_dictionary_port)
+  dictionary: EZMQ::Client.new(port: configatron.job_dictionary_port),
+  logger: EZMQ::Logger.new('assembler')
 }
-@sockets.each { |_key, socket| socket.serialize_with_json! }
+@sockets.each do |key, socket|
+  socket.serialize_with_json! unless key == :logger
+end
 
+@sockets[:logger].write 'Started'
 loop do
   job = job_search.resume
   begin
@@ -104,7 +106,7 @@ loop do
     assembled_job = assemble job, payload, definition
     @sockets[:worker].send assembled_job
   rescue RuntimeError => error
-    puts error.message
+    @sockets[:logger].write error.message
     finish job[:job_id]
     next
   end

data/bin/sweatshop-conveyor

@@ -5,6 +5,7 @@ require 'contracts'
 require 'robot_sweatshop/config'
 require 'robot_sweatshop/connections'
 using ExtendedEZMQ
+include Contracts
 
 queue_settings = {
   name: 'test',
@@ -12,30 +13,28 @@ queue_settings = {
   file: "#{configatron.database_path}/conveyor.db"
 }
 @items = StubbornQueue.new queue_settings
-$stdout.sync = true
-include Contracts
 
 Contract Hash => Fixnum
 def enqueue(item)
-  puts "enqueue #{item}"
+  @sockets[:logger].write "enqueue #{item}"
   @items.enqueue item
 end
 
 Contract None => Maybe[Fixnum]
 def dequeue
-  puts 'dequeue'
+  @sockets[:logger].write 'dequeue'
   @items.dequeue
 end
 
 Contract Fixnum => Hash
 def lookup(id)
-  puts "lookup #{id}"
+  @sockets[:logger].write "lookup #{id}"
   @items.lookup(id)
 end
 
 Contract Fixnum => Bool
 def finish(id)
-  puts "finish #{id}"
+  @sockets[:logger].write "finish #{id}"
   @items.finish id
 end
 
@@ -50,11 +49,17 @@ def complete(request)
   send(request[:method])
 end
 
-puts 'Starting the Conveyor'
-server = EZMQ::Server.new port: configatron.conveyor_port
-server.serialize_with_json!
-server.listen do |request|
-  puts "Received: #{request.inspect}"
+@sockets = {
+  server: EZMQ::Server.new(port: configatron.conveyor_port),
+  logger: EZMQ::Logger.new('conveyor')
+}
+@sockets.each do |key, socket|
+  socket.serialize_with_json! unless key == :logger
+end
+
+@sockets[:logger].write 'Started'
+@sockets[:server].listen do |request|
+  @sockets[:logger].write "Received: #{request.inspect}"
   next unless request.is_a? Hash
   next unless supported? request[:method]
   complete request

data/bin/sweatshop-job-dictionary

@@ -4,7 +4,6 @@ require 'ezmq'
 require 'contracts'
 require 'robot_sweatshop/config'
 require 'robot_sweatshop/connections'
-$stdout.sync = true
 include Contracts
 using ExtendedEZMQ
 
@@ -15,13 +14,13 @@ end
 
 Contract String => Or[Hash, nil]
 def load_if_exists(config_path)
-  puts "Reading job configuration from #{config_path}"
+  @sockets[:logger].write "Reading job configuration from #{config_path}"
   YAML.load_file config_path if File.exist? config_path
 end
 
 Contract None => Hash
 def empty_config
-  puts 'Job configuration not found or empty'
+  @sockets[:logger].write 'Job configuration not found or empty'
   {}
 end
 
@@ -43,10 +42,16 @@ def define(job_name)
   formatted config
 end
 
-puts 'Started'
-server = EZMQ::Server.new port: configatron.job_dictionary_port
-server.serialize_with_json!
-server.listen do |job_name|
-  puts "Looking up: #{job_name}"
+@sockets = {
+  server: EZMQ::Server.new(port: configatron.job_dictionary_port),
+  logger: EZMQ::Logger.new('job_dictionary')
+}
+@sockets.each do |key, socket|
+  socket.serialize_with_json! unless key == :logger
+end
+
+@sockets[:logger].write 'Started'
+@sockets[:server].listen do |job_name|
+  @sockets[:logger].write "Looking up: #{job_name}"
   define job_name
 end

data/bin/sweatshop-logger

@@ -0,0 +1,32 @@
+#!/usr/bin/env ruby
+require 'ezmq'
+require 'robot_sweatshop/config'
+require 'robot_sweatshop/connections'
+using ExtendedEZMQ
+
+@sockets = {
+  subscriber: EZMQ::Subscriber.new(:bind, port: configatron.logger_port, topic: 'robot-sweatshop-logging'),
+  reflector: EZMQ::Publisher.new(:bind, port: configatron.reflector_port)
+}
+@sockets.each do |key, socket|
+  socket.serialize_with_json! unless key == :logger
+end
+
+def write(text, for_process:)
+  log_file = File.expand_path "#{configatron.logfile_path}/#{for_process}.log"
+  File.open log_file, 'a' do |log|
+    log.write "[#{Time.now.utc}] #{text}\n"
+  end
+end
+
+def parse_worker_id_from(data)
+  process, worker_id = data[:process].split '-'
+  data.merge process: process, text: "[#{worker_id}] #{data[:text]}"
+end
+
+write 'Started', for_process: 'logger'
+@sockets[:subscriber].listen do |data, topic|
+  data = parse_worker_id_from data if data[:process] =~ /worker/
+  write data[:text], for_process: data[:process]
+  @sockets[:reflector].send data, topic: topic
+end

data/bin/sweatshop-overseer

@@ -3,7 +3,6 @@ require 'sinatra'
 require 'sinatra/cross_origin'
 require 'robot_sweatshop/config'
 require 'robot_sweatshop/overseer'
-
 helpers OverseerHelper
 
 configure do

data/bin/sweatshop-payload-parser

@@ -5,7 +5,6 @@ require 'contracts'
 require 'robot_sweatshop/config'
 require 'robot_sweatshop/payload'
 require 'robot_sweatshop/connections'
-$stdout.sync = true
 include Contracts
 using ExtendedEZMQ
 
@@ -57,11 +56,17 @@ def parse(raw_payload, format)
   formatted payload
 end
 
-puts 'Started'
-server = EZMQ::Server.new port: configatron.payload_parser_port
-server.serialize_with_json!
-server.listen do |request|
-  puts "Parsing: #{request}"
+@sockets = {
+  server: EZMQ::Server.new(port: configatron.payload_parser_port),
+  logger: EZMQ::Logger.new('payload_parser')
+}
+@sockets.each do |key, socket|
+  socket.serialize_with_json! unless key == :logger
+end
+
+@sockets[:logger].write 'Started'
+@sockets[:server].listen do |request|
+  @sockets[:logger].write "Parsing: #{request}"
   format = detect_format_of request
   parse request[:payload], format
 end

data/bin/sweatshop-worker

@@ -5,10 +5,8 @@ require 'fileutils'
 require 'contracts'
 require 'robot_sweatshop/config'
 require 'robot_sweatshop/connections'
-using ExtendedEZMQ
-$stdout.sync = true
-$stderr.sync = true
 include Contracts
+using ExtendedEZMQ
 
 # TODO: check existing worker ids. it'd be a problem to share a workspace
 @worker_id = ARGV[0] || "#{Faker::Name.first_name}"
@@ -16,7 +14,7 @@ include Contracts
 Contract Hash, Proc => Any
 def from_workspace(named:)
   workspace = "#{named}-#{@worker_id}"
-  puts "Workspace: #{workspace}"
+  @sockets[:logger].write "Workspace: #{workspace}"
   path = File.expand_path "#{configatron.workspace_path}/#{workspace}"
   FileUtils.mkpath path
   Dir.chdir(path) { yield if block_given? }
@@ -31,25 +29,25 @@ Contract String, Hash => Maybe[IO]
 def stream(command, with_context:)
   command = ensure_shell_for command
   IO.popen(with_context, command) do |stream|
-    puts stream.gets until stream.eof?
+    @sockets[:logger].write stream.gets until stream.eof?
   end
 end
 
 Contract String, Hash => nil
 def execute(command, context)
-  puts "Executing '#{command}'"
+  @sockets[:logger].write "Executing '#{command}'"
   begin
     stream command, with_context: context
   rescue Errno::ENOENT => error
-    puts error.message
+    @sockets[:logger].write error.message
   end
-  puts "Execution complete with exit status: #{$?.exitstatus}"
+  @sockets[:logger].write "Execution complete with exit status: #{$?.exitstatus}"
 end
 
 Contract Fixnum => nil
 def finish(id)
   @sockets[:conveyor].request({ method: 'finish', data: id }, {})
-  puts "Job finished.\n\n"
+  @sockets[:logger].write "Job finished.\n\n"
 end
 
 Contract Array, Hash => Any
@@ -65,15 +63,18 @@ def run_implicit_job
   run implicit_job['commands'], with_context: implicit_job['environment']
 end
 
-puts 'Starting'
 @sockets = {
   conveyor: EZMQ::Client.new(port: configatron.conveyor_port),
-  puller: EZMQ::Puller.new(:connect, port: configatron.worker_port)
+  puller: EZMQ::Puller.new(:connect, port: configatron.worker_port),
+  logger: EZMQ::Logger.new("worker-#{@worker_id}")
 }
-@sockets.each { |_key, socket| socket.serialize_with_json! }
+@sockets.each do |key, socket|
+  socket.serialize_with_json! unless key == :logger
+end
 
+@sockets[:logger].write 'Started'
 @sockets[:puller].listen do |data|
-  puts "Running: #{data}"
+  @sockets[:logger].write "Running: #{data}"
   from_workspace named: data[:job_name] do
     run data[:commands], with_context: data[:context]
     run_implicit_job

data/config.defaults.yaml

@@ -13,5 +13,7 @@ conveyor_port: 10556
 payload_parser_port: 10557
 job_dictionary_port: 10558
 worker_port: 10559
+logger_port: 10560
+reflector_port: 10561
 api_url: http://localhost:10555
 job_timeout_length: 300 # 5 minutes

data/lib/robot_sweatshop/connections.rb

@@ -16,3 +16,29 @@ module ExtendedEZMQ
     end
   end
 end
+
+# A logger that publishes what it writes
+module EZMQ
+  class Logger
+    using ExtendedEZMQ
+
+    def initialize(process)
+      @process = process
+      @logger = EZMQ::Publisher.new :connect, port: configatron.logger_port
+      @logger.serialize_with_json!
+      @user = `whoami`.chomp
+      @host = `hostname`.chomp
+    end
+
+    def write(text)
+      data = {
+        text: text,
+        process: @process,
+        user: @user,
+        host: @host
+      }
+      @logger.send data, topic: 'robot-sweatshop-logging'
+      nil
+    end
+  end
+end

data/robot_sweatshop.eye

@@ -4,7 +4,6 @@ require 'yaml'
 CONFIG_PATH = File.expand_path '~/.robot_sweatshop/compiled_config.yaml'
 CONFIG = YAML.load_file CONFIG_PATH
 PID_PATH = CONFIG[:pidfile_path]
-LOG_PATH = CONFIG[:logfile_path]
 
 Eye.config do
   logger "#{CONFIG[:logfile_path]}/eye.log"
@@ -18,19 +17,16 @@ Eye.application :robot_sweatshop do
   group 'services' do
     process :job_dictionary do
       pid_file "#{PID_PATH}/job-dictionary.pid"
-      stdall "#{LOG_PATH}/job-dictionary.log"
       start_command "#{__dir__}/bin/sweatshop-job-dictionary"
       daemonize true
     end
     process :payload_parser do
       pid_file "#{PID_PATH}/payload-parser.pid"
-      stdall "#{LOG_PATH}/payload-parser.log"
       start_command "#{__dir__}/bin/sweatshop-payload-parser"
       daemonize true
     end
     process :conveyor do
       pid_file "#{PID_PATH}/conveyor.pid"
-      stdall "#{LOG_PATH}/conveyor.log"
       start_command "#{__dir__}/bin/sweatshop-conveyor"
       daemonize true
     end
@@ -38,26 +34,27 @@ Eye.application :robot_sweatshop do
 
   process :overseer do
     pid_file "#{PID_PATH}/overseer.pid"
-    stdall "#{LOG_PATH}/overseer.log"
     start_command "#{__dir__}/bin/sweatshop-overseer"
     daemonize true
   end
   process :api do
     pid_file "#{PID_PATH}/api.pid"
-    stdall "#{LOG_PATH}/api.log"
     start_command "#{__dir__}/bin/sweatshop-api"
     daemonize true
   end
   process :assembler do
     pid_file "#{PID_PATH}/assembler.pid"
-    stdall "#{LOG_PATH}/assembler.log"
     start_command "#{__dir__}/bin/sweatshop-assembler"
     daemonize true
   end
   process :worker do
     pid_file "#{PID_PATH}/worker.pid"
-    stdall "#{LOG_PATH}/worker.log"
     start_command "#{__dir__}/bin/sweatshop-worker"
     daemonize true
   end
+  process :logger do
+    pid_file "#{PID_PATH}/logger.pid"
+    start_command "#{__dir__}/bin/sweatshop-logger"
+    daemonize true
+  end
 end

data/robot_sweatshop.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |gem|
   gem.name        = 'robot_sweatshop'
-  gem.version     = '0.4.12'
+  gem.version     = '0.4.13'
   gem.licenses    = 'MIT'
   gem.authors     = ['Justin Scott']
   gem.email       = 'jvscott@gmail.com'

data/test/all.rb

@@ -26,6 +26,7 @@ tests = %w(
   assembler
   worker
   overseer
+  logger
   end-to-end
 )
 exit_statuses = []

data/test/logger_spec.rb

@@ -0,0 +1,36 @@
+require 'kintama'
+require 'ezmq'
+require 'timeout'
+require 'robot_sweatshop/config'
+require 'robot_sweatshop/connections'
+require_relative 'shared/scaffolding'
+require_relative 'shared/helpers'
+
+Kintama.on_start do
+  @pids = TestProcess.start %w(logger)
+end
+
+Kintama.on_finish do
+  TestProcess.stop @pids
+end
+
+describe 'the Logger' do
+  given 'log data from a process' do
+    setup do
+      @custom_logger = TestProcess.stub :logger
+      EZMQ::Logger.new('test_spec').write 'success'
+      sleep $a_while # TODO: Timeout wasn't working for some reason
+                     #       Also breaks if this is in the `should` block :S
+    end
+
+    should 'log the data to file' do
+      log_file = File.expand_path "#{configatron.logfile_path}/test_spec.log"
+      assert_match /success/, File.read(log_file)
+    end
+
+    should 'reflect the published message' do
+      reflection = File.read @custom_logger.output_file
+      assert_match /success/, reflection
+    end
+  end
+end

data/test/overseer_spec.rb

@@ -7,7 +7,7 @@ require_relative 'shared/helpers'
 $stdout.sync = true
 
 Kintama.on_start do
-  @pids = TestProcess.start %w(overseer)
+  @pids = TestProcess.start %w(overseer logger)
   sleep $a_while
 end
 
@@ -23,7 +23,7 @@ given 'the Overseer' do
     should('respond') { assert_equal 200, @response.code }
     should('link to process logs') do
       page = Nokogiri::HTML(@response.to_s)
-      links = page.css('a').select { |link| link.text.include? 'overseer' }
+      links = page.css('a').select { |link| link.text.include? 'logger' }
       assert_not_equal 0, links.count
     end
     should('have a form for running jobs') do
@@ -36,12 +36,12 @@ given 'the Overseer' do
     setup { Timeout.timeout($a_while) { @response = HTTP.get overseer_url('log') } }
     should('redirect') { assert_equal 303, @response.code }
   end
-  context '/log?for=overseer' do
-    setup { Timeout.timeout($a_while) { @response = HTTP.get overseer_url('log?for=overseer') } }
+  context '/log?for=logger' do
+    setup { Timeout.timeout($a_while) { @response = HTTP.get overseer_url('log?for=logger') } }
     should('respond') { assert_equal 200, @response.code }
     should('show logs from file') do
       page = Nokogiri::HTML(@response.to_s)
-      log = File.read "#{configatron.logfile_path}/overseer.log"
+      log = File.read "#{configatron.logfile_path}/logger.log"
       output = page.css('.raw_log').first
       assert_equal log, output.text, 'Expected raw log output'
     end

data/test/shared/helpers/input.rb

@@ -34,8 +34,8 @@ module InputHelper
 
   def implicit_job
     job = {
-      'commands' => ['echo $this >> test.txt'],
-      'environment' => { 'this' => 'implicit'}
+      'commands' => ['echo implicit >> test.txt'],
+      'environment' => {}
     }
     YAML.dump job
   end

data/test/shared/scaffolding.rb

@@ -41,7 +41,8 @@ module TestProcess
   def self.stub(process_name)
     process = {
       conveyor: {type: 'Server', port: configatron.conveyor_port},
-      worker:   {type: 'Puller', port: configatron.worker_port}
+      worker:   {type: 'Puller', port: configatron.worker_port},
+      logger:   {type: 'Subscriber', port: configatron.reflector_port}
     }[process_name]
     Stub.new process[:type], on_port: process[:port]
   end

data/test/shared/stub.rb

@@ -29,6 +29,8 @@ class Stub
     listener = case type
     when 'Puller'
       EZMQ::Puller.new :connect, port: port
+    when 'Subscriber'
+      EZMQ::Subscriber.new port: port, topic: 'robot-sweatshop-logging'
     else
       EZMQ.const_get(type).new port: port
     end

data/test/worker_spec.rb

@@ -24,9 +24,13 @@ describe 'the Worker' do
 
   setup do
     @conveyor = TestProcess.stub :conveyor
+    @workspace_file = "#{configatron.workspace_path}/test_job-testingid/test.txt"
     @pusher = EZMQ::Pusher.new :bind, port: configatron.worker_port
     @pusher.serialize_with_json!
-    clear_worker_output
+    @pusher.send worker_push
+    Timeout.timeout($a_while) do
+      loop while @conveyor.output_empty?
+    end
   end
 
   teardown do
@@ -35,33 +39,29 @@ describe 'the Worker' do
 
   given 'valid job data is pushed' do
     setup do
-      @pusher.send worker_push
-      Timeout.timeout($a_while) do
-        loop until File.exist? worker_output
-        loop while @conveyor.output_empty?
-      end
-      @worker_data = eval File.read(@conveyor.output_file)
+      @logger_data = File.read @workspace_file
+      @conveyor_data = eval File.read(@conveyor.output_file)
     end
 
     should 'run the commands' do
-      assert_equal true, File.file?(worker_output)
+      assert_not_equal '', @logger_data
     end
 
     should 'run with the context as environment variables' do
-      assert_match /hello world/, File.read(worker_output)
+      assert_match /hello world/, @logger_data
     end
 
     should 'run with custom scripts in the path' do
-      assert_match /custom/, File.read(worker_output)
+      assert_match /custom/, @logger_data
     end
 
     should 'run implicit jobs' do
-      assert_match /implicit/, File.read(worker_output)
+      assert_match /implicit/, @logger_data
     end
 
     should 'tell the conveyor that job is complete' do
-      assert_equal 'finish', @worker_data[:method]
-      assert_kind_of Fixnum, @worker_data[:data]
+      assert_equal 'finish', @conveyor_data[:method]
+      assert_kind_of Fixnum, @conveyor_data[:data]
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: robot_sweatshop
 version: !ruby/object:Gem::Version
-  version: 0.4.12
+  version: 0.4.13
 platform: ruby
 authors:
 - Justin Scott
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-14 00:00:00.000000000 Z
+date: 2015-08-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: faker
@@ -270,6 +270,7 @@ executables:
 - sweatshop-assembler
 - sweatshop-conveyor
 - sweatshop-job-dictionary
+- sweatshop-logger
 - sweatshop-overseer
 - sweatshop-payload-parser
 - sweatshop-worker
@@ -289,6 +290,7 @@ files:
 - bin/sweatshop-assembler
 - bin/sweatshop-conveyor
 - bin/sweatshop-job-dictionary
+- bin/sweatshop-logger
 - bin/sweatshop-overseer
 - bin/sweatshop-payload-parser
 - bin/sweatshop-worker
@@ -331,6 +333,7 @@ files:
 - test/data/weird_job.yaml
 - test/end-to-end_spec.rb
 - test/job_dictionary_spec.rb
+- test/logger_spec.rb
 - test/overseer_spec.rb
 - test/payload_parser_spec.rb
 - test/shared/helpers.rb