robot_sweatshop 0.4.6 → 0.4.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: acc02c9c05d562880ac2ae2856bcb5697395ca1f
-  data.tar.gz: df5ffe594c0e6c3f0af75d1a96b10fe7ef6ced63
+  metadata.gz: 7f1852d96c663c8d61e07123a2bf84162ba15698
+  data.tar.gz: 5b592ff4da66bb363625e9beeeb3dd0c64c8e85c
 SHA512:
-  metadata.gz: 4b442e68f7963fb3f9b681d445e4d88ba0b9940a7a556506a9bac03a031cf7853c6e3385e424ca28df2356164444a8aa4ecffa04ff7cedb83d0faed15bf26cfc
-  data.tar.gz: 482a8ccdb5cc3be30969a5a671ea4f82e0aed3dfb20f2979bfda1a127d015308b5b1da5d74eaa7c29d2aae21a331aeb64cafd46c5d6ba4fc14c5027ceab055d0
+  metadata.gz: ecdb8c16cbee4b20c667745c251953edc3850b59110ed850413a62c9454102c7455021c599af4b3b780f5acf3488d93b3d88c04d48877be6cd57b5c4a488aca8
+  data.tar.gz: ee78994a812f67a8623051e77f4252ea3c8b8f9746230861494e74f18bdb9a2718ab76dc4552b5eb296244993879b70b61980b74cba1e147310e683f651e459e

data/.rubocop.yml

@@ -7,7 +7,7 @@ Style/Encoding:
   Enabled: false
 
 Metrics/LineLength:
-  Max: 80
+  Max: 90
 
 Style/FileName:
   Enabled: false

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    robot_sweatshop (0.4.5)
+    robot_sweatshop (0.4.6)
       bundler
       commander
       configatron

data/README.md

@@ -40,13 +40,13 @@ You probably don't want to run Robot Sweatshop as a sudo user. Create a testing
 
 1.0
 
-- Docs on architecture
-- Easier way to run multiple workers
 - Mascot
 
 1.1
 
-- push/pull out node that worker streams output to
+- Easier way to run multiple workers
+- Push/pull out node that worker streams output to
+- Add worker tags to output so it can all go to one file
 
 Beyond
 

data/bin/sweatshop

@@ -7,7 +7,7 @@ require 'robot_sweatshop/config'
 require 'robot_sweatshop/create-config-directories'
 
 program :name, 'Robot Sweatshop'
-program :version, '0.4.6'
+program :version, '0.4.7'
 program :description, 'A lightweight, nonopinionated CI server'
 program :help, 'Author', 'Justin Scott <jvscott@gmail.com>'
 
@@ -16,8 +16,8 @@ command :job do |c|
   c.description = 'Creates and edits jobs.'
   c.option '--auto', 'Create the file without opening the editor.'
   c.action do |args, options|
-    options.default :auto => false
-    raise "Specify a job name as the command argument." if args.count < 1
+    options.default auto: false
+    fail 'Specify a job name as the command argument.' if args.count < 1
     job_file = CLI::Job.path_for args.first
     CLI.create job_file, with_contents: CLI::Job.default
     CLI.edit job_file unless options.auto
@@ -27,7 +27,7 @@ end
 command :'job-list' do |c|
   c.syntax = 'sweatshop job-list'
   c.description = 'Lists available job configurations.'
-  c.action do |args, options|
+  c.action do
     CLI::Job.list
   end
 end
@@ -37,7 +37,7 @@ command :config do |c|
   c.description = 'Creates and edits the user configuration file.'
   c.option '--auto', 'Create the file without opening the editor.'
   c.action do |args, options|
-    options.default :auto => false
+    options.default auto: false
     for_scope = args.first || 'local'
     config_file = CLI::Config.path for_scope
     CLI.create config_file, with_contents: CLI::Config.default
@@ -50,7 +50,7 @@ command :start do |c|
   c.description = 'Start the Sweatshop.'
   c.option '--testing', 'Load the testing Eye configuration.'
   c.action do |_args, options|
-    options.default :testing => false
+    options.default testing: false
     environment = options.testing ? 'testing' : 'production'
     CLI::Start.sweatshop for_environment: environment
   end

data/bin/sweatshop-assembler

@@ -12,8 +12,8 @@ using ExtendedEZMQ
 Contract Hash => Hash
 def sanitize(data)
   return {} if data.empty?
-  data = data.map { |key, value| {key => value.to_s} }
-  data.reduce(:merge)
+  data = data.map { |key, value| { key => value.to_s } }
+  data.reduce :merge
 end
 
 Contract Hash => String
@@ -40,11 +40,11 @@ def assemble(job, payload, definition)
   }
 end
 
-Contract None => Or[Hash,nil]
+Contract None => Maybe[Hash]
 def request_job
-  job_id = @sockets[:conveyor].request({method: 'dequeue'}, {})
+  job_id = @sockets[:conveyor].request method: 'dequeue'
   return nil if job_id.nil?
-  raw_job = @sockets[:conveyor].request({method: 'lookup', data: job_id}, {})
+  raw_job = @sockets[:conveyor].request method: 'lookup', data: job_id
   puts "Assembling: '#{raw_job}'"
   raw_job.merge job_id: job_id
 end
@@ -52,7 +52,7 @@ end
 Contract EZMQ::Socket, Any => Hash
 def request(socket, data)
   response = socket.request data, {}
-  raise response[:error] unless response[:error].empty?
+  fail response[:error] unless response[:error].empty?
   response[:data]
 end
 
@@ -63,19 +63,20 @@ def request_data_for(job)
   [payload, definition]
 end
 
-Contract Or[Array,nil], Or[String, nil] => nil
+Contract Maybe[Array], Maybe[String] => nil
 def check_whitelist(whitelist, branch)
   return if whitelist.nil?
-  raise 'Branch not whitelisted' unless whitelist.include? branch
+  fail 'Branch not whitelisted' unless whitelist.include? branch
 end
 
 Contract Fixnum => Bool
 def finish(job_id)
-  @sockets[:conveyor].request({method: 'finish', data: job_id}, {})
+  @sockets[:conveyor].request method: 'finish', data: job_id
 end
 
 job_search = Fiber.new do
-  # TODO: profiler to get a better idea of how long we should wait based on historical information
+  # TODO: profiler to get a better idea of how long we should
+  #       wait based on historical information
   timer = ExponentialBackoff.new 0.1, 3
   loop do
     sleep timer.next_interval
@@ -101,7 +102,7 @@ loop do
     payload, definition = request_data_for job
     check_whitelist definition['branch_whitelist'], payload['branch']
     assembled_job = assemble job, payload, definition
-    @sockets[:worker].send(assembled_job, {})
+    @sockets[:worker].send assembled_job
   rescue RuntimeError => error
     puts error.message
     finish job[:job_id]

data/bin/sweatshop-conveyor

@@ -21,9 +21,9 @@ def enqueue(item)
   @items.enqueue item
 end
 
-Contract None => Or[Fixnum,nil]
+Contract None => Maybe[Fixnum]
 def dequeue
-  puts "dequeue"
+  puts 'dequeue'
   @items.dequeue
 end
 
@@ -46,7 +46,6 @@ end
 
 Contract Hash => Any
 def complete(request)
-  arguments = []
   return send(request[:method], request[:data]) if request[:data]
   send(request[:method])
 end

data/bin/sweatshop-job-dictionary

@@ -16,12 +16,12 @@ end
 Contract String => Or[Hash, nil]
 def load_if_exists(config_path)
   puts "Reading job configuration from #{config_path}"
-  YAML.load_file config_path if File.exists? config_path
+  YAML.load_file config_path if File.exist? config_path
 end
 
 Contract None => Hash
 def empty_config
-  puts "Job configuration not found or empty"
+  puts 'Job configuration not found or empty'
   {}
 end
 
@@ -31,8 +31,8 @@ def load_config_for(job_name)
 end
 
 Contract Hash, Hash => Hash
-def formatted(payload={}, error:'')
-  {data: payload, error: error}
+def formatted(payload = {}, error: '')
+  { data: payload, error: error }
 end
 
 Contract String => Hash

data/bin/sweatshop-payload-parser

@@ -11,12 +11,10 @@ using ExtendedEZMQ
 
 Contract String => Bool
 def json?(string)
-  begin
-    Oj.load string
-    true
-  rescue Oj::ParseError => e
-    false
-  end
+  Oj.load string
+  true
+rescue Oj::ParseError
+  false
 end
 
 Contract String => Bool
@@ -40,16 +38,14 @@ end
 
 Contract String, String => Or[Hash, nil]
 def payload_hash_from(payload, format)
-  begin
-    Object.const_get("#{format}Payload").new(payload).to_hash
-  rescue NameError
-    nil
-  end
+  Object.const_get("#{format}Payload").new(payload).to_hash
+rescue NameError
+  nil
 end
 
 Contract Hash, Hash => Hash
-def formatted(payload={}, error:'')
-  {data: payload, error: error}
+def formatted(payload = {}, error:'')
+  { data: payload, error: error }
 end
 
 Contract String, String => Hash

data/bin/sweatshop-worker

@@ -30,10 +30,8 @@ end
 Contract Hash, String => Maybe[IO]
 def stream_command(context, command)
   command = ensure_shell_for command
-  IO.popen(context, command) do |io_stream|
-    while line = io_stream.gets
-      puts line
-    end
+  IO.popen(context, command) do |stream|
+    puts stream.gets until stream.eof? # TODO: stdout not being tested properly
   end
 end
 
@@ -50,7 +48,7 @@ end
 
 Contract Fixnum => nil
 def finish(id)
-  @sockets[:conveyor].request({method: 'finish', data: id}, {})
+  @sockets[:conveyor].request({ method: 'finish', data: id }, {})
   puts "Job finished.\n\n"
 end
 

data/docs/README.md

@@ -0,0 +1,71 @@
+# Process Architecture
+
+![Robot Sweatshop Processes](architecture.gif)
+
+# Message Data Schema
+
+## input
+
+```
+PUSH {
+  payload Hash
+  user_agent String
+  job_name String
+}
+```
+
+## conveyor
+
+```
+REQ {
+  method String
+  data String
+}
+```
+
+method | parameters | reply
+-------|------------|-------
+enqueue|item        |id Fixnum
+dequeue|-           |id Fixnum
+lookup |id          |item Hash
+finish |id          |true Bool
+
+## payload-parser
+
+```
+REQ {
+  payload String
+  user_agent String
+}
+
+REP {
+  payload Hash
+}
+```
+
+## job-dictionary
+
+```
+REQ {
+  job_name String
+}
+
+REP {
+  definition Hash
+}
+```
+
+## assembler
+
+```
+PUSH {
+  context Hash
+  commands Array
+  job_name String
+  job_id Fixnum
+}
+```
+
+# Notes
+
+Context is passed around with string keys because it's user provided. Everything else is passed with symbol keys

data/docs/architecture.dot

@@ -0,0 +1,22 @@
+digraph architecture {
+  label="Robot Sweatshop Processes"
+  labelloc="top"
+
+  I [label="Input"]
+  A [label="Assembler"]
+  W [label="Worker"]
+  subgraph cluster_0 {
+    label="Req/Rep Services"
+    labelloc="bottom"
+    style="dashed"
+    C [label="Conveyor"]
+    P [label="Payload Parser"]
+    D [label="Job Dictionary"]
+  }
+
+  I->C [taillabel="1..*"]
+  A->C [dir="both"]
+  A->P [dir="both"]
+  A->D [dir="both"]
+  A->W [headlabel="1..*",labeldistance=2]
+}

data/lib/robot_sweatshop/cli/common.rb

@@ -2,6 +2,7 @@ require 'fileutils'
 require 'robot_sweatshop/config'
 require 'terminal-announce'
 
+# Methods common to every CLI command
 module CLI
   def self.edit(file)
     editor = ENV['EDITOR']
@@ -9,8 +10,8 @@ module CLI
       Announce.info "Manually editing file '#{file}'"
       system editor, file
     else
-      Announce.warning "No editor specified in $EDITOR environment variable"
-      Announce.info "Displaying file contents instead"
+      Announce.warning 'No editor specified in $EDITOR environment variable'
+      Announce.info 'Displaying file contents instead'
       system 'cat', file
     end
   end
@@ -18,7 +19,7 @@ module CLI
   def self.create(file, with_contents: '')
     file = File.expand_path file
     if File.file?(file)
-    Announce.info "'#{file}' already exists"
+      Announce.info "'#{file}' already exists"
     else
       FileUtils.mkdir_p File.dirname(file)
       File.write file, with_contents

data/lib/robot_sweatshop/cli/config.rb

@@ -1,6 +1,8 @@
+require 'fileutils'
 require 'robot_sweatshop/config'
 
 module CLI
+  # Methods for configuring Robot Sweatshop
   module Config
     def self.default
       File.read "#{__dir__}/../../../config.defaults.yaml"
@@ -9,12 +11,30 @@ module CLI
     def self.path(scope)
       case scope
       when 'system'
-        "/etc/robot_sweatshop/config.yaml"
+        '/etc/robot_sweatshop/config.yaml'
       when 'user'
-        "~/.robot_sweatshop/config.yaml"
+        '~/.robot_sweatshop/config.yaml'
       else
-        ".robot_sweatshop/config.yaml"
+        '.robot_sweatshop/config.yaml'
       end
     end
+
+    def self.expand_paths_in(config_hash)
+      config_hash.each do |key, value|
+        config_hash[key] = File.expand_path value if key.to_s.match(/_path/)
+      end
+    end
+
+    def self.write(config_hash)
+      config_home = File.expand_path('~/.robot_sweatshop')
+      FileUtils.mkpath config_home
+      File.write "#{config_home}/compiled_config.yaml", config_hash.to_yaml
+    end
+
+    def self.compile_to_file
+      config = expand_paths_in configatron.to_h
+      config[:working_path] = Dir.pwd
+      write config
+    end
   end
 end

data/lib/robot_sweatshop/cli/job.rb

@@ -1,9 +1,19 @@
 require 'robot_sweatshop/config'
 
 module CLI
+  # Methods for creating and editing jobs
   module Job
     def self.default
-      "---\n# branch_whitelist:\n# - master\n\ncommands:\n- echo \"Hello $WORLD!\"\n\nenvironment:\n  WORLD: Earth\n"
+      "---
+      # branch_whitelist:
+      # - master
+
+      commands:
+      - echo \"Hello $WORLD!\"
+
+      environment:
+        WORLD: Earth
+      "
     end
 
     def self.path_for(job)

data/lib/robot_sweatshop/cli/start.rb

@@ -3,21 +3,13 @@ require 'terminal-announce'
 require 'robot_sweatshop/config'
 
 module CLI
+  # Methods for starting Robot Sweatshop
   module Start
-    def self.store_config_for_eye
-      config = configatron.to_h
-      config = config.each do |key, value|
-        config[key] = File.expand_path value if key.to_s.match /_path/
-      end
-      config[:working_path] = Dir.pwd
-      File.write '/tmp/.robot_sweatshop-eye-config.yaml', config.to_yaml
-    end
-
     def self.sweatshop(for_environment:)
-      store_config_for_eye
+      Config.compile_to_file
       eye_config = File.expand_path "#{__dir__}/../../../robot_sweatshop.eye"
       output = `eye load #{eye_config}`
-      raise output if $?.exitstatus != 0
+      fail output if $?.exitstatus != 0
       Announce.success "Robot Sweatshop loaded with a #{for_environment} configuration"
       Announce.info `eye restart robot_sweatshop`
       Announce.info 'Run \'eye --help\' for more info on debugging'

data/lib/robot_sweatshop/config.rb

@@ -5,9 +5,9 @@ configatron.reset!
 
 configurations = [
   "#{__dir__}/../../config.defaults.yaml",
-  "/etc/robot_sweatshop/config.yaml",
-  "~/.robot_sweatshop/config.yaml",
-  ".robot_sweatshop/config.yaml",
+  '/etc/robot_sweatshop/config.yaml',
+  '~/.robot_sweatshop/config.yaml',
+  '.robot_sweatshop/config.yaml'
 ]
 
 configurations.each do |config_path|

data/lib/robot_sweatshop/connections.rb

@@ -2,6 +2,7 @@ require 'ezmq'
 require 'oj'
 require 'robot_sweatshop/config'
 
+# Add some common methods to ZMQ sockets that get repeated everywhere
 module ExtendedEZMQ
   refine EZMQ::Socket do
     def serialize_with_json!
@@ -10,8 +11,8 @@ module ExtendedEZMQ
     end
 
     def close
-      self.socket.close
-      self.context.terminate
+      socket.close
+      context.terminate
     end
   end
 end

data/lib/robot_sweatshop/create-config-directories.rb

@@ -1,19 +1,12 @@
 require 'fileutils'
 
 def create_path(path)
-  begin
-    FileUtils.mkdir_p path
-  rescue Errno::EACCES
-    puts "Permission denied to create '#{path}'"
-  end
+  FileUtils.mkdir_p path
+rescue Errno::EACCES
+  puts "Permission denied to create '#{path}'"
 end
 
 config = configatron.to_h
 config.each do |key, value|
-  if key.to_s.match /_path/
-    create_path File.expand_path(value)
-  end
-end
-%w(pidfile_path logfile_path).each do |path|
-  create_path File.expand_path("#{configatron[path]}/gears")
+  create_path File.expand_path(value) if key.to_s.match(/_path/)
 end

data/robot_sweatshop.eye

@@ -1,7 +1,8 @@
 require 'yaml'
 
 # TODO: per-user temp file
-CONFIG = YAML.load_file '/tmp/.robot_sweatshop-eye-config.yaml'
+CONFIG_PATH = File.expand_path '~/.robot_sweatshop/compiled_config.yaml'
+CONFIG = YAML.load_file CONFIG_PATH
 PID_PATH = CONFIG[:pidfile_path]
 LOG_PATH = CONFIG[:logfile_path]
 

data/robot_sweatshop.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |gem|
   gem.name        = 'robot_sweatshop'
-  gem.version     = '0.4.6'
+  gem.version     = '0.4.7'
   gem.licenses    = 'MIT'
   gem.authors     = ['Justin Scott']
   gem.email       = 'jvscott@gmail.com'

data/test/shared/helpers.rb

@@ -1,5 +1,5 @@
 require_relative 'helpers/input'
 require_relative 'helpers/output'
 
-$a_moment = 0.1
+$a_moment = 0.2
 $a_while = 1

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: robot_sweatshop
 version: !ruby/object:Gem::Version
-  version: 0.4.6
+  version: 0.4.7
 platform: ruby
 authors:
 - Justin Scott
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-05 00:00:00.000000000 Z
+date: 2015-06-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -276,7 +276,9 @@ files:
 - bin/sweatshop-payload-parser
 - bin/sweatshop-worker
 - config.defaults.yaml
-- lib/README.md
+- docs/README.md
+- docs/architecture.dot
+- docs/architecture.gif
 - lib/robot_sweatshop.rb
 - lib/robot_sweatshop/cli.rb
 - lib/robot_sweatshop/cli/common.rb

data/lib/README.md

@@ -1,45 +0,0 @@
-# Job Lifecycle
-
-```
-input -> conveyor
-{ payload:, user_agent:, job_name: }
-
-assembler <-> conveyor
-{ method:, data: } <-> data
-
-assembler <-> payload-parser
-{ payload:, user_agent: } <-> { payload:, error: }
-
-assembler <-> job-dictionary
-{ job_name: } <-> { payload:, error: }
-
-assembler -> worker
-{ context:, commands:, job_name:, job_id }
-```
-
-context is passed around with string keys because it's user provided
-everything else is passed with symbol keys
-
-# Services
-
-sweatshop-conveyor
-```
-{ method:, data: }
-
-id = enqueue(item)
-id = dequeue
-item = lookup(id)
-something = finish(id)
-```
-
-sweatshop-payload-parser
-```
-req: { payload:, user_agent: }
-rep: { data:, error: }
-```
-
-sweatshop-job-dictionary
-```
-req: { job_name: }
-rep: { data:, error: }
-```