robot_sweatshop 0.1.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1eec24a1025b81640f28412b5601d5999e56040f
+  data.tar.gz: 9e3efeb22f6e15c8ec7dcf8c60760901eaf8be34
+SHA512:
+  metadata.gz: 72eb1ee064569d102f7ac6f1e0a2b772dea5bd790b52f963793ca161bca22fb5a5a5edbad0082e9ffc7fe9d0a9378269ad55b943632b02d5018a889326fd9ef9
+  data.tar.gz: 6d0680dd1d160940db79f436240451489e843f8c0af4a127e00d38ec4413db15e2f57ba3e81e89f3fc8fc4cf2be75ef254a6412ed7065267e58253d4c1986849

data/.gitignore

@@ -0,0 +1,6 @@
+config.user.yaml
+*.gem
+jobs/*.yaml
+workspaces/**/*
+!workspaces/.keep
+Gemfile.lock

data/.rubocop.yml

@@ -0,0 +1,12 @@
+AllCops:
+  Exclude:
+    - kintama/**/*
+
+Style/Encoding:
+  Enabled: false
+
+Metrics/LineLength:
+  Max: 80
+
+Style/FileName:
+  Enabled: false

data/Gemfile

@@ -0,0 +1,2 @@
+source 'http://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Justin Scott
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,45 @@
+# Robot Sweatshop
+
+[Jenkins](http://jenkins-ci.org/) is horrible to maintain and is problematic when automating installation and configuration. [Drone](https://drone.io/) assumes that you use Docker. [Travis-CI](https://travis-ci.org/recent) is difficult to self-host. All of these frameworks are highly opinionated in one way or another, forcing you to do things _their_ way.
+
+Robot Sweatshop is a single-purpose CI server that runs collections of arbitrary scripts when it needs to, usually when new code is pushed. There's no assumptions about what you want to report, what front-end you need, or even what repositories you want to clone because you can do that better than I can. It's just you, your code, and the scripts that test and deploy it.
+
+# Usage
+
+```
+gem install robot_sweatshop
+sweatshop start
+```
+
+Robot Sweatshop uses Eye to handle its services and will set up and configure everything for you.
+
+After configuring a job, POST a payload to `localhost:8080/:format/payload-for/:job`. For example, triggering a Bitbucket Git POST hook on `localhost:8080/bitbucket/payload-for/example` will parse the payload and run the 'example' job with the payload data in the environment.
+
+You can see what jobs are available with `sweatshop job --list`.
+
+Currently supported formats:
+
+- bitbucket
+
+# Configuration
+
+The server isn't much help without a job to run. Run `sudo -E sweatshop job <name>` to create a new job or edit an existing one.
+
+You can also use `sudo -E sweatshop config` to create and edit a user configuration at `/etc/robot_sweatshop/config.yaml`.
+
+Not sure if your job is valid? Run `sweatshop job --inspection <name>` to see if there's something you overlooked.
+
+# Security
+
+_TODO: Support for running as a custom user via eye uid/gid_
+
+# Roadmap
+
+- Enable Github support
+- CLI job running
+- Common scrips such as git repo syncing
+- Support for multiple workers
+- Better logging for the processes
+- Improved architecture:
+
+![Improved architecture diagram](http://40.media.tumblr.com/8a5b6ca59c0d93c4ce6fc6b733932a5f/tumblr_nko478zp9N1qh941oo1_1280.jpg)

data/Rakefile

@@ -0,0 +1,3 @@
+task :test do
+  require_relative 'kintama/run_all'
+end

data/bin/lib/common.rb

@@ -0,0 +1,49 @@
+require 'fileutils'
+
+def notify(type = :success, string)
+  color = case type
+  when :success
+    :green
+  when :failure
+    :red
+  when :warning
+    :yellow
+  when :info
+    :light_blue
+  else
+    ''
+  end
+  puts "[#{type.to_s.capitalize.colorize(color)}] #{string}"
+end
+
+def get_job_file(for_job:)
+  for_job = "#{__dir__}/../../jobs/#{for_job}.yaml"
+  if for_job.nil?
+    notify :failure, 'Please specify the job to create or edit. See --help for details'
+    nil
+  else
+    File.expand_path for_job
+  end
+end
+
+def edit(file)
+  edit = ENV['EDITOR']
+  if edit
+    notify :info, "Manually editing file '#{file}'"
+    system edit, file
+  else
+    notify :failure, "No editor specified in $EDITOR environment variable"
+    notify :info, "Displaying file contents instead"
+    system 'cat', file
+  end
+end
+
+def create_and_edit(file, with_default: '')
+  file = File.expand_path file
+  FileUtils.mkdir_p File.dirname(file)
+  unless File.file?(file)
+    File.write file, with_default
+    notify :success, "Created new file '#{file}'"
+  end
+  edit file
+end

data/bin/lib/config.rb

@@ -0,0 +1,20 @@
+require 'yaml'
+require 'fileutils'
+require_relative '../../config'
+
+def default_config
+  File.read "#{__dir__}/../../config.yaml"
+end
+
+def user_config_file
+  '/etc/robot_sweatshop/config.yaml'
+end
+
+def remove_user_config
+  FileUtils.rm_rf user_config_file
+  notify :success, 'Removing current user config file'
+end
+
+def create_and_edit_user_config
+  create_and_edit user_config_file, with_default: default_config
+end

data/bin/lib/inspect.rb

@@ -0,0 +1,42 @@
+def valid_lists?(job)
+  errors = false
+  %w(branch_whitelist commands).each do |list|
+    if job[list].nil? || job[list].empty?
+      notify :failure, "#{list} empty or not found"
+      errors = true
+    end
+  end
+  errors
+end
+
+def valid_environment?(job)
+  errors = false
+  job['environment'].each do |key, value|
+    unless value.is_a? String
+      notify :warning, "Non-string value for '#{key}'"
+      errors = true
+    end
+  end
+  errors
+end
+
+def valid_yaml?(job)
+  if job
+    true
+  else
+    notify :failure, "Invalid YAML"
+    false
+  end
+end
+
+def validate(job_file)
+  unless File.file?(job_file)
+    notify :failure, 'Job not found. Create it with \'workshop job\''
+  else
+    job = YAML.load_file job_file
+    return unless valid_yaml?(job)
+    errors = valid_lists?(job) |
+             valid_environment?(job)
+    notify :success, 'Valid job configuration' unless errors
+  end
+end

data/bin/lib/job.rb

@@ -0,0 +1,13 @@
+require_relative 'common'
+
+def empty_job
+  "---\nbranch_whitelist:\n- master\n\ncommands:\n- echo 'Hello $WORLD!'\n\nenvironment:\n  WORLD: Earth\n"
+end
+
+def create_and_edit_job(job_path)
+  create_and_edit job_path, with_default: empty_job
+end
+
+def list_jobs
+  puts Dir.glob("#{__dir__}/../../jobs/*").map { |file| File.basename(file, '.yaml') }
+end

data/bin/lib/start.rb

@@ -0,0 +1,11 @@
+def start_sweatshop(for_environment:)
+  eye_config = "#{__dir__}/../../robot_sweatshop.#{for_environment}.eye"
+  output = `eye load #{eye_config}`
+  if $?.exitstatus != 0
+    notify :failure, output
+  else
+    notify :success, "Robot Sweatshop loaded with a #{for_environment} configuration"
+    notify :info, `eye restart robot_sweatshop`
+    puts 'Check \'eye --help\' for more info on debugging'
+  end
+end

data/bin/sweatshop

@@ -0,0 +1,73 @@
+#!/usr/bin/env ruby
+require 'yaml'
+require 'commander/import'
+require 'colorize'
+require_relative 'lib/common'
+require_relative 'lib/job'
+require_relative 'lib/inspect'
+require_relative 'lib/start'
+require_relative 'lib/config'
+require_relative '../config'
+
+program :name, 'Robot Sweatshop'
+program :version, '0.1.1'
+program :description, 'A lightweight, unopinionated CI server'
+program :help, 'Author', 'Justin Scott <jvscott@gmail.com>'
+
+command :job do |c|
+  c.syntax = 'sweatshop job <name>'
+  c.description = 'Creates and edits jobs.'
+  c.option '--list', 'List all available jobs.'
+  c.option '--inspection', 'Inspect the given job.'
+  c.action do |args, options|
+    options.default :list => false
+    options.default :inspect => false
+    
+    job_file = get_job_file for_job: args.first
+    if options.list
+      list_jobs
+    elsif options.inspection
+      validate job_file unless job_file.nil?
+    else
+      create_and_edit_job job_file unless job_file.nil?
+    end
+  end
+end
+
+command :start do |c|
+  c.syntax = 'sweatshop start [options]'
+  c.description = 'Start the Sweatshop.'
+  c.option '--testing', 'Load the testing Eye configuration.'
+  c.action do |args, options|
+    options.default :testing => false
+    environment = options.testing ? 'testing' : 'production'
+    start_sweatshop for_environment: environment
+  end
+end
+
+command :stop do |c|
+  c.syntax = 'sweatshop stop'
+  c.description = 'Stop the Sweatshop.'
+  c.action do
+    notify :info, `eye stop robot_sweatshop`
+  end
+end
+
+command :directory do |c|
+  c.syntax = 'sweatshop directory'
+  c.description = 'Return the path that Robot Sweatshop runs from.'
+  c.action do
+    puts File.expand_path("#{__dir__}/..")
+  end
+end
+
+command :config do |c|
+  c.syntax = 'sweatshop config [options]'
+  c.description = 'Creates and edits the user configuration file.'
+  c.option '--reset', 'Reset to factory defaults before editing.'
+  c.action do |args, options|
+    options.default :reset => false
+    remove_user_config if options.reset
+    create_and_edit_user_config
+  end
+end

data/config.rb

@@ -0,0 +1,24 @@
+require 'fileutils'
+require 'configatron'
+require 'yaml'
+
+configurations = [
+  "#{__dir__}/config.yaml",
+  "#{__dir__}/config.user.yaml",
+  "/etc/robot_sweatshop/config.yaml"
+]
+configurations.each do |config_path|
+  if File.file? config_path
+    hash = YAML.load_file config_path
+    configatron.configure_from_hash hash
+  end
+end
+
+config_directories = [
+  configatron.common.logfile_directory,
+  configatron.common.pidfile_directory,
+  configatron.queue.moneta_directory
+]
+config_directories.each do |directory|
+  FileUtils.mkdir_p directory unless Dir.exists? directory
+end

data/config.yaml

@@ -0,0 +1,12 @@
+common:
+  pidfile_directory: /var/run/robot_sweatshop
+  logfile_directory: /var/log/robot_sweatshop
+  user_config_directory: /etc/robot_sweatshop
+
+input:
+  http:
+    port: 8080
+    bind: 0.0.0.0
+
+queue:
+  moneta_directory: /var/db/robot_sweatshop

data/jobs/example.yaml

@@ -0,0 +1,10 @@
+---
+branch_whitelist:
+- develop
+
+commands:
+- echo $custom_env > test.txt
+
+environment:
+  custom_env: success
+  converts_to_string: { string: only }

data/kintama/README.md

@@ -0,0 +1,3 @@
+You can run individual `_spec.rb` files with Ruby or `run_all.rb` to go through each.
+
+_Warning_: You must run http-in manually. Sinatra is a pain to start and stop programmatically. All other processes will be started by the helper.

data/kintama/data/payload_data.yaml

@@ -0,0 +1,6 @@
+---
+# http://www.url-encode-decode.com/
+malformed: |
+  payload=%7B%22nothing%22%3A%22here%22%7D
+bitbucket: |
+  payload=%7B%22repository%22%3A+%7B%22website%22%3A+%22%22%2C+%22fork%22%3A+false%2C+%22name%22%3A+%22Project+X%22%2C+%22scm%22%3A+%22git%22%2C+%22owner%22%3A+%22marcus%22%2C+%22absolute_url%22%3A+%22%2Fmarcus%2Fproject-x%2F%22%2C+%22slug%22%3A+%22project-x%22%2C+%22is_private%22%3A+true%7D%2C+%22truncated%22%3A+false%2C+%22commits%22%3A+%5B%7B%22node%22%3A+%22620ade18607a%22%2C+%22files%22%3A+%5B%7B%22type%22%3A+%22modified%22%2C+%22file%22%3A+%22README%22%7D%5D%2C+%22raw_author%22%3A+%22Marcus+Bertrand+%3Cmarcus%40somedomain.com%3E%22%2C+%22utctimestamp%22%3A+%222014-08-08+00%3A04%3A07%2B00%3A00%22%2C+%22author%22%3A+%22marcus%22%2C+%22timestamp%22%3A+%222014-08-08+02%3A04%3A07%22%2C+%22raw_node%22%3A+%22620ade18607ac42d872b568bb92acaa9a28620e9%22%2C+%22parents%22%3A+%5B%22702c70160afc%22%5D%2C+%22branch%22%3A+%22develop%22%2C+%22message%22%3A+%22Added+some+more+things+to+somefile.py%5Cn%22%2C+%22revision%22%3A+null%2C+%22size%22%3A+-1%7D%5D%2C+%22canon_url%22%3A+%22https%3A%2F%2Fbitbucket.org%22%2C+%22user%22%3A+%22marcus%22%7D

data/kintama/end-to-end_spec.rb

@@ -0,0 +1,30 @@
+require 'kintama'
+require 'ezmq'
+require 'http'
+require_relative 'shared/process_spawning'
+require_relative 'shared/helpers'
+
+describe 'Robot Sweatshop' do
+  include QueueHelper
+  include InHelper
+  include JobHelper
+
+  setup do
+    @client = EZMQ::Client.new port: 5556
+    @job_name = 'example'
+    @test_file = reset_test_file
+    clear_all_queues
+  end
+
+  context "POST git data to the HTTP Input" do
+    setup do
+      url = input_http_url for_job: @job_name
+      HTTP.post url, body: load_payload('bitbucket')
+      sleep $for_everything
+    end
+
+    should 'run jobs with the context as environment variables' do
+      assert_equal "success\n", File.read(@test_file)
+    end
+  end
+end

data/kintama/input_http_spec.rb

@@ -0,0 +1,45 @@
+require 'kintama'
+require 'ezmq'
+require 'json'
+require 'timeout'
+require 'http'
+require_relative 'shared/process_spawning'
+require_relative 'shared/helpers'
+
+given 'the HTTP Input' do
+  include QueueHelper
+  include InHelper
+
+  setup do
+    @subscriber = EZMQ::Subscriber.new port: 5557, topic: 'busy-queues'
+    @client = EZMQ::Client.new port: 5556
+    @job_name = 'example'
+    @raw_payload_queue = 'raw-payload'
+    clear_all_queues
+  end
+  
+  %w(Bitbucket).each do |git_host|
+    context "POST data from #{git_host}" do
+      setup do
+        url = input_http_url for_job: @job_name, in_format: git_host
+        HTTP.post url, body: load_payload(git_host)
+      end
+
+      should 'enqueue to \'raw-payload\'' do
+        Timeout.timeout($for_a_moment) do
+          @subscriber.listen do |message, topic|
+            break if message == @raw_payload_queue
+          end
+        end
+      end
+
+      should 'enqueue payload details' do
+        response = @client.request "mirror-#{@raw_payload_queue}"
+        data = JSON.parse response
+        %w(payload format job_name).each do |type|
+          assert_kind_of String, data[type]
+        end
+      end
+    end
+  end
+end