gorgon 0.0.1

data/.gitignore

@@ -0,0 +1,8 @@
+.bundle
+*.swp
+.rvmrc
+tags
+TAGS
+pkg
+.idea
+.rbenv-version

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in gorgon.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,52 @@
+PATH
+  remote: .
+  specs:
+    gorgon (0.0.1)
+      amqp (~> 0.9.7)
+      awesome_print
+      bunny (~> 0.8.0)
+      colorize (~> 0.5.8)
+      open4 (~> 1.3.0)
+      ruby-progressbar (~> 1.0.1)
+      test-unit
+      uuidtools (~> 2.1.3)
+      yajl-ruby (~> 1.1.0)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    amq-client (0.9.4)
+      amq-protocol (>= 0.9.4)
+      eventmachine
+    amq-protocol (0.9.4)
+    amqp (0.9.7)
+      amq-client (~> 0.9.4)
+      amq-protocol (>= 0.9.4)
+      eventmachine
+    awesome_print (1.0.2)
+    bunny (0.8.0)
+    colorize (0.5.8)
+    diff-lcs (1.1.3)
+    eventmachine (1.0.0)
+    open4 (1.3.0)
+    rake (0.9.2.2)
+    rspec (2.8.0)
+      rspec-core (~> 2.8.0)
+      rspec-expectations (~> 2.8.0)
+      rspec-mocks (~> 2.8.0)
+    rspec-core (2.8.0)
+    rspec-expectations (2.8.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.8.0)
+    ruby-progressbar (1.0.1)
+    test-unit (2.5.2)
+    uuidtools (2.1.3)
+    yajl-ruby (1.1.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  gorgon!
+  rake
+  rspec

data/README.md

@@ -0,0 +1,53 @@
+Gorgon
+=====================
+
+About
+---------------------
+
+Gorgon provides a method for distributing the workload of running a ruby test suites. It relies on amqp for message passing, and rsync for the synchronization of source code.
+
+Usage
+---------------------
+
+To queue the current test suite, run `bundle exec gorgon start`, or `bundle exec gorgon`. _gorgon_ will read the application configuration out of _gorgon.json_, connect to the AMQP server, and publish the job.
+
+In order for the job to run, _gorgon job listeners_ must be started that can process the job. To start a gorgon listener, run `bundle exec gorgon listen`. This command will read the listener configuration out of _gorgon\_listener.json_, then start the listener process in the background.
+
+Configuration
+---------------------
+
+### gorgon.json
+This file contains project-specific settings for gorgon, such as:
+
+* A glob for generating the list of test files
+* The connection information for AMQP
+* Information about how clients can rsync the working directory
+* Files containing Ruby code to be used as callbacks
+
+### gorgon_listener.json
+This file contains the listener-specific settings, such as:
+
+* How many worker slots are provided by this listener
+* The connection information for AMQP
+* The file used for logs
+
+Architecture
+---------------------
+
+By running `bundle exec gorgon start`, the originating computer will publish a *job definition* to the AMQP server. This object contains all of the information required to run the tests:
+
+* The rsync information with which to fetch the source tree
+* The name of a AMQP queue that contains the list of files that require testing
+* The name of a AMQP exchange to send replies to
+* Application-specific setup/teardown, either per-job or per-worker [scheduled for post-alpha]
+
+The job listener subscribes to the job publish event, and maintains its own queue of jobs. When a job has available *worker slots*, it will prepare the workspace:
+
+* Create a unique temporary workspace directory for the job
+* Rsync the source tree to the temporary workspace
+* Run per-job application-specific setup [scheduled for post-alpha]
+* Invoke *n* workers, where *n* is the number of available *worker slots*.
+
+To invoke a job worker, the listener passes the name of the *file queue*, *reply queue*, and *listener queue* to the worker initialization. After all workers have been started, the listener will block until an event appears on the *listener queue*.
+
+The worker process will run any application-specific startup, start a test environment, and load a stub test file that dynamically pulls files out of the *file queue*. It runs the test, posts the results to the *reply queue*, and repeats until the *file queue* is empty. When the *file queue* becomes empty, the worker runs application-specific teardown, then reports its completion to the *listener queue*, and shuts down.

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/gorgon

@@ -0,0 +1,41 @@
+require "rubygems"
+require 'gorgon/originator'
+require 'gorgon/listener'
+require 'gorgon/worker_manager'
+
+def start
+  o = Originator.new
+  o.originate
+end
+
+def listen
+  l = Listener.new
+  l.listen
+end
+
+def manage_workers
+  config_path = ENV["GORGON_CONFIG_PATH"]
+
+  manager = WorkerManager.build config_path
+  manager.manage
+
+  # For some reason I have to 'exit' here, otherwise WorkerManager process crashes
+  exit
+end
+
+def usage
+  #print instructions on how to use gorgon
+end
+
+case ARGV[0]
+when nil
+  start
+when "start"
+  start
+when "listen"
+  listen
+when "manage_workers"
+  manage_workers
+else
+  usage
+end

data/gorgon.gemspec

@@ -0,0 +1,33 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "gorgon/version"
+
+Gem::Specification.new do |s|
+  s.name        = "gorgon"
+  s.version     = Gorgon::VERSION
+  s.authors     = ["Justin Fitzsimmons", "Sean Kirby", "Victor Savkin", "Clemens Park", "Arturo Pie"]
+  s.email       = ["justin@fitzsimmons.ca"]
+  s.homepage    = ""
+  s.summary     = %q{Distributed testing for ruby with centralized management}
+  s.description = %q{Gorgon provides a method for distributing the workload of running a ruby test suites. It relies on amqp for message passing, and rsync for the synchronization of source code.}
+
+  s.rubyforge_project = "gorgon"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "rake"
+
+  s.add_runtime_dependency "amqp", '~>0.9.7'
+  s.add_runtime_dependency "awesome_print"
+  s.add_runtime_dependency "open4", '~>1.3.0'
+  s.add_runtime_dependency "yajl-ruby", '~>1.1.0'
+  s.add_runtime_dependency "uuidtools", '~>2.1.3'
+  s.add_runtime_dependency "test-unit"
+  s.add_runtime_dependency "bunny", '~>0.8.0'
+  s.add_runtime_dependency "ruby-progressbar", '~>1.0.1'
+  s.add_runtime_dependency "colorize", '~>0.5.8'
+end

data/lib/gorgon.rb

@@ -0,0 +1,6 @@
+require "gorgon/version"
+require "gorgon/originator"
+require "gorgon/listener"
+
+module Gorgon
+end

data/lib/gorgon/amqp_service.rb

@@ -0,0 +1,39 @@
+require 'bunny'
+require 'yajl'
+
+class AmqpQueueDecorator
+  def initialize queue
+    @queue = queue
+  end
+
+  def pop
+    m = @queue.pop
+    p = m[:payload]
+    p == :queue_empty ? nil : p
+  end
+end
+
+class AmqpExchangeDecorator
+  def initialize exchange
+    @exchange = exchange
+  end
+
+  def publish msg
+    serialized_msg = Yajl::Encoder.encode(msg)
+    @exchange.publish serialized_msg
+  end
+end
+
+class AmqpService
+  def initialize connection_config
+    @connection_config = connection_config.merge(:spec => "09")
+  end
+
+  def start_worker file_queue_name, reply_exchange_name
+    Bunny.run @connection_config do |b|
+      queue = b.queue file_queue_name
+      exchange = b.exchange reply_exchange_name
+      yield AmqpQueueDecorator.new(queue), AmqpExchangeDecorator.new(exchange)
+    end
+  end
+end

data/lib/gorgon/callback_handler.rb

@@ -0,0 +1,21 @@
+class CallbackHandler
+  def initialize(config)
+    @config = config || {}
+  end
+
+  def before_start
+    load(@config[:before_start]) if @config[:before_start]
+  end
+
+  def after_complete
+    load(@config[:after_complete]) if @config[:after_complete]
+  end
+
+  def before_creating_workers
+    load(@config[:before_creating_workers]) if @config[:before_creating_workers]
+  end
+
+  def after_sync
+    load(@config[:after_sync]) if @config[:after_sync]
+  end
+end

data/lib/gorgon/configuration.rb

@@ -0,0 +1,9 @@
+require "yajl"
+
+module Configuration
+  extend self
+  def load_configuration_from_file(filename)
+    file = File.new(filename, "r")
+    Yajl::Parser.new(:symbolize_keys => true).parse(file)
+  end
+end

data/lib/gorgon/failures_printer.rb

@@ -0,0 +1,37 @@
+require 'yajl'
+
+class FailuresPrinter
+  OUTPUT_FILE = "/tmp/gorgon-failed-files.json"
+
+  def initialize job_state
+    @job_state = job_state
+    @job_state.add_observer(self)
+  end
+
+  def update payload
+    return unless @job_state.is_job_complete? || @job_state.is_job_cancelled?
+
+    File.open(OUTPUT_FILE, 'w+') do |fd|
+      fd.write(Yajl::Encoder.encode(failed_files + unfinished_files))
+    end
+  end
+
+  private
+
+  def failed_files
+    failed_files = []
+    @job_state.each_failed_test do |test|
+      failed_files << "#{test[:filename]}"
+    end
+    failed_files
+  end
+
+  def unfinished_files
+    unfinished_files = []
+    @job_state.each_running_file do |hostname, filename|
+      unfinished_files << "#{filename}"
+    end
+    unfinished_files
+  end
+end
+

data/lib/gorgon/g_logger.rb

@@ -0,0 +1,22 @@
+require "logger"
+
+module GLogger
+  def initialize_logger log_file
+    return unless log_file
+    @logger =
+      if log_file == "-"
+        Logger.new($stdout)
+      else
+        Logger.new(log_file)
+      end
+    @logger.datetime_format = "%Y-%m-%d %H:%M:%S "
+  end
+
+  def log text
+    @logger.info(text) if @logger
+  end
+
+  def log_error text
+    @logger.error(text) if @logger
+  end
+end

data/lib/gorgon/host_state.rb

@@ -0,0 +1,31 @@
+class HostState
+  def initialize
+    @running_workers = {}
+  end
+
+  def file_started worker_id, filename
+    if @running_workers.has_key? worker_id
+      puts "WARNING: worker #{worker_id} started running a new file, but a 'finish' message has not been received for file #{@running_workers[:filename]}"
+    end
+
+    @running_workers[worker_id] = filename
+  end
+
+  def file_finished worker_id, filename
+    if !@running_workers.has_key? worker_id || @running_workers[:worker_id] != filename
+      puts "WARNING: worker #{worker_id} finished running a file, but a 'start' message for that file was not received. File: #{filename}"
+    end
+
+    @running_workers.delete(worker_id)
+  end
+
+  def total_running_workers
+    @running_workers.size
+  end
+
+  def each_running_file
+    @running_workers.each_value do |filename|
+      yield filename
+    end
+  end
+end

data/lib/gorgon/job.rb

@@ -0,0 +1,26 @@
+class Job
+  def initialize(listener, job_definition)
+    @workers = []
+    @definition = job_definition
+  end
+
+
+
+  def add_worker
+
+  end
+
+  def on_worker_complete
+    @available_worker_slots += 1
+    on_current_job_complete if current_job_complete?
+  end
+
+  def setup_child_process
+    worker = ChildProcess.build("gorgon", "work", @worker_communication.name, @config_filename)
+
+    worker_output = Tempfile.new("gorgon-worker")
+    worker.io.stdout = worker_output
+    worker.io.stderr = worker_output
+    worker
+  end
+end

data/lib/gorgon/job_definition.rb

@@ -0,0 +1,24 @@
+require 'yajl'
+
+class JobDefinition
+  attr_accessor :file_queue_name, :reply_exchange_name, :source_tree_path, :sync_exclude, :callbacks
+
+  def initialize(opts={})
+    @file_queue_name = opts[:file_queue_name]
+    @reply_exchange_name = opts[:reply_exchange_name]
+    @source_tree_path = opts[:source_tree_path]
+    @callbacks = opts[:callbacks]
+    @sync_exclude = opts[:sync_exclude]
+  end
+
+  def to_json
+    Yajl::Encoder.encode(to_hash)
+  end
+
+  private
+
+  #This can probably be done with introspection somehow, but this is way easier despite being very verbose
+  def to_hash
+    {:file_queue_name => @file_queue_name, :reply_exchange_name => @reply_exchange_name, :source_tree_path => @source_tree_path, :sync_exclude => @sync_exclude, :callbacks => @callbacks}
+  end
+end

data/lib/gorgon/job_state.rb

@@ -0,0 +1,119 @@
+require 'gorgon/host_state'
+
+require 'observer'
+
+class JobState
+  include Observable
+
+  attr_reader :total_files, :remaining_files_count, :state
+
+  def initialize total_files
+    @total_files = total_files
+    @remaining_files_count = total_files
+    @failed_tests = []
+    @hosts = {}
+
+    if @remaining_files_count > 0
+      @state = :starting
+    else
+      @state = :complete
+    end
+  end
+
+  def failed_files_count
+    @failed_tests.count
+  end
+
+  def finished_files_count
+    total_files - remaining_files_count
+  end
+
+  def file_started payload
+    raise_if_completed_or_cancelled
+
+    if @state == :starting
+      @state = :running
+    end
+
+    file_started_update_host_state payload
+
+    changed
+    notify_observers payload
+  end
+
+  def file_finished payload
+    raise_if_completed_or_cancelled
+
+    @remaining_files_count -= 1
+    @state = :complete if @remaining_files_count == 0
+
+    handle_failed_test payload if failed_test?(payload)
+
+    @hosts[payload[:hostname]].file_finished payload[:worker_id], payload[:filename]
+
+    changed
+    notify_observers payload
+  end
+
+  def cancel
+    @remaining_files_count = 0
+    @state = :cancelled
+    changed
+    notify_observers({})
+  end
+
+  def each_failed_test
+    @failed_tests.each do |test|
+      yield test
+    end
+  end
+
+  def each_running_file
+    @hosts.each do |hostname, host|
+      host.each_running_file do |filename|
+        yield hostname, filename
+      end
+    end
+  end
+
+  def total_running_hosts
+    @hosts.size
+  end
+
+  def total_running_workers
+    result = 0
+    @hosts.each do |hostname, host|
+      result += host.total_running_workers
+    end
+    result
+  end
+
+  def is_job_complete?
+    @state == :complete
+  end
+
+  def is_job_cancelled?
+    @state == :cancelled
+  end
+
+  private
+
+  def file_started_update_host_state payload
+    hostname = payload[:hostname]
+    @hosts[hostname] = HostState.new if @hosts[hostname].nil?
+    @hosts[hostname].file_started payload[:worker_id], payload[:filename]
+  end
+
+  def handle_failed_test payload
+    @failed_tests << payload
+  end
+
+  def raise_if_completed_or_cancelled
+    raise "JobState#file_finished called when job was already complete" if is_job_complete?
+    raise "JobState#file_finished called after job was cancelled" if is_job_cancelled?
+  end
+
+  def failed_test? payload
+    payload[:type] == "fail"
+  end
+end