gorgon 0.7.1 → 0.8.0

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    ODFiMzc4NjIzZTU3NmVjYjUyZDA0NDk0MWEzYTQ4MTg5MzZmZjBlZg==
-  data.tar.gz: !binary |-
-    YzljZGEyNTE5ZTU3OWFiYjhiNTdhMDU2YTkyMzg1NjU4YjA4MzI1Zg==
+SHA1:
+  metadata.gz: 2697c5404498b1f1605746aa77f4812b3c106538
+  data.tar.gz: 8785ea9e49a5994a35e83f08b65106cfbc9369a4
 SHA512:
-  metadata.gz: !binary |-
-    ZWY1MWNmOWIwZjM3NjE4YTc5MTFiNjQzOWY0NmU2Y2FmMTUxOGJlZmMyZDky
-    YmQ3NGRjNzM1MTMwNTY2Zjg5YzhmZWYxNDJiMTEzZWYyNmQ1ZGMxYzI5ZTAx
-    NzkzMDA4N2U3MDc3NzJhYmMzNDk4NzE1YWEwNDNlNTZlZTBjMzM=
-  data.tar.gz: !binary |-
-    NjZiMTAyYTIxZjRjYzcxNmRhOTM0NmYwNmY2ZjU3MmU5MzdjMGE4Mjc4ZWUw
-    MzQ1M2ZmZGQxYjM1ZjI1NzAyMGMyNmI1NjFhNDBhZDE1OGEyNmFlNDllZWE4
-    NTJjOTY0ODhiYWYyZWZhYjZkODljYjZhMzczZWM5MWEwNjVhODY=
+  metadata.gz: b407fe9b9ccff0e2f6eefe038dff6bae1b71ab323622621fa98dbff3b92bb5ac55188990fdd7182b01349e7f6c13136f2c6b3ec1bdd1a76ddf214d0bce362361
+  data.tar.gz: e2846f05b7d0cd1213261356973a756dc75ee507a9c24e80e67bcbf3e22687ab59c15e8dd0449a07dcc9b27c3a9e3a8d319d7874cb54a33650adf94afe9c6144

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gorgon (0.7.1)
+    gorgon (0.8.0)
       amqp (~> 1.1.0)
       awesome_print
       colorize (~> 0.5.8)

data/README.md

@@ -11,34 +11,20 @@ Gorgon provides a method for distributing the workload of running ruby test suit
 
 Installing Gorgon
 -----------------
-1. `sudo apt-get install rabbitmq-server`
-1. cd to your project
-1. `gem install gorgon`
-1. if using rails, `gorgon init rails` will create initial files for a typical rails project. Otherwise, you can use `gorgon init`
-1. check gorgon.json to see and modify any necessary setting
-1. add the following lines to your _database.yml_ file
-
-```yaml
-remote_test: &remote_test
-  <<: *defaults
-  database: <my-app>_remote_test_<%=ENV['TEST_ENV_NUMBER']%>
-  min_messages: warning
-```
-
-Where `<<: *defaults` are the default values used in _database.yml_, like for example, adapter, username, password, and host. Replace `<my-app>` with a name to identify this application's dbs
+This [tutorial](/tutorial.md) explains how to install gorgon in a sample app. 
 
 Installing listener as a Daemon process (Ubuntu 9.10 or later)
 ----------------------------------------------------------------
 1. run `gorgon install_listener` from the directory where gorgon.json is
 1. run `gorgon ping` to check if the listener is running
 
-Try it out!
------------
-1. run `gorgon` to run all the tests.
+Gotchas
+----------------------------------------------------------------
 
-**NOTE:** if you get `cannot load such file -- qrack/qrack (LoadError)`, just add `gem 'gorgon', '~> 0.4.1' , :group => :remote_test` to your Gemfile, and run tests using `bundle exec gorgon`
+* if you get `cannot load such file -- qrack/qrack (LoadError)`, just add `gem 'gorgon', '~> 0.4.1' , :group => :remote_test` to your Gemfile, and run tests using `bundle exec gorgon`
+* If `gorgon install_listener` didn't work for you, you can try [these steps](/daemon_with_upstart_and_rvm.md)
 
-Also note that these steps are **not** meant to work on every project, they will only give you initial settings. You will probably have to modify the following files:
+Also note that the steps in the tutorial are **not** meant to work on every project, they will only give you initial settings. You will probably have to modify the following files:
 * gorgon.json
 * test/gorgon_callbacks/after\_sync.rb
 * test/gorgon_callbacks/before\_creating\_workers.rb
@@ -49,13 +35,6 @@ Also note that these steps are **not** meant to work on every project, they will
 
 If you modify ~/.gorgon/gorgon_listener.json, make sure you restart the listener by running `sudo restart gorgon`
 
-Usage
----------------------
-
-To queue the current test suite, run `gorgon start`, or `gorgon`. _gorgon_ will read the application configuration out of _gorgon.json_, connect to the AMQP server, and publish the job.
-
-If you want to run the listener manually (didn't install Daemon process), you must run _gorgon job listeners_. To start a gorgon listener, run `gorgon listen`. This command will read the listener configuration out of _gorgon\_listener.json_, then start the listener process in the background.
-
 Configuration
 ---------------------
 
@@ -80,9 +59,6 @@ This file contains the listener-specific settings, such as:
 
 See [gorgon_listener.json example](/gorgon_listener.json.sample) for more details.
 
-### Manually setting up gorgon listener as a daemon process (Ubuntu 9.10 or later)
-If `gorgon install_listener` didn't work for you, you can try [these steps](/daemon_with_upstart_and_rvm.md)
-
 Contributing
 ---------------------
 Read overview [architecture](/architecture.md)

data/bin/gorgon

@@ -1,4 +1,5 @@
 require "rubygems"
+require 'gorgon'
 require 'gorgon/originator'
 require 'gorgon/listener'
 require 'gorgon/rsync_daemon'

data/gorgon.json.sample

@@ -8,18 +8,20 @@
   },
 
   "job": {
-    "sync_exclude": [
+    "sync": {
+      "exclude": [
+        ".git",
+        ".rvmrc",
         "tmp",
         "log",
-        "doc",
-        ".git",
-        ".rvmrc"
-    ],
+        "doc"
+      ],
+
+      "rsync_transport": "ssh" // or "anonymous"
+    },
+
     "callbacks": {
-        "before_start": "test/gorgon_callbacks/before_start_callback.rb",
-        "after_complete": "test/gorgon_callbacks/after_complete.rb",
-        "before_creating_workers": "test/gorgon_callbacks/before_creating_workers.rb",
-        "after_sync": "test/gorgon_callbacks/after_sync.rb"
+        "callbacks_class_file": "test/gorgon_callbacks.rb"
     }
   },
 

data/lib/gorgon.rb

@@ -1,6 +1,13 @@
 require "gorgon/version"
 require "gorgon/originator"
 require "gorgon/listener"
+require "gorgon/default_callbacks"
 
 module Gorgon
+  class << self
+    attr_accessor :callbacks
+  end
 end
+
+# defaults
+Gorgon.callbacks = Gorgon::DefaultCallbacks.new

data/lib/gorgon/callback_handler.rb

@@ -1,25 +1,35 @@
 class CallbackHandler
   def initialize(config)
     @config = config || {}
+    load(@config[:callbacks_class_file]) if @config[:callbacks_class_file]
   end
 
-  def before_start
-    load(@config[:before_start]) if @config[:before_start]
+  def before_originate
+    cluster_id = Gorgon.callbacks.before_originate
+    return cluster_id if cluster_id.is_a?(String)
   end
 
-  def after_complete
-    load(@config[:after_complete]) if @config[:after_complete]
+  def after_sync
+    Gorgon.callbacks.after_sync
   end
 
   def before_creating_workers
-    load(@config[:before_creating_workers]) if @config[:before_creating_workers]
+    Gorgon.callbacks.before_creating_workers
   end
 
-  def after_sync
-    load(@config[:after_sync]) if @config[:after_sync]
+  def before_start
+    Gorgon.callbacks.before_start
   end
 
   def after_creating_workers
-    load(@config[:after_creating_workers]) if @config[:after_creating_workers]
+    Gorgon.callbacks.after_creating_workers
+  end
+
+  def after_complete
+    Gorgon.callbacks.after_complete
+  end
+
+  def after_job_finishes
+    Gorgon.callbacks.after_job_finishes
   end
 end

data/lib/gorgon/default_callbacks.rb

@@ -0,0 +1,25 @@
+module Gorgon
+  class DefaultCallbacks
+    # @return cluster id. Cluster id is used to identify the queue in RabbitMQ
+    def before_originate
+    end
+
+    def after_sync
+    end
+
+    def before_creating_workers
+    end
+
+    def before_start
+    end
+
+    def after_creating_workers
+    end
+
+    def after_complete
+    end
+
+    def after_job_finishes
+    end
+  end
+end

data/lib/gorgon/job_definition.rb

@@ -1,14 +1,13 @@
 require 'yajl'
 
 class JobDefinition
-  attr_accessor :file_queue_name, :reply_exchange_name, :source_tree_path, :sync_exclude, :callbacks
+  attr_accessor :file_queue_name, :reply_exchange_name, :sync, :callbacks
 
   def initialize(opts={})
     @file_queue_name = opts[:file_queue_name]
     @reply_exchange_name = opts[:reply_exchange_name]
-    @source_tree_path = opts[:source_tree_path]
     @callbacks = opts[:callbacks]
-    @sync_exclude = opts[:sync_exclude]
+    @sync = opts[:sync]
   end
 
   def to_json
@@ -19,6 +18,12 @@ class JobDefinition
 
   #This can probably be done with introspection somehow, but this is way easier despite being very verbose
   def to_hash
-    {:type => "job_definition", :file_queue_name => @file_queue_name, :reply_exchange_name => @reply_exchange_name, :source_tree_path => @source_tree_path, :sync_exclude => @sync_exclude, :callbacks => @callbacks}
+    {
+      :type => "job_definition",
+      :file_queue_name => @file_queue_name,
+      :reply_exchange_name => @reply_exchange_name,
+      :sync => @sync,
+      :callbacks => @callbacks
+    }
   end
 end

data/lib/gorgon/listener.rb

@@ -7,6 +7,7 @@ require "gorgon/version"
 require "gorgon/worker_manager"
 require "gorgon/crash_reporter"
 require "gorgon/gem_command_handler"
+require 'gorgon/originator_protocol'
 
 require "yajl"
 require "gorgon_bunny/lib/gorgon_bunny"
@@ -44,7 +45,7 @@ class Listener
 
   def initialize_personal_job_queue
     @job_queue = @bunny.queue("", :exclusive => true)
-    exchange = @bunny.exchange("gorgon.jobs", :type => :fanout)
+    exchange = @bunny.exchange(job_queue_name, :type => :fanout)
     @job_queue.bind(exchange)
   end
 
@@ -78,8 +79,7 @@ class Listener
     @job_definition = JobDefinition.new(payload)
     @reply_exchange = @bunny.exchange(@job_definition.reply_exchange_name, :auto_delete => true)
 
-    @callback_handler = CallbackHandler.new(@job_definition.callbacks)
-    copy_source_tree(@job_definition.source_tree_path, @job_definition.sync_exclude)
+    copy_source_tree(@job_definition.sync)
 
     if !@syncer.success? || !run_after_sync
       clean_up
@@ -100,7 +100,7 @@ class Listener
   def run_after_sync
     log "Running after_sync callback..."
     begin
-      @callback_handler.after_sync
+      callback_handler.after_sync
     rescue Exception => e
       log_error "Exception raised when running after_sync callback_handler. Please, check your script in #{@job_definition.callbacks[:after_sync]}:"
       log_error e.message
@@ -117,10 +117,13 @@ class Listener
     true
   end
 
-  def copy_source_tree source_tree_path, exclude
+  def callback_handler
+    @callback_handler ||= CallbackHandler.new(@job_definition.callbacks)
+  end
+
+  def copy_source_tree(sync_configuration)
     log "Downloading source tree to temp directory..."
-    @syncer = SourceTreeSyncer.new source_tree_path
-    @syncer.exclude = exclude
+    @syncer = SourceTreeSyncer.new sync_configuration
     @syncer.sync
     if @syncer.success?
       log "Command '#{@syncer.sys_command}' completed successfully."
@@ -145,7 +148,7 @@ class Listener
     stdin.write(@job_definition.to_json)
     stdin.close
 
-    ignore, status = Process.waitpid2 pid
+    _, status = Process.waitpid2 pid
     log "Worker Manager #{pid} finished"
 
     if status.exitstatus != 0
@@ -172,6 +175,10 @@ class Listener
     reply_exchange.publish(Yajl::Encoder.encode(message))
   end
 
+  def job_queue_name
+    OriginatorProtocol.job_queue_name(configuration.fetch(:cluster_id, nil))
+  end
+
   def connection_information
     configuration[:connection]
   end

data/lib/gorgon/originator.rb

@@ -1,3 +1,4 @@
+require 'gorgon'
 require 'gorgon/originator_protocol'
 require 'gorgon/configuration'
 require 'gorgon/job_state'
@@ -5,7 +6,8 @@ require 'gorgon/progress_bar_view'
 require 'gorgon/originator_logger'
 require 'gorgon/failures_printer'
 require 'gorgon/source_tree_syncer'
-require 'gorgon/shutdown_manager.rb'
+require 'gorgon/shutdown_manager'
+require 'gorgon/callback_handler'
 
 require 'awesome_print'
 require 'etc'
@@ -37,8 +39,7 @@ class Originator
   end
 
   def cancel_job
-    ShutdownManager.new(protocol: @protocol,
-                        job_state: @job_state).cancel_job
+    ShutdownManager.new(protocol: @protocol, job_state: @job_state).cancel_job
   end
 
   def ctrl_c
@@ -54,31 +55,42 @@ class Originator
       exit 2
     end
 
+    cluster_id = callback_handler.before_originate
+
     push_source_code
 
-    @protocol = OriginatorProtocol.new @logger
+    @protocol = OriginatorProtocol.new(@logger, cluster_id)
 
     EventMachine.run do
-      @logger.log "Connecting..."
-      @protocol.connect connection_information, :on_closed => method(:on_disconnect)
-
-      @logger.log "Publishing files..."
-      @protocol.publish_files files
-      create_job_state_and_observers
-
-      @logger.log "Publishing Job..."
-      @protocol.publish_job job_definition
-      @logger.log "Job Published"
+      publish_files_and_job
 
       @protocol.receive_payloads do |payload|
         handle_reply(payload)
       end
     end
+
+    callback_handler.after_job_finishes
+  end
+
+  def publish_files_and_job
+    @logger.log "Connecting..."
+    @protocol.connect connection_information, :on_closed => method(:on_disconnect)
+
+    @logger.log "Publishing files..."
+    @protocol.publish_files files
+    create_job_state_and_observers
+
+    @logger.log "Publishing Job..."
+    @protocol.publish_job job_definition
+    @logger.log "Job Published"
+  end
+
+  def callback_handler
+    @callback_handler ||= CallbackHandler.new(configuration[:job][:callbacks])
   end
 
   def push_source_code
-    syncer = SourceTreeSyncer.new(source_tree_path)
-    syncer.exclude = configuration[:job][:sync_exclude]
+    syncer = SourceTreeSyncer.new(sync_configuration)
     syncer.push
     if syncer.success?
       @logger.log "Command '#{syncer.sys_command}' completed successfully."
@@ -125,7 +137,7 @@ class Originator
     @job_state = JobState.new files.count
     @progress_bar_view = ProgressBarView.new @job_state
     @progress_bar_view.show
-    failures_printer = FailuresPrinter.new @job_state
+    FailuresPrinter.new @job_state
   end
 
   def on_disconnect
@@ -143,20 +155,31 @@ class Originator
   end
 
   def job_definition
+    # TODO: remove duplication. Use sync_configuration
     job_config = configuration[:job]
-    if !job_config.has_key?(:source_tree_path)
-      job_config[:source_tree_path] = source_tree_path
-    end
+    job_config[:sync] = {} unless job_config.has_key?(:sync)
+    job_config[:sync][:source_tree_path] = source_tree_path(job_config[:sync])
     JobDefinition.new(configuration[:job])
   end
 
   private
 
-  def source_tree_path
+  def sync_configuration
+    configuration[:job].
+      fetch(:sync, {}).
+      merge(source_tree_path: source_tree_path(configuration[:job][:sync])
+    )
+  end
+
+  def source_tree_path(sync_config)
     hostname = Socket.gethostname
     source_code_root = File.basename(Dir.pwd)
 
-    "rsync://#{file_server_host}:43434/src/#{hostname}_#{source_code_root}"
+    if sync_config && sync_config[:rsync_transport] == SourceTreeSyncer::RSYNC_TRANSPORT_SSH
+      "#{file_server_host}:#{hostname}_#{source_code_root}"
+    else
+      "rsync://#{file_server_host}:43434/src/#{hostname}_#{source_code_root}"
+    end
   end
 
   def file_server_host

data/lib/gorgon/originator_protocol.rb

@@ -5,10 +5,19 @@ require 'amqp'
 require 'uuidtools'
 
 class OriginatorProtocol
-  def initialize logger
+  def initialize(logger, cluster_id=nil)
+    @job_queue_name = OriginatorProtocol.job_queue_name(cluster_id)
     @logger = logger
   end
 
+  def self.job_queue_name(cluster_id)
+    if cluster_id
+      "gorgon.jobs.#{cluster_id}"
+    else
+      'gorgon.jobs'
+    end
+  end
+
   def connect connection_information, options={}
     @connection = AMQP.connect(connection_information)
     @channel = AMQP::Channel.new(@connection)
@@ -28,13 +37,13 @@ class OriginatorProtocol
     job_definition.file_queue_name = @file_queue.name
     job_definition.reply_exchange_name = @reply_exchange.name
 
-    @channel.fanout("gorgon.jobs").publish(job_definition.to_json)
+    @channel.fanout(@job_queue_name).publish(job_definition.to_json)
   end
 
   def send_message_to_listeners type, body={}
     # TODO: we probably want to use a different exchange for this type of messages
     message = {:type => type, :reply_exchange_name => @reply_exchange.name, :body => body}
-    @channel.fanout("gorgon.jobs").publish(Yajl::Encoder.encode(message))
+    @channel.fanout(@job_queue_name).publish(Yajl::Encoder.encode(message))
   end
 
   def receive_payloads

data/lib/gorgon/settings/initial_files_creator.rb

@@ -33,7 +33,10 @@ module Settings
         connection: {host: content.amqp_host},
         file_server: {host: content.file_server_host},
         job: {
-          sync_exclude: content.sync_exclude
+          sync: {
+            exclude: content.sync_exclude
+            # TODO: add rsync_transport config
+          }
         },
         files: content.files
       }

data/lib/gorgon/source_tree_syncer.rb

@@ -1,25 +1,32 @@
 require 'open4'
 
 class SourceTreeSyncer
-  attr_accessor :exclude
+  RSYNC_TRANSPORT_SSH = 'ssh'
+  RSYNC_TRANSPORT_ANONYMOUS = 'anonymous'
+
   attr_reader :sys_command, :output, :errors
 
   SYS_COMMAND = 'rsync'
-  OPTS = "-azr --timeout=5 --delete"
-  EXCLUDE_OPT = "--exclude"
-
-  def initialize source_tree_path
-    @source_tree_path = source_tree_path
-    @exclude = []
+  OPTS = '-azr --timeout=5 --delete'
+  RSH_OPTS = 'ssh -o NumberOfPasswordPrompts=0 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i gorgon.pem'
+  EXCLUDE_OPT = '--exclude'
+
+  def initialize(sync_config)
+    if sync_config
+      @source_tree_path = sync_config[:source_tree_path]
+      @exclude = sync_config[:exclude]
+      @rsync_transport = sync_config[:rsync_transport]
+    end
   end
 
+  # TODO: rename sync to pull
   def sync
     return if blank_source_tree_path?
 
     @tempdir = Dir.mktmpdir("gorgon")
     Dir.chdir(@tempdir)
 
-    @sys_command = "#{SYS_COMMAND} #{OPTS} #{build_exclude_opt} #{@source_tree_path}/ ."
+    @sys_command = "#{SYS_COMMAND} #{rsync_options} #{@source_tree_path}/ ."
 
     execute_command
   end
@@ -27,7 +34,7 @@ class SourceTreeSyncer
   def push
     return if blank_source_tree_path?
 
-    @sys_command = "#{SYS_COMMAND} #{OPTS} #{build_exclude_opt} . #{@source_tree_path}"
+    @sys_command = "#{SYS_COMMAND} #{rsync_options} . #{@source_tree_path}"
 
     execute_command
   end
@@ -68,7 +75,15 @@ class SourceTreeSyncer
     end
   end
 
-  def build_exclude_opt
+  def rsync_options
+    if @rsync_transport == RSYNC_TRANSPORT_SSH
+      "#{OPTS} #{exclude_options} --rsh='#{RSH_OPTS}'"
+    else
+      "#{OPTS} #{exclude_options}"
+    end
+  end
+
+  def exclude_options
     return "" if @exclude.nil? or @exclude.empty?
 
     exclude = [""] + @exclude

data/lib/gorgon/version.rb

@@ -1,3 +1,3 @@
 module Gorgon
-  VERSION = "0.7.1"
+  VERSION = "0.8.0"
 end

data/spec/callback_handler_spec.rb

@@ -1,94 +1,39 @@
-require 'gorgon/callback_handler'
+require 'gorgon'
 
 describe CallbackHandler do
 
   let(:config) {
     {
-      :before_start => "some/file.rb",
-      :after_complete => "some/other/file.rb",
-      :before_creating_workers => "callbacks/before_creating_workers_file.rb",
-      :after_sync => "callbacks/after_sync_file.rb",
-      :after_creating_workers => "callbacks/after_creating_workers.rb"
+      :callbacks_class_file => "callback_file.rb"
     }
   }
-
-  it "calls before hook" do
-    handler = CallbackHandler.new(config)
-
-    handler.should_receive(:load).with("some/file.rb")
-
-    handler.before_start
-  end
-
-  it "does not attempt to load the before start script when before_start is not defined" do
-    handler = CallbackHandler.new({})
-
-    handler.should_not_receive(:load)
-
-    handler.before_start
-  end
-
-  it "calls after hook" do
-    handler = CallbackHandler.new(config)
-
-    handler.should_receive(:load).with("some/other/file.rb")
-
-    handler.after_complete
-  end
-
-  it "does not attempt to load the after complete script when before_start is not defined" do
-    handler = CallbackHandler.new({})
-
-    handler.should_not_receive(:load)
-
-    handler.after_complete
+  before do
+    CallbackHandler.any_instance.stub(:load)
   end
 
-  it "calls before fork hook" do
-    handler = CallbackHandler.new(config)
+  it "loads callback file" do
+    CallbackHandler.any_instance.should_receive(:load).with config[:callbacks_class_file]
 
-    handler.should_receive(:load).with("callbacks/before_creating_workers_file.rb")
-
-    handler.before_creating_workers
-  end
-
-  it "does not attempt to load the before creating workers script when before_creating_workers is not defined" do
-    handler = CallbackHandler.new({})
-
-    handler.should_not_receive(:load)
-
-    handler.before_creating_workers
+    CallbackHandler.new(config)
   end
 
-  it "calls after sync hook" do
-    handler = CallbackHandler.new(config)
-
-    handler.should_receive(:load).with("callbacks/after_sync_file.rb")
+  it "does not load callback file if it's not specified" do
+    CallbackHandler.any_instance.should_not_receive(:load)
 
-    handler.after_sync
+    CallbackHandler.new({})
   end
 
-  it "does not attempt to load the after-sync script when after_sync is not defined" do
-    handler = CallbackHandler.new({})
-
-    handler.should_not_receive(:load)
-
-    handler.after_sync
-  end
-
-  it "calls the after creating workers hook" do
-    handler = CallbackHandler.new(config)
-
-    handler.should_receive(:load).with("callbacks/after_creating_workers.rb")
-
-    handler.after_creating_workers
-  end
-
-  it "does not attempt to load the after creating workers hook when after_creating_workers is not defined" do
-    handler = CallbackHandler.new({})
-
-    handler.should_not_receive(:load)
+  context "#before_originate" do
+    it "returns value from callbacks#before_originate if it's a string" do
+      handler = CallbackHandler.new(config)
+      Gorgon.callbacks.stub(:before_originate).and_return('my_job_id')
+      expect(handler.before_originate).to eq('my_job_id')
+    end
 
-    handler.after_creating_workers
+    it "returns nil if callbacks#before_originate did not return a string" do
+      handler = CallbackHandler.new(config)
+      Gorgon.callbacks.stub(:before_originate).and_return(Object.new)
+      expect(handler.before_originate).to eq(nil)
+    end
   end
 end

data/spec/job_definition_spec.rb

@@ -8,7 +8,13 @@ describe JobDefinition do
 
   describe "#to_json" do
     it "should serialize itself to json" do
-      expected_hash = {:type => "job_definition", :file_queue_name => "string 1", :reply_exchange_name => "string 2", :source_tree_path => "string 3", :sync_exclude => "string 4", :callbacks => {}}
+      expected_hash = {
+        :type => "job_definition",
+        :file_queue_name => "string 1",
+        :reply_exchange_name => "string 2",
+        :sync => {:exclude => "string 4"},
+        :callbacks => {}
+      }
 
       jd = JobDefinition.new(expected_hash)
 

data/spec/listener_spec.rb

@@ -88,7 +88,13 @@ describe Listener do
         listener.initialize_personal_job_queue
       end
 
-      it "binds the exchange to the queue" do
+      it "build job_queue_name using job_id from configuration" do
+        Listener.any_instance.stub(:configuration).and_return(:cluster_id => 'cluster5')
+        bunny.should_receive(:exchange).with('gorgon.jobs.cluster5', anything).and_return(exchange)
+        listener.initialize_personal_job_queue
+      end
+
+      it "binds the exchange to the queue. Uses gorgon.jobs if there is no job_queue_name in configuration" do
         bunny.should_receive(:exchange).with("gorgon.jobs", :type => :fanout).and_return(exchange)
         queue.should_receive(:bind).with(exchange)
         listener.initialize_personal_job_queue
@@ -177,8 +183,7 @@ describe Listener do
 
     describe "#run_job" do
       let(:payload) {{
-          :source_tree_path => "path/to/source",
-          :sync_exclude => ["log"], :callbacks => {:a_callback => "path/to/callback"}
+          :sync => {:source_tree_path => "path/to/source", :exclude => ["log"]}, :callbacks => {:a_callback => "path/to/callback"}
         }}
 
       let(:syncer) { double("SourceTreeSyncer", :sync => nil, :exclude= => nil, :success? => true,
@@ -196,8 +201,9 @@ describe Listener do
       end
 
       it "copy source tree" do
-        SourceTreeSyncer.should_receive(:new).once.with("path/to/source").and_return syncer
-        syncer.should_receive(:exclude=).with(["log"])
+        SourceTreeSyncer.should_receive(:new).once.
+          with(source_tree_path: "path/to/source", exclude: ["log"]).
+          and_return(syncer)
         syncer.should_receive(:sync)
         syncer.should_receive(:success?).and_return(true)
         @listener.run_job(payload)

data/spec/originator_protocol_spec.rb

@@ -67,8 +67,7 @@ describe OriginatorProtocol do
 
   describe "#publish_job" do
     before do
-      @originator_p.connect @conn_information
-      @originator_p.publish_files []
+      connect_and_publish_files(@originator_p)
     end
 
     it "add queue's names to job_definition and fanout using 'gorgon.jobs' exchange" do
@@ -80,6 +79,14 @@ describe OriginatorProtocol do
       exchange.should_receive(:publish).with(exp_job_definition.to_json)
       @originator_p.publish_job JobDefinition.new
     end
+
+    it "uses cluster_id in job_queue_name, when it is specified" do
+      originator_p = connect_and_publish_files(OriginatorProtocol.new(logger, "cluster1"))
+
+      channel.should_receive(:fanout).with("gorgon.jobs.cluster1")
+
+      originator_p.publish_job JobDefinition.new
+    end
   end
 
   describe "#send_message_to_listeners" do
@@ -150,4 +157,10 @@ describe OriginatorProtocol do
       @originator_p.disconnect
     end
   end
+
+  def connect_and_publish_files(originator_p)
+    originator_p.connect @conn_information
+    originator_p.publish_files []
+    originator_p
+  end
 end

data/spec/originator_spec.rb

@@ -20,7 +20,7 @@ describe Originator do
     @originator = Originator.new
   end
 
-  describe "#publish_job" do
+  describe "#publish" do
     before do
       stub_methods
     end
@@ -55,6 +55,24 @@ describe Originator do
 
       expect { @originator.publish }.to raise_error(SystemExit)
     end
+
+    it "calls before_originate callback" do
+      CallbackHandler.any_instance.should_receive(:before_originate)
+      @originator.publish
+    end
+
+    it "uses results of before_originate callback to build a job_queue_name" do
+      CallbackHandler.any_instance.stub(:before_originate).and_return('job_1')
+      OriginatorProtocol.should_receive(:new).with(anything, 'job_1')
+
+      @originator.publish
+    end
+
+    it "calls after_job_finishes callback" do
+      CallbackHandler.any_instance.should_receive(:after_job_finishes)
+
+      @originator.publish
+    end
   end
 
   describe "#cancel_job" do
@@ -136,17 +154,23 @@ describe Originator do
       @originator.job_definition.should equal job_definition
     end
 
-    it "builds source_tree_path if it was not specified in the configuration" do
+    it "builds anonymous source_tree_path if it was not specified in the configuration" do
       @originator.stub(:configuration).and_return(configuration.merge(:file_server => {:host => 'host-name'}))
       Socket.stub(:gethostname => 'my-host')
       Dir.stub(:pwd => 'dir')
 
-      @originator.job_definition.source_tree_path.should == "rsync://host-name:43434/src/my-host_dir"
+      @originator.job_definition.sync[:source_tree_path].should == "rsync://host-name:43434/src/my-host_dir"
     end
 
-    it "returns source_tree_path specified in configuration if it is present" do
-      @originator.stub(:configuration).and_return({:job => {:source_tree_path => "login@host:path/to/dir"}})
-      @originator.job_definition.source_tree_path.should == "login@host:path/to/dir"
+    it "builds ssh source_tree_path if using ssh rsync transport" do
+      @originator.stub(:configuration).and_return(configuration.merge(
+        :file_server => {:host => 'host-name'},
+        :job => { :sync => { :rsync_transport => 'ssh'}}
+      ))
+      Socket.stub(:gethostname => 'my-host')
+      Dir.stub(:pwd => 'dir')
+
+      @originator.job_definition.sync[:source_tree_path].should == "host-name:my-host_dir"
     end
   end
 

data/spec/source_tree_syncer_spec.rb

@@ -1,7 +1,6 @@
 require 'gorgon/source_tree_syncer'
 
-describe SourceTreeSyncer.new("") do
-  it { should respond_to :exclude= }
+describe SourceTreeSyncer.new(source_tree_path: "") do
   it { should respond_to :sync }
   it { should respond_to :push }
   it { should respond_to :sys_command }
@@ -16,7 +15,7 @@ describe SourceTreeSyncer.new("") do
   let(:status) { double("Process Status", :exitstatus => 0)}
 
   before do
-    @syncer = SourceTreeSyncer.new "path/to/source"
+    @syncer = SourceTreeSyncer.new(source_tree_path: "path/to/source")
     stub_utilities_methods
   end
 
@@ -25,11 +24,12 @@ describe SourceTreeSyncer.new("") do
       Dir.should_receive(:mktmpdir).and_return("tmp/dir")
       Dir.should_receive(:chdir).with("tmp/dir")
       @syncer.sync
+      expect(@syncer.success?).to be_true, "Syncer error: #{@syncer.errors}"
     end
 
     context "invalid source_tree_path" do
       it "gives error if source_tree_path is empty string" do
-        syncer = SourceTreeSyncer.new "  "
+        syncer = SourceTreeSyncer.new(source_tree_path: "  ")
         Dir.should_not_receive(:mktmpdir)
         syncer.sync
         syncer.success?.should be_false
@@ -53,7 +53,7 @@ describe SourceTreeSyncer.new("") do
       end
 
       it "exclude files when they are specified" do
-        @syncer.exclude = ["log", ".git"]
+        @syncer = SourceTreeSyncer.new(source_tree_path: "path/to/source", exclude: ["log", ".git"])
         Open4.should_receive(:popen4).with(/--exclude log --exclude .git/)
         @syncer.sync
       end
@@ -98,7 +98,7 @@ describe SourceTreeSyncer.new("") do
 
   describe "#remove_temp_dir" do
     before do
-      @syncer = SourceTreeSyncer.new "path/to/source"
+      @syncer = SourceTreeSyncer.new(source_tree_path: "path/to/source")
       stub_utilities_methods
       @syncer.sync
     end

data/spec/worker_manager_spec.rb

@@ -20,7 +20,7 @@ describe WorkerManager do
   describe ".build" do
     it "should load_configuration_from_file" do
       STDIN.should_receive(:read).and_return '{"source_tree_path":"path/to/source",
-             "sync_exclude":["log"]}'
+             "sync":{"exclude":["log"]}}'
 
       Configuration.should_receive(:load_configuration_from_file).with("file.json").and_return({})
 

data/tutorial.md

@@ -0,0 +1,97 @@
+# Gorgon Setup Tutorial
+
+This is an example on how to setup [Gorgon](https://github.com/Fitzsimmons/Gorgon). For this tutorial, we will use a version of a Rails app from [Ruby on Rails Tutorial](https://www.railstutorial.org/) by Michael Hartl.
+
+### Setup the sample Rails app
+
+First, let's clone and setup the sample Rails app.
+
+```bash
+git clone git@github.com:arturopie/sample_app-1.git
+cd sample_app-1/
+bundle
+rake db:setup
+rake db:test:prepare
+```
+
+Run `rspec` and make sure all tests pass.
+
+
+### Setup Gorgon
+
+1. Install [RabbitMQ](https://www.rabbitmq.com/download.html).
+
+2. Add Gorgon to the Gemfile: `gem 'gorgon', '0.7.1'`
+
+3. `bundle`
+
+4. Run `gorgon init rails` to create initial files for a typical Rails project. Gorgon will ask you to enter the AMQP host name and the File Server host name. Just press enter to use the default host (localhost). 
+
+  Gorgon does not use Rails environment 'test' to run tests. Instead, one of the callbacks generated by `gorgon init rails` will set Rails environment to 'remote_test' before running any test. We need to setup this new environment in our Rails app.
+
+5. Add the following lines to config/database.yml. The callbacks will set TEST_ENV_NUMBER to its process id so that no two Gorgon runners use the same database.
+
+  ```yml
+  remote_test: &remote_test
+    adapter: sqlite3
+    database: db/remote_test_<%=ENV['TEST_ENV_NUMBER']%>
+    pool: 5
+    timeout: 5000
+  ```
+
+6. In the Gemfile, add :remote_test to every :test group. For example:
+
+  ```ruby
+  ...
+  group :development, :test, :remote_test do
+    gem 'sqlite3', '1.3.5'
+  ...
+  group :test, :remote_test do
+    gem 'capybara', '1.1.2'
+  ...
+  ```
+
+
+7. Create config/environments/remote_test.rb file. This file can be just a copy of config/environments/test.rb
+
+  ```bash
+  cp config/environments/test.rb config/environments/remote_test.rb
+  ```
+
+### Run Gorgon Listener
+
+1. Download a listener configuration sample:
+
+  ```bash
+  wget https://raw.githubusercontent.com/Fitzsimmons/Gorgon/master/gorgon_listener.json.sample
+  mv gorgon_listener.json.sample gorgon_listener.json
+  ```
+
+2. Start listener
+
+  ```bash
+  gorgon listen
+  ```
+
+### Run File Server
+
+  ```bash
+  mkdir -p ~/.gorgon/file_dir             # here is where gorgon will push files under test
+  gorgon start_rsync ~/.gorgon/file_dir
+  ```
+
+### Run Gorgon
+
+Now, you are ready to run Gorgon. 
+
+  ```bash
+  gorgon
+  ```
+
+### Result
+
+If everything went well, this is what you should see:
+
+![image](/gorgon-done-screenshot.png)
+
+ 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gorgon
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.8.0
 platform: ruby
 authors:
 - Justin Fitzsimmons
@@ -12,34 +12,34 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-11 00:00:00.000000000 Z
+date: 2014-11-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
@@ -74,14 +74,14 @@ dependencies:
   name: awesome_print
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
@@ -173,6 +173,7 @@ files:
 - architecture.md
 - bin/gorgon
 - daemon_with_upstart_and_rvm.md
+- gorgon-done-screenshot.png
 - gorgon.gemspec
 - gorgon.json.sample
 - gorgon_listener.json.sample
@@ -182,6 +183,7 @@ files:
 - lib/gorgon/colors.rb
 - lib/gorgon/configuration.rb
 - lib/gorgon/crash_reporter.rb
+- lib/gorgon/default_callbacks.rb
 - lib/gorgon/failures_printer.rb
 - lib/gorgon/g_logger.rb
 - lib/gorgon/gem_command_handler.rb
@@ -346,6 +348,7 @@ files:
 - spec/unknown_runner_spec.rb
 - spec/worker_manager_spec.rb
 - spec/worker_spec.rb
+- tutorial.md
 homepage: ''
 licenses: []
 metadata: {}
@@ -355,17 +358,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: gorgon
-rubygems_version: 2.2.2
+rubygems_version: 2.0.14
 signing_key: 
 specification_version: 4
 summary: Distributed testing for ruby with centralized management