gorgon 0.8.4 → 0.9.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ODE4OGJiNzI0ZTBkNmRhYjJjNGJkNzg0Y2FjNmQ0M2ZlYWY2Mjk0NA==
+    NTA0MWUwMjQ4ZmIzZmM0ZjAyYWJjOWYxMTA1NWQwZGFiZTQ0MzQ5Yg==
   data.tar.gz: !binary |-
-    NjVkNjkwNGE1ZTE0N2NmMTNjNzE3ZjExYWVkMjYwZjUzYTE3NjdjZA==
+    MTA4NGNjZjU2OWQxOWQxMmZkYTlkOGNjYjI0MmIxYTI5ODMzZGEzYQ==
 SHA512:
   metadata.gz: !binary |-
-    NDQxMzVmNjk2YTM3NThjZWNkMjkxNDc5MzVlMWE2ZWIwZmZhOWU0ZjJhZTlj
-    MWYxZDZmM2RkNTFlNDJmM2FhNWIxZDJkODA4NTIyZjVhYzgzYmQ2NGY4Zjgy
-    MzBkMzg0ZmE3MjU5ODdjOTBhYjA0MGY1MTRiYjg2OThlZmEwMmY=
+    N2U2ZWZlMWNkNDI2YTEyOTkwNGYwZjQwNGI0NWJmMGEzOGI0ZDg2YTA0MWYw
+    MDgxNGQzMTcwOTZmYmY3OTk4MzY3YTk3NDBmNjEyYjhlMjQwMGEwNDQ0Y2Jk
+    MGJhYTk3M2EyZWE2ZjU5MzZiNTAyNWNhOTdkZTYzYWU0YzViYzk=
   data.tar.gz: !binary |-
-    NTM0ZDExMTcyMTM4MGIyNzU1ZmRjM2RkZTY4NTU2NGUxYTJjYzQ5YzliZmEy
-    ZmVjMmVjMGQ1NWI2N2U5YjdhNTc2YWU1YmY4NmI5ZGQ0MWU4ZGQwMDM3ODcw
-    MjU3Njg3OTE1ZDAwZWMyMWE4ZjQ5ZTM2Mzk3ZTFmZTZmYTUzYTE=
+    Mzg3NDc4MWZjNzE0MDYwMjZmNWMzMjE1MzNhY2JkYTU2MzBhZTQ0ZmYzNTdm
+    MjdkYTM3YTY5MWVlMjE2ZjRiZjMyYjdmMDNjZDZmMTQ4ODM5Njk3NGU0ZjIy
+    MmYyMDYxOTJkNGFiMzVjZjgxZWRlMDk3NzZhMzQ3M2E5NzEwNjk=

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gorgon (0.8.4)
+    gorgon (0.9.0)
       amqp (~> 1.1.0)
       awesome_print
       colorize (~> 0.5.8)
@@ -17,10 +17,10 @@ GEM
     amqp (1.1.8)
       amq-protocol (>= 1.9.2)
       eventmachine
-    awesome_print (1.2.0)
+    awesome_print (1.6.1)
     colorize (0.5.8)
     diff-lcs (1.1.3)
-    eventmachine (1.0.4)
+    eventmachine (1.0.7)
     open4 (1.3.4)
     rake (0.9.2.2)
     rspec (2.11.0)

data/README.md

@@ -21,7 +21,7 @@ Installing listener as a Daemon process (Ubuntu 9.10 or later)
 Gotchas
 ----------------------------------------------------------------
 
-* if you get `cannot load such file -- qrack/qrack (LoadError)`, just add `gem 'gorgon', '~> 0.8.3' , :group => :remote_test` to your Gemfile, and run tests using `bundle exec gorgon`
+* if you get `cannot load such file -- qrack/qrack (LoadError)`, just add `gem 'gorgon', '~> 0.8.4' , :group => :remote_test` to your Gemfile, and run tests using `bundle exec gorgon`
 * If `gorgon install_listener` didn't work for you, you can try [these steps](/daemon_with_upstart_and_rvm.md)
 
 Also note that the steps in the tutorial are **not** meant to work on every project, they will only give you initial settings. You will probably have to modify the following files:

data/lib/gorgon/listener.rb

@@ -28,6 +28,7 @@ class Listener
     log "Listener #{Gorgon::VERSION} initializing"
     connect
     initialize_personal_job_queue
+    announce_readiness_to_originators
   end
 
   def listen
@@ -44,11 +45,17 @@ class Listener
   end
 
   def initialize_personal_job_queue
-    @job_queue = @bunny.queue("", :exclusive => true)
-    exchange = @bunny.exchange(job_queue_name, :type => :fanout)
+    @job_queue = @bunny.queue("job_queue_" + UUIDTools::UUID.timestamp_create.to_s, :auto_delete => true)
+    exchange = @bunny.exchange(job_exchange_name, :type => :fanout)
     @job_queue.bind(exchange)
   end
 
+  def announce_readiness_to_originators
+    exchange = @bunny.exchange(originator_exchange_name, :type => :fanout)
+    data = {:listener_queue_name => @job_queue.name}
+    exchange.publish(Yajl::Encoder.encode(data))
+  end
+
   def poll
     message = @job_queue.pop
     return false if message == [nil, nil, nil]
@@ -175,8 +182,12 @@ class Listener
     reply_exchange.publish(Yajl::Encoder.encode(message))
   end
 
-  def job_queue_name
-    OriginatorProtocol.job_queue_name(configuration.fetch(:cluster_id, nil))
+  def job_exchange_name
+    OriginatorProtocol.job_exchange_name(configuration.fetch(:cluster_id, nil))
+  end
+
+  def originator_exchange_name
+    OriginatorProtocol.originator_exchange_name(configuration.fetch(:cluster_id, nil))
   end
 
   def connection_information

data/lib/gorgon/originator.rb

@@ -69,6 +69,10 @@ class Originator
       @protocol.receive_payloads do |payload|
         handle_reply(payload)
       end
+
+      @protocol.receive_new_listener_notifications do |payload|
+        handle_new_listener_notification(payload)
+      end
     end
 
     callback_handler.after_job_finishes
@@ -83,7 +87,7 @@ class Originator
     create_job_state_and_observers
 
     @logger.log "Publishing Job..."
-    @protocol.publish_job job_definition
+    @protocol.publish_job_to_all job_definition
     @logger.log "Job Published"
   end
 
@@ -135,6 +139,17 @@ class Originator
     cleanup_if_job_complete
   end
 
+  def handle_new_listener_notification(payload)
+    payload = Yajl::Parser.new(:symbolize_keys => true).parse(payload)
+
+    if payload[:listener_queue_name]
+      @protocol.publish_job_to_one(job_definition, payload[:listener_queue_name])
+    else
+      puts "Received unexpected payload on originator queue"
+      ap payload
+    end
+  end
+
   def create_job_state_and_observers
     @job_state = JobState.new files.count
     RuntimeRecorder.new @job_state, configuration[:runtime_file]

data/lib/gorgon/originator_protocol.rb

@@ -6,11 +6,20 @@ require 'uuidtools'
 
 class OriginatorProtocol
   def initialize(logger, cluster_id=nil)
-    @job_queue_name = OriginatorProtocol.job_queue_name(cluster_id)
+    @originator_exchange_name = OriginatorProtocol.originator_exchange_name(cluster_id)
+    @job_exchange_name = OriginatorProtocol.job_exchange_name(cluster_id)
     @logger = logger
   end
 
-  def self.job_queue_name(cluster_id)
+  def self.originator_exchange_name(cluster_id)
+    if cluster_id
+      "gorgon.originators.#{cluster_id}"
+    else
+      "gorgon.originators"
+    end
+  end
+
+  def self.job_exchange_name(cluster_id)
     if cluster_id
       "gorgon.jobs.#{cluster_id}"
     else
@@ -33,17 +42,29 @@ class OriginatorProtocol
     end
   end
 
-  def publish_job job_definition
+  def publish_job_to_all job_definition
+    job_definition = append_protocol_information_to_job_definition(job_definition)
+    @channel.fanout(@job_exchange_name).publish(job_definition.to_json)
+  end
+
+  def publish_job_to_one job_definition, listener_queue_name
+    job_definition = append_protocol_information_to_job_definition(job_definition)
+    @channel.default_exchange.publish(job_definition.to_json, :routing_key => listener_queue_name)
+  end
+
+  def append_protocol_information_to_job_definition job_definition
+    job_definition = job_definition.dup
+
     job_definition.file_queue_name = @file_queue.name
     job_definition.reply_exchange_name = @reply_exchange.name
 
-    @channel.fanout(@job_queue_name).publish(job_definition.to_json)
+    return job_definition
   end
 
   def send_message_to_listeners type, body={}
     # TODO: we probably want to use a different exchange for this type of messages
     message = {:type => type, :reply_exchange_name => @reply_exchange.name, :body => body}
-    @channel.fanout(@job_queue_name).publish(Yajl::Encoder.encode(message))
+    @channel.fanout(@job_exchange_name).publish(Yajl::Encoder.encode(message))
   end
 
   def receive_payloads
@@ -52,6 +73,12 @@ class OriginatorProtocol
     end
   end
 
+  def receive_new_listener_notifications
+    @originator_queue.subscribe do |payload|
+      yield payload
+    end
+  end
+
   def cancel_job
     @file_queue.purge if @file_queue
     @channel.fanout("gorgon.worker_managers").publish(cancel_message) if @channel
@@ -69,12 +96,18 @@ class OriginatorProtocol
     @reply_queue = @channel.queue("reply_queue_" + UUIDTools::UUID.timestamp_create.to_s, :auto_delete => true)
     @reply_exchange = @channel.direct("reply_exchange_" + UUIDTools::UUID.timestamp_create.to_s, :auto_delete => true)
     @reply_queue.bind(@reply_exchange)
+
+    # Provides a way for new listeners to announce their presence to originators that have already started the job
+    @originator_queue = @channel.queue("originator_queue_" + UUIDTools::UUID.timestamp_create.to_s, :auto_delete => true)
+    @originator_exchange = @channel.fanout(@originator_exchange_name)
+    @originator_queue.bind(@originator_exchange)
   end
 
   def cleanup_queues_and_exchange
     @reply_queue.delete if @reply_queue
     @file_queue.delete if @file_queue
     @reply_exchange.delete if @reply_exchange
+    @originator_queue.delete if @originator_queue
   end
 
   def cancel_message

data/lib/gorgon/settings/rails_project_files_content.rb

@@ -8,7 +8,7 @@ module Settings
       @file_server_host = FilesContent.get_file_server_host
       @sync_exclude = [".git", ".rvmrc","doc","log","tmp"]
       @originator_log_file = 'log/gorgon-originator.log'
-      @failed_files = 'tmp/gorgon-failed-files.json'
+      @failed_files = 'gorgon-failed-files.json'
       create_callbacks
     end
 
@@ -88,7 +88,7 @@ class GorgonCallbacks < Gorgon::DefaultCallbacks
     load './Rakefile'
 
     begin
-      if Rails.env = 'remote_test'
+      if Rails.env == 'remote_test'
         Rake::Task['db:drop'].execute
       end
     rescue Exception => ex

data/lib/gorgon/version.rb

@@ -1,3 +1,3 @@
 module Gorgon
-  VERSION = "0.8.4"
+  VERSION = "0.9.0"
 end

data/spec/listener_spec.rb

@@ -2,7 +2,7 @@ require 'gorgon/listener'
 
 describe Listener do
   let(:connection_information) { double }
-  let(:queue) { double("GorgonBunny Queue", :bind => nil) }
+  let(:queue) { double("GorgonBunny Queue", :bind => nil, :name => "some supposedly unique string") }
   let(:exchange) { double("GorgonBunny Exchange", :publish => nil) }
   let(:bunny) { double("GorgonBunny", :start => nil, :queue => queue, :exchange => exchange) }
   let(:logger) { double("Logger", :info => true, :datetime_format= => "")}
@@ -14,23 +14,6 @@ describe Listener do
     Listener.any_instance.stub(:connection_information => connection_information)
   end
 
-  describe "initialization" do
-
-    before do
-      Listener.any_instance.stub(:connect => nil, :initialize_personal_job_queue => nil)
-    end
-
-    it "connects" do
-      Listener.any_instance.should_receive(:connect)
-      Listener.new
-    end
-
-    it "initializes the personal job queue" do
-      Listener.any_instance.should_receive(:initialize_personal_job_queue)
-      Listener.new
-    end
-  end
-
   describe "logging to a file" do
     context "passing a log file path in the configuration" do
       before do
@@ -84,23 +67,36 @@ describe Listener do
 
     describe "#initialize_personal_job_queue" do
       it "creates the job queue" do
-        bunny.should_receive(:queue).with("", :exclusive => true)
+        UUIDTools::UUID.stub(:timestamp_create => "abcd1234")
+
+        bunny.should_receive(:queue).with("job_queue_abcd1234", :auto_delete => true)
         listener.initialize_personal_job_queue
       end
 
-      it "build job_queue_name using job_id from configuration" do
+      it "builds job_exchange_name using cluster_id from configuration" do
         Listener.any_instance.stub(:configuration).and_return(:cluster_id => 'cluster5')
         bunny.should_receive(:exchange).with('gorgon.jobs.cluster5', anything).and_return(exchange)
         listener.initialize_personal_job_queue
       end
 
-      it "binds the exchange to the queue. Uses gorgon.jobs if there is no job_queue_name in configuration" do
+      it "binds the exchange to the queue. Uses gorgon.jobs if there is no job_exchange_name in configuration" do
         bunny.should_receive(:exchange).with("gorgon.jobs", :type => :fanout).and_return(exchange)
         queue.should_receive(:bind).with(exchange)
         listener.initialize_personal_job_queue
       end
     end
 
+    describe "#announce_readiness_to_originators" do
+      it "publishes data to the originator exchange" do
+        originator_exchange = double
+
+        bunny.should_receive(:exchange).with("gorgon.originators", :type => :fanout).and_return(originator_exchange)
+        originator_exchange.should_receive(:publish).with({:listener_queue_name => "some supposedly unique string"}.to_json)
+
+        listener.announce_readiness_to_originators
+      end
+    end
+
     describe "#poll" do
 
       let(:empty_queue) { [nil, nil, nil] }

data/spec/originator_protocol_spec.rb

@@ -65,19 +65,19 @@ describe OriginatorProtocol do
     end
   end
 
-  describe "#publish_job" do
+  describe "#publish_job_to_all" do
     before do
       connect_and_publish_files(@originator_p)
     end
 
-    it "add queue's names to job_definition and fanout using 'gorgon.jobs' exchange" do
+    it "adds queue's names to job_definition and fanout using 'gorgon.jobs' exchange" do
       channel.should_receive(:fanout).with("gorgon.jobs")
-      exp_job_definition = JobDefinition.new
-      exp_job_definition.file_queue_name = "queue"
-      exp_job_definition.reply_exchange_name = "exchange"
+      expected_job_definition = JobDefinition.new
+      expected_job_definition.file_queue_name = "queue"
+      expected_job_definition.reply_exchange_name = "exchange"
 
-      exchange.should_receive(:publish).with(exp_job_definition.to_json)
-      @originator_p.publish_job JobDefinition.new
+      exchange.should_receive(:publish).with(expected_job_definition.to_json)
+      @originator_p.publish_job_to_all JobDefinition.new
     end
 
     it "uses cluster_id in job_queue_name, when it is specified" do
@@ -85,7 +85,24 @@ describe OriginatorProtocol do
 
       channel.should_receive(:fanout).with("gorgon.jobs.cluster1")
 
-      originator_p.publish_job JobDefinition.new
+      originator_p.publish_job_to_all JobDefinition.new
+    end
+  end
+
+  describe "#publish_job_to_one" do
+    before do
+      connect_and_publish_files(@originator_p)
+    end
+
+    it "publishes the job to the specified listener queue" do
+      expected_listener_queue_name = "abcd1234"
+      expected_job_definition = JobDefinition.new
+      expected_job_definition.file_queue_name = "queue"
+      expected_job_definition.reply_exchange_name = "exchange"
+
+      exchange.should_receive(:publish).with(expected_job_definition.to_json, {:routing_key => expected_listener_queue_name})
+
+      @originator_p.publish_job_to_one(JobDefinition.new, expected_listener_queue_name)
     end
   end
 
@@ -147,7 +164,7 @@ describe OriginatorProtocol do
 
     it "deletes reply_exchange and reply and file queues" do
       @originator_p.publish_files []
-      queue.should_receive(:delete).twice
+      queue.should_receive(:delete).exactly(3).times
       exchange.should_receive(:delete)
       @originator_p.disconnect
     end

data/spec/originator_spec.rb

@@ -2,16 +2,17 @@ require 'gorgon/originator'
 
 describe Originator do
   let(:protocol){ double("Originator Protocol", :connect => nil, :publish_files => nil,
-                       :publish_job => nil, :receive_payloads => nil, :cancel_job => nil,
-                       :disconnect => nil)}
+    :publish_job_to_all => nil, :publish_job_to_one => nil, :receive_payloads => nil, :cancel_job => nil,
+    :disconnect => nil, :receive_new_listener_notifications => nil)}
 
   let(:configuration){ {:job => {}, :files => ["some/file"], :file_server => {:host => 'host-name'}}}
   let(:job_state){ double("JobState", :is_job_complete? => false, :file_finished => nil,
-                        :add_observer => nil)}
+    :add_observer => nil)}
   let(:progress_bar_view){ double("Progress Bar View", :show => nil)}
   let(:originator_logger){ double("Originator Logger", :log => nil, :log_message => nil)}
   let(:source_tree_syncer) { double("Source Tree Syncer", :push => nil, :exclude= => nil, :success? => true,
-                                  :sys_command => 'command')}
+    :sys_command => 'command')}
+  let(:job_definition){ JobDefinition.new }
 
   before do
     OriginatorLogger.stub(:new).and_return originator_logger
@@ -146,6 +147,16 @@ describe Originator do
     end
   end
 
+  describe "#handle_new_listener_notification" do
+    it "re-publishes the job definition directly to the queue specified by the notification" do
+      stub_methods
+      @originator.publish
+
+      protocol.should_receive(:publish_job_to_one).with(job_definition, 'abcd1234')
+      @originator.handle_new_listener_notification({:listener_queue_name => 'abcd1234'}.to_json)
+    end
+  end
+
   describe "#job_definition" do
     it "returns a JobDefinition object" do
       @originator.stub(:configuration).and_return configuration
@@ -182,7 +193,7 @@ describe Originator do
     OriginatorProtocol.stub(:new).and_return protocol
     @originator.stub(:configuration).and_return configuration
     @originator.stub(:connection_information).and_return 'host'
-    @originator.stub(:job_definition).and_return JobDefinition.new
+    @originator.stub(:job_definition).and_return job_definition
   end
 
   def start_payload

data/tutorial.md

@@ -21,7 +21,14 @@ Run `rspec` and make sure all tests pass.
 
 1. Install [RabbitMQ](https://www.rabbitmq.com/download.html).
 
-2. Add Gorgon to the Gemfile: `gem 'gorgon', '0.8.3'`
+  If you are using Homebrew on OSX, run the following:
+
+  ```bash
+  brew install rabbitmq
+  /usr/local/sbin/rabbitmq-server    # runs the server
+  ```
+
+2. Add Gorgon to the Gemfile: `gem 'gorgon', '0.8.4'`
 
 3. `bundle`
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gorgon
 version: !ruby/object:Gem::Version
-  version: 0.8.4
+  version: 0.9.0
 platform: ruby
 authors:
 - Justin Fitzsimmons
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-16 00:00:00.000000000 Z
+date: 2015-05-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake