manband 0.5.1 → 0.6.0

data/README

@@ -102,7 +102,11 @@ By default, bucket used is "manband". It can be specified in command-line.
 ## Gem creation
 
 Execute:
-    rack package
+    rake package
 
 Gem is created in pkg directory
 Webband may use this gem, or use current files with same hierarchy
+
+## History
+
+0.6.0: Remove mb-minion dependency. New bunny gem is incompatible now, code has been rewriten to remove the dependency and use only amqp.

data/bin/jobhandler.rb

@@ -3,9 +3,7 @@ require 'yaml'
 require 'optparse'
 require 'amqp'
 require 'json'
-require 'mb-minion'
 
-include Minion
 
 # The program manages job commands. Master nodes (workflow handler) sends
 # messages to jobhandler instances. Each instance manages one and
@@ -93,7 +91,33 @@ end
 
 log.info "Start handler "+handler
 
-job "manband.node"+options[:queue]  do |args|
+urlconfig = FlowConfig.config()
+
+
+EventMachine.run do
+  connection = AMQP.connect(FlowConfig.config())
+
+  show_stopper = Proc.new do
+    $stdout.puts "Stopping..."
+    connection.close {
+      EventMachine.stop { exit }
+    }
+  end
+
+  Signal.trap 'INT', show_stopper
+  Signal.trap 'TERM', show_stopper
+
+  channel_exception_handler = Proc.new { |ch, channel_close| EventMachine.stop; raise "channel error: #{channel_close.reply_text}" }
+
+  channel    = AMQP::Channel.new(connection)
+  #channel.on_error(&channel_exception_handler)
+  channel.prefetch(1)
+
+  exchange = channel.direct()
+  queue = channel.queue('manband.node'+options[:queue], :auto_delete => false, :durable => true)
+  queue.bind(exchange).subscribe(:ack => true) do |metadata, message|
+    puts "##DEBUG "+message
+    args = JSON.parse(message)
     msg = JSON.parse(args["msg"])
     if args["operation"] == OP_DESTROY
       myworkflow = WorkFlow.get(msg["id"])
@@ -127,5 +151,10 @@ job "manband.node"+options[:queue]  do |args|
         end
       end
     end
+
+
+  channel.acknowledge(metadata.delivery_tag, false)
+  end
+
 end
 

data/bin/manband.rb

@@ -13,6 +13,7 @@ require 'manband/workflow.rb'
 require 'manband/flowconfig.rb'
 require 'manband/job.rb'
 require 'manband/bandmanager.rb'
+require 'manband/util.rb'
 
 # The program sends a workflow execution request to manband orchestrator.
 # Author: Olivier Sallou <olivier.sallou@irisa.fr>

data/bin/remoteband.rb

@@ -3,9 +3,13 @@ require 'yaml'
 require 'optparse'
 require 'amqp'
 require 'json'
-require 'mb-minion'
+require 'logger'
+
+$:.push File.expand_path("../lib")
+
+require 'manband/flowconfig.rb'
+require 'manband/util.rb'
 
-include Minion
 
 # The program sends a workflow execution request to manband orchestrator.
 # This program does not need database credentials but does not get workflow id in return
@@ -78,4 +82,12 @@ msg["wfile"]=options[:workflow]
 msg["var"]=options[:var]
 msg["uid"]=options[:uid]
 msg["bucket"]=options[:bucket]
-Minion.enqueue("manband.master", { "operation" => "new", "msg" => msg.to_json })
+
+
+Utils.publish('manband.master',{ "operation" => "new", "msg" => msg.to_json })
+
+urlconfig = FlowConfig.config()
+
+
+
+

data/bin/workflowhandler.rb

@@ -3,9 +3,7 @@ require 'yaml'
 require 'optparse'
 require 'amqp'
 require 'json'
-require 'mb-minion'
 
-include Minion
 
 # The program s the main orchestrator of the workflow.
 # It starts with a workflow execution request and sends messages to job
@@ -26,6 +24,7 @@ end
 
 $:.push File.expand_path("../lib")
 
+require 'manband/flowconfig.rb'
 require 'manband/workflow.rb'
 require 'manband/job.rb'
 require 'manband/bandmanager.rb'
@@ -33,8 +32,17 @@ require 'manband/bandmanager.rb'
 log = Logger.new(STDOUT)
 log.level = Logger::INFO
 
+
 @@options = {}
 
+def savemessage(id,operation,msg)
+   if @@options[:debug] == false
+     return
+   end
+   bmsg = BandMessage.new(:wid => id, :message => '{ "operation" => '+operation+', "msg" => '+msg+' }' )
+   bmsg.save
+end
+
 optparse = OptionParser.new do|opts|
   
   # This displays the help screen, all programs are
@@ -75,8 +83,37 @@ if @@options[:conf]!=nil
   FlowConfig.setworkdir(conf["workdir"])
 end
 
-job "manband.master"  do |args|
+
+EventMachine.run do
+  connection = AMQP.connect(FlowConfig.config())
+
+  show_stopper = Proc.new do
+    $stdout.puts "Stopping..."
+    connection.close {
+      EventMachine.stop { exit }
+    }
+  end
+
+  Signal.trap 'INT', show_stopper
+  Signal.trap 'TERM', show_stopper
+
+  channel_exception_handler = Proc.new { |ch, channel_close| EventMachine.stop; raise "channel error: #{channel_close.reply_text}" }
+
+  channel    = AMQP::Channel.new(connection)
+  #channel.on_error(&channel_exception_handler)
+  channel.prefetch(1)
+
+  exchange = channel.direct()
+  queue = channel.queue('manband.master', :auto_delete => false, :durable => true)
+  queue.bind(exchange).subscribe(:ack => true) do |metadata, message|
+
+    begin
+    args = JSON.parse(message)
     msg = JSON.parse(args["msg"])
+    rescue Exception => e
+      log.error("Could not parse message "+args["msg"])
+      channel.acknowledge(metadata.delivery_tag, false)
+    end
     if args["operation"] == OP_NEW
       BandManager.launch(msg["wfile"],msg["var"],msg["uid"],msg["bucket"])
     end
@@ -180,12 +217,10 @@ job "manband.master"  do |args|
         curjob.resume
       end
     end
-end
 
-def savemessage(id,operation,msg)
-   if @@options[:debug] == false
-     return
-   end
-   bmsg = BandMessage.new(:wid => id, :message => '{ "operation" => '+operation+', "msg" => '+msg+' }' )
-   bmsg.save
+
+  channel.acknowledge(metadata.delivery_tag, false)
+  end
+
 end
+

data/lib/manband/bandmanager.rb

@@ -3,10 +3,8 @@ require 'yaml'
 require 'optparse'
 require 'amqp'
 require 'json'
-require 'mb-minion'
 require 'fileutils'
 
-include Minion
 
 require 'manband/workflow.rb'
 require 'manband/flowconfig.rb'
@@ -136,9 +134,9 @@ class BandManager
       # Request workflow management
       msg = '{ "id" : "'+workflow.id.to_s+'", "root" : "'+rootjob.id.to_s+'"}'
       if debug
-        Minion.enqueue("manband.master", { "operation" => OP_SKIP, "msg" => msg })
+        Utils.publish("manband.master", { "operation" => OP_SKIP, "msg" => msg })
       else
-      Minion.enqueue("manband.master", { "operation" => OP_START, "msg" => msg })
+      Utils.publish("manband.master", { "operation" => OP_START, "msg" => msg })
       end
     end
   end
@@ -153,7 +151,10 @@ class BandManager
     newworkflow.parse('root',rootjob.id)
     jobmsg = '{ "id" : "'+newworkflow.id.to_s+'", "root" : "'+rootjob.id.to_s+'"}'
     job = Job.new(:wid => newworkflow.id, :node => "fake", :command => "", :status => STATUS_FAKE, :instances => 0, :workdir => '')
-    job.sendmessage(OP_START,jobmsg)
+    
+    Utils.publish('manband.master', { "operation" => OP_START, "msg" => jobmsg })
+    job.logMessage(OP_START,jobmsg)
+
     return newworkflow.id
   end
 

data/lib/manband/flowconfig.rb

@@ -113,6 +113,22 @@ class FlowConfig
      end
    end
 
+  def self.config
+    url ||= (ENV["AMQP_URL"] || "amqp://guest:guest@localhost/")
+    uri = URI.parse(url)
+    {
+      :vhost => uri.path,
+      :host => uri.host,
+      :user => uri.user,
+      :port => (uri.port || 5672),
+      :pass => uri.password,
+      :heartbeat => 0
+    }
+    rescue Object => e
+      raise("invalid AMQP_URL: #{uri.inspect} (#{e})")
+  end
+
+
 
 end
 

data/lib/manband/job.rb

@@ -3,10 +3,9 @@ require 'dm-migrations'
 require 'yaml'
 require 'manband/workflow.rb'
 require 'manband/flowconfig.rb'
-require 'mb-minion'
+require 'manband/util.rb'
 require 'logger'
 
-include Minion
 
 #DataMapper.setup(:default, 'sqlite:///home/osallou/Desktop/genflow/project.db')
 DataMapper.setup(:default, ENV['MYSQL_URL'])
@@ -68,6 +67,7 @@ class Job
     if instance>0
       workdir = workdir + "/node" + instance.to_s
     end
+    EventMachine.defer do
     err = runcommand(workdir,instance)
     if err == nil || err == false
       # An error occured
@@ -82,6 +82,7 @@ class Job
       jobmsg = '{ "workflow" : "'+@wid.to_s+'" ,  "node" : "'+@node+'", "id" : "'+@id.to_s+'", "handler" : "'+curhandler.to_s+'", "instance" : "'+instance.to_s+'" }'
       sendmessage(OP_FINISH,jobmsg)
     end
+    end # End EventMachine
   end
 
   # Skip treatment, just answer, for debug
@@ -117,7 +118,7 @@ class Job
       cmd = "sudo -u "+wflow.uid+" '"+script+"'"
     end
     @@log.debug cmd
-    system(cmd)
+      system(cmd)
   end
 
   # Change instance counter
@@ -292,12 +293,17 @@ class Job
        queue = "manband.node"+jobqueue
      end
      if queue != nil
-       Minion.enqueue(queue, { "operation" => operation, "msg" => msg })
+       Utils.enqueue(queue, { "operation" => operation, "msg" => msg })
+       logMessage(operation,msg)
+     end
+   end
+
+   # If debug, log the message in the database
+   def logMessage(operation,msg)
        if @@debug == true
          bmsg = BandMessage.new(:wid => @wid, :message => '{ "operation" => '+operation+', "msg" => '+msg+' }' )
          bmsg.save
        end
-     end
    end
 
 end

data/lib/manband/util.rb

@@ -0,0 +1,50 @@
+require 'manband/flowconfig.rb'
+require 'amqp'
+
+# Utility classes
+class Utils
+
+
+  @@connection = nil
+  @@channel    = nil
+  @@exchange = nil
+
+  def self.setcontext()
+    if(@@connection==nil)
+      @@connection = AMQP.connect(FlowConfig.config())
+      @@channel    = AMQP::Channel.new(@@connection)
+      @@exchange = @@channel.direct()
+    end
+  end
+
+  # Send a message in context
+  # args:
+  # dest name of destination queue
+  # msg  message to send. Message is a hash with keys operation and msg.
+  def self.enqueue(dest,data= {})
+    Utils.setcontext()
+    msg = data.to_json
+    queue = @@channel.queue(dest, :auto_delete => false, :durable => true)
+    queue.bind(@@exchange)
+    @@exchange.publish(msg)
+  end
+
+  # Creates a context and sends a message
+  # args:
+  # dest name of destination queue
+  # msg  message to send. Message is a hash with keys operation and msg.
+  def self.publish(dest,data= {})
+  msg = data.to_json
+  EventMachine.run do
+    connection = AMQP.connect(FlowConfig.config())
+    channel    = AMQP::Channel.new(connection)
+    exchange = channel.direct()   
+    queue = channel.queue(dest, :auto_delete => false, :durable => true)
+    queue.bind(exchange)
+    exchange.publish(msg) do
+    connection.close { EventMachine.stop }
+    end
+  end
+  end
+
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: manband
 version: !ruby/object:Gem::Version 
-  hash: 9
+  hash: 7
   prerelease: 
   segments: 
   - 0
-  - 5
-  - 1
-  version: 0.5.1
+  - 6
+  - 0
+  version: 0.6.0
 platform: ruby
 authors: 
 - Olivier Sallou
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-07-16 00:00:00 Z
+date: 2012-07-17 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: amqp
@@ -88,7 +88,7 @@ dependencies:
   type: :runtime
   version_requirements: *id005
 - !ruby/object:Gem::Dependency 
-  name: mb-minion
+  name: mb-aws-s3
   prerelease: false
   requirement: &id006 !ruby/object:Gem::Requirement 
     none: false
@@ -102,7 +102,7 @@ dependencies:
   type: :runtime
   version_requirements: *id006
 - !ruby/object:Gem::Dependency 
-  name: mb-aws-s3
+  name: uuid
   prerelease: false
   requirement: &id007 !ruby/object:Gem::Requirement 
     none: false
@@ -115,20 +115,6 @@ dependencies:
         version: "0"
   type: :runtime
   version_requirements: *id007
-- !ruby/object:Gem::Dependency 
-  name: uuid
-  prerelease: false
-  requirement: &id008 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  type: :runtime
-  version_requirements: *id008
 description: 
 email: olivier.sallou@irisa.fr
 executables: []
@@ -148,6 +134,7 @@ files:
 - lib/manband/bandmanager.rb
 - lib/manband/user.rb
 - lib/manband/workflow.rb
+- lib/manband/util.rb
 - lib/manband/job.rb
 - lib/manband/flowconfig.rb
 - README