manband 0.6.1 → 0.7.0

data/README

@@ -108,6 +108,9 @@ Webband may use this gem, or use current files with same hierarchy
 
 ## History
 
+0.7: 
+     Add IF management
+     Fix some message dispatching issues
 0.6.1: Increase file path max size
 0.6.0: Remove mb-minion dependency. New bunny gem is incompatible now, code has been rewriten to remove the dependency and use only amqp.
 

data/bin/env.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+MYSQL_URL=mysql://root:neptune@localhost/genflow
+AMQP_URL=amqp://guest:guest@localhost/
+
+export MYSQL_URL
+export AMQP_URL

data/bin/jobhandler.rb

@@ -28,7 +28,7 @@ require 'manband/store.rb'
 handler = Time.now.to_s
 
 log = Logger.new(STDOUT)
-log.level = Logger::INFO
+log.level = Logger::DEBUG
 
 options = {}
 
@@ -116,7 +116,8 @@ EventMachine.run do
   exchange = channel.direct()
   queue = channel.queue('manband.node'+options[:queue], :auto_delete => false, :durable => true)
   queue.bind(exchange).subscribe(:ack => true) do |metadata, message|
-    puts "##DEBUG "+message
+    log.debug "Received msg: "+message.to_s
+    doack = true
     args = JSON.parse(message)
     msg = JSON.parse(args["msg"])
     if args["operation"] == OP_DESTROY
@@ -137,6 +138,8 @@ EventMachine.run do
         log.error "Error: job does not exists "+msg["id"]
       else 
         if args["operation"] == OP_RUN
+          myjob.setAck(channel,metadata.delivery_tag)
+          doack = false
           if msg["instance"]!=nil
             myjob.run(handler, msg["instance"].to_i)
           else
@@ -153,7 +156,9 @@ EventMachine.run do
     end
 
 
-  channel.acknowledge(metadata.delivery_tag, false)
+    if doack
+      channel.acknowledge(metadata.delivery_tag, false)
+    end
   end
 
 end

data/bin/workflowhandler.rb

@@ -30,7 +30,7 @@ require 'manband/job.rb'
 require 'manband/bandmanager.rb'
 
 log = Logger.new(STDOUT)
-log.level = Logger::INFO
+log.level = Logger::DEBUG
 
 
 @@options = {}
@@ -108,6 +108,7 @@ EventMachine.run do
   queue.bind(exchange).subscribe(:ack => true) do |metadata, message|
 
     begin
+    log.debug "Received msg: "+message.to_s
     args = JSON.parse(message)
     msg = JSON.parse(args["msg"])
     rescue Exception => e
@@ -130,12 +131,12 @@ EventMachine.run do
     end
     if args["operation"] == OP_SKIP
       savemessage(msg["id"],args["operation"],args["msg"])
-      workflow = WorkFlow.get(msg["id"])
+      workflow = WorkFlow.get(msg["workflow"])
       if workflow == nil
-        log.error "Error: workflow does not exists "+msg["id"]
+        log.error "Error: workflow does not exists "+msg["workflow"]
       else
         workflow.update(:status => STATUS_RUNNING)
-        myjob = Job.get(msg["root"])
+        myjob = Job.get(msg["id"])
         myjob.runnext(true)
       end
     end
@@ -156,15 +157,19 @@ EventMachine.run do
     if args["operation"] == OP_FINISH
       savemessage(msg["workflow"],args["operation"],args["msg"])
       myjob = Job.get(msg["id"])
+      doSkip = false
       if myjob == nil
         log.error "Error: job does not exists "+msg["id"]
       else
+        if myjob.status == STATUS_SKIP
+          doSkip = true
+        end
         if msg["handler"]!=nil && myjob.handler != msg["handler"]
           log.debug "Finish received from a different handler "+myjob.handler+" vs "+msg["handler"]+", skipping message"
         else
           myjob.finish
           if myjob.isover?
-            myjob.runnext
+            myjob.runnext(doSkip)
           end
         end
       end

data/lib/manband/bandmanager.rb

@@ -134,9 +134,10 @@ class BandManager
       # Request workflow management
       msg = '{ "id" : "'+workflow.id.to_s+'", "root" : "'+rootjob.id.to_s+'"}'
       if debug
+        msg = '{ "id" : "'+rootjob.id.to_s+'","workflow" : "'+workflow.id.to_s+'", "node" : "'+rootjob.id.to_s+'"}'
         Utils.publish("manband.master", { "operation" => OP_SKIP, "msg" => msg })
       else
-      Utils.publish("manband.master", { "operation" => OP_START, "msg" => msg })
+        Utils.publish("manband.master", { "operation" => OP_START, "msg" => msg })
       end
     end
   end

data/lib/manband/flowconfig.rb

@@ -35,6 +35,13 @@ OP_CLEAN = "clean"  # Delete work dirs of workflow
 OP_DESTROY = "destroy"  # Delete workflow and its work dirs
 OP_STORE = "store" # Store result to S3
 
+# Basic actor
+ACTOR = -1
+# If actor, i.e. ony one of the following jobs will be executed. Selected job match the exit code of the command. Type of the node will be set to selected output
+# WARNING: input of IF actors MUST NOT be a pattern matchng multiple files. The IF condition must be executed only once, not on a bunch of files.
+IF_ACTOR = -2
+
+
 # This class holds the base config
 class FlowConfig
    @@workdir='/tmp'

data/lib/manband/job.rb

@@ -34,7 +34,19 @@ class Job
   end
 
   @@log = Logger.new(STDOUT)
-  @@log.level = Logger::INFO
+  @@log.level = Logger::DEBUG
+
+  @channel = nil
+  @delivery_tag = nil
+
+  def setAck(channel,tag)
+    @channel = channel
+    @tag = tag
+  end
+
+  def ack()
+    @channel.acknowledge(@tag, false)
+  end
 
   property :id, Serial
   property :wid, Integer # id of the worflow
@@ -48,6 +60,7 @@ class Job
   property :queue, String, :default => ""  # Queue to handle the job
   property :error, Text, :default => "[]" # JSON Array of error, per instance
   property :store, Integer, :default => STORE_NO # Storage status
+  property :type, Integer, :default => ACTOR # Type of the job
 
   # Update job status and launch the command locally
   # Sends a finish message or an error message according to the job status.
@@ -67,9 +80,17 @@ class Job
     if instance>0
       workdir = workdir + "/node" + instance.to_s
     end
+    
     EventMachine.defer do
     err = runcommand(workdir,instance)
-    if err == nil || err == false
+    # In the case of IF_ACTOR, exit code is not an error
+    # but the job to select. Reset error code after updating the type.
+    if @type == IF_ACTOR
+      @type = err + 1
+      curjob.update(:type => (err + 1))
+      err = 0
+    end 
+    if err == nil || err > 0
       # An error occured
       jobmsg = '{ "workflow" : "'+@wid.to_s+'" ,  "node" : "'+@node+'", "id" : "'+@id.to_s+'", "handler" : "'+curhandler.to_s+'", "instance" : "'+instance.to_s+'" }'
       sendmessage(OP_ERROR,jobmsg)
@@ -82,6 +103,7 @@ class Job
       jobmsg = '{ "workflow" : "'+@wid.to_s+'" ,  "node" : "'+@node+'", "id" : "'+@id.to_s+'", "handler" : "'+curhandler.to_s+'", "instance" : "'+instance.to_s+'" }'
       sendmessage(OP_FINISH,jobmsg)
     end
+    ack()
     end # End EventMachine
   end
 
@@ -119,6 +141,7 @@ class Job
     end
     @@log.debug cmd
       system(cmd)
+      return $?.exitstatus
   end
 
   # Change instance counter
@@ -245,12 +268,17 @@ class Job
       workflow.isover?
       return true
     end
+    # Which next job is it?
+    # Start at 1 because 0 is a valid status code, consider if type from value 1
+    count = 1
+    # For each job to follow
     nexts.each do |link|
     job = Job.get(link.to)
     #nexts.each do |nextnode|
       #nextnode.strip!
       #queue = workflow.getnodequeue(nextnode)
-      if skip==true  || @status == STATUS_SKIP
+      # Explicit skip, or IF actor, we skip all actors but one
+      if skip==true  || @status == STATUS_SKIP  || (@type>0 && count!= @type)
         jobstatus = STATUS_SKIP
         operation = OP_SKIP
         commands = [ "skipcommand"]
@@ -259,6 +287,7 @@ class Job
         operation = OP_RUN
         commands = workflow.getnodecommand(job.node) 
       end
+      count=count+1
       if commands == nil
         @@log.error "Could not get command for node "+job.id.to_s
         job.update(:command => "", :status => STATUS_ERROR, :instances => 0, :maxinstances => 0, :workdir => FlowConfig.getjobdir(workflow.workdir))
@@ -293,6 +322,7 @@ class Job
        queue = "manband.node"+jobqueue
      end
      if queue != nil
+       @@log.debug "Send #"+operation+","+msg
        Utils.enqueue(queue, { "operation" => operation, "msg" => msg })
        logMessage(operation,msg)
      end

data/lib/manband/store.rb

@@ -11,7 +11,7 @@ require 'manband/user.rb'
 class Storeband
 
   @@log = Logger.new(STDOUT)
-  @@log.level = Logger::DEBUG
+  @@log.level = Logger::INFO
 
   # Compress a directory
   def compress(dir,name)

data/lib/manband/util.rb

@@ -26,7 +26,8 @@ class Utils
     msg = data.to_json
     queue = @@channel.queue(dest, :auto_delete => false, :durable => true)
     queue.bind(@@exchange)
-    @@exchange.publish(msg)
+    #@@exchange.publish(msg, :routing_key => dest)
+    @@channel.direct("").publish(msg, :routing_key => dest)
   end
 
   # Creates a context and sends a message
@@ -38,10 +39,11 @@ class Utils
   EventMachine.run do
     connection = AMQP.connect(FlowConfig.config())
     channel    = AMQP::Channel.new(connection)
-    exchange = channel.direct()   
-    queue = channel.queue(dest, :auto_delete => false, :durable => true)
-    queue.bind(exchange)
-    exchange.publish(msg) do
+    #exchange = channel.direct()   
+    #queue = channel.queue(dest, :auto_delete => false, :durable => true)
+    #queue.bind(exchange)
+    #exchange.publish(msg, :routing_key => dest) do
+    channel.direct("").publish(msg, :routing_key => dest) do
     connection.close { EventMachine.stop }
     end
   end

data/lib/manband/workflow.rb

@@ -19,7 +19,7 @@ class WorkFlow
   include DataMapper::Resource
 
   @@log = Logger.new(STDOUT)
-  @@log.level = Logger::INFO
+  @@log.level = Logger::DEBUG
 
   property :id,         Serial    # An auto-increment integer key
   property :uid,   Text  # user id
@@ -109,11 +109,15 @@ class WorkFlow
           @@log.debug "Add store option for job "+job
           store = STORE_DO
         end
+        type = ACTOR
+        if fworkflow["workflow"][job]["type"]=='if'
+          type = IF_ACTOR
+        end
         workdir = FlowConfig.getjobdir(@workdir)
         if curnode == "root"
           workdir = self.workdir + "/root";
         end
-        newjob = Job.new(:wid => @id, :node => job, :command => "", :status => status, :instances => 0, :maxinstances => 0, :queue => queue, :workdir => FlowConfig.getjobdir(@workdir), :store => store)
+        newjob = Job.new(:wid => @id, :node => job, :command => "", :status => status, :instances => 0, :maxinstances => 0, :queue => queue, :workdir => FlowConfig.getjobdir(@workdir), :store => store, :type => type)
         newjob.save
         if id != nil
           @@log.debug "Add link "+id.to_s+"->"+newjob.id.to_s+","+newjob.node

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: manband
 version: !ruby/object:Gem::Version 
-  hash: 5
+  hash: 3
   prerelease: 
   segments: 
   - 0
-  - 6
-  - 1
-  version: 0.6.1
+  - 7
+  - 0
+  version: 0.7.0
 platform: ruby
 authors: 
 - Olivier Sallou
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-07-17 00:00:00 Z
+date: 2012-07-18 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: amqp
@@ -125,6 +125,7 @@ extra_rdoc_files:
 - README
 - LICENSE
 files: 
+- bin/env.sh
 - bin/manband.rb
 - bin/remoteband.rb
 - bin/jobhandler.rb