manband 0.4.0

data/lib/manband/bandmanager.rb

@@ -0,0 +1,176 @@
+require 'rubygems'
+require 'yaml'
+require 'optparse'
+require 'amqp'
+require 'json'
+require 'mb-minion'
+require 'fileutils'
+
+include Minion
+
+require 'manband/workflow.rb'
+require 'manband/flowconfig.rb'
+
+# Library to launch new workflows
+class BandManager
+
+  @@log = Logger.new(STDOUT)
+  @@log.level = Logger::DEBUG
+
+
+  # Load workflow file
+  def self.load(wfile) 
+    if !File.exists?(wfile)
+      @@log.error "Workflow file "+wfile+" does not exist!"
+      return nil
+    end
+    begin
+      workflow = YAML.load_file(wfile)
+    rescue
+      @@log.error "Error while load file"
+      return nil
+    end
+    return workflow
+  end
+
+  # Add a new workflow based on input workflow object
+  # wfile: workflow input file
+  # var: optional runtime variables
+  # uid: optional user id
+  # debug: activate debug mode, do not execute the commands
+  # @return workflow id
+  def self.add(wfile, var = [], uid='admin', bucket='manband' , debug=false)
+    rvariables = Hash.new
+    #fworkflow = YAML.load_file(wfile)
+    fworkflow = self.load(wfile)
+    if fworkflow==nil
+      return nil
+    end
+    terminals = 0
+    fworkflow["workflow"].each do |node|
+      if node[1]["command"]!=nil
+        # First init with vars defined in workflow
+        exprs = node[1]["command"].scan(/#var\.(.*?)#/)
+        if exprs.length > 0
+          for reg in 0..exprs.length-1
+            rvariables[exprs[reg][0]] = nil
+          end
+        end
+        if node[1]["next"]==nil
+          # This is a terminal node
+          terminals += 1
+          @@log.debug "terminal: "+node[0]
+        end
+      end
+    end
+    @@log.debug "nb terminals: "+terminals.to_s
+
+
+    if var!=nil
+      var.each do |rvariable|
+        rvardef = rvariable.split('=')
+        rvariables[rvardef[0]]=rvardef[1]
+      end
+      @@log.debug "Using runtime variables: "+rvariables.to_json
+    end
+
+    workflow = WorkFlow.new(:uid => uid , :name => fworkflow["workflow"]["name"], :description => fworkflow["workflow"]["description"], :created_at => Time.now, :file => wfile, :terminals => terminals, :status => STATUS_NEW, :workdir => FlowConfig.getjobdir(), :vars => rvariables.to_json, :bucket => bucket)
+
+    err =workflow.save
+
+    if err == false
+      return nil
+    end
+
+    instances = 0
+    regexp = nil
+    # Check if workflow need to be run for many files
+    if fworkflow["workflow"]["root"]["url"]!=nil
+      nodepath = fworkflow["workflow"]["root"]["url"]
+      if fworkflow["workflow"]["root"]["regexp"]!=nil
+        regexp = Regexp.new(fworkflow["workflow"]["root"]["regexp"])
+      end
+        filelist = Array.new
+        Dir.new(nodepath).entries.each do |n|
+          @@log.debug "Test file "+n
+          if (regexp==nil || regexp.match(n)) && File.file?(nodepath+"/"+n)
+            @@log.debug "New sub workflow"
+            instances += 1
+            subworkflow = WorkFlow.new(:uid => uid , :name => fworkflow["workflow"]["name"], :description => fworkflow["workflow"]["description"], :created_at => Time.now, :file => wfile, :terminals => terminals, :status => STATUS_NEW, :workdir => FlowConfig.getjobdir(), :vars => rvariables.to_json, :bucket => bucket, :parent => workflow.id)
+            err = subworkflow.save
+            if !File.exists?(nodepath+"/root")
+              FileUtils.mkdir_p  subworkflow.workdir+"/root"
+            end
+            File.symlink(nodepath+"/"+n,subworkflow.workdir+"/root/"+n)
+            @@log.debug("Add new sub workflow "+subworkflow.id.to_s)            
+          end
+        end
+     end
+     @@log.debug "Number of instances: "+instances.to_s
+     workflow.update(:instances => instances)
+
+
+    return workflow.id
+  end
+
+  # Start a workflow in state NEW
+  # In debug mode sends a OP_SKIP instead of OP_START
+  def self.start(id,debug=false)
+    workflow = WorkFlow.get(id)
+    wlist = Array.new
+    wflows = WorkFlow.all(:parent => id)
+    if wflows == nil || wflows.length==0
+      wlist.push(id)
+    else
+      wflows.each do |wflow|
+        wlist.push wflow.id
+      end
+    end
+
+    @@log.debug "Execute workflow list: "+wlist.to_s
+    wlist.each do |wflow|
+      workflow = WorkFlow.get(wflow)
+      rootjob = Job.new(:wid => workflow.id, :node => "root", :command => "", :status => STATUS_NEW, :instances => 0, :maxinstances => 0, :workdir => '')
+      rootjob.save
+      workflow.parse('root',rootjob.id)
+      # Request workflow management
+      msg = '{ "id" : "'+workflow.id.to_s+'", "root" : "'+rootjob.id.to_s+'"}'
+      if debug
+        Minion.enqueue("manband.master", { "operation" => OP_SKIP, "msg" => msg })
+      else
+      Minion.enqueue("manband.master", { "operation" => OP_START, "msg" => msg })
+      end
+    end
+  end
+
+  # Execute a workflow from an other one
+  def self.launchclone(id)
+    workflow = WorkFlow.get(id)
+    newworkflow = WorkFlow.new(:uid => workflow.uid , :name => workflow.name, :description => workflow.description, :created_at => Time.now, :file => workflow.file, :terminals => workflow.terminals, :status => STATUS_NEW, :workdir => FlowConfig.getjobdir(), :vars => workflow.vars, :bucket => workflow.bucket)
+    newworkflow.save
+    rootjob = Job.new(:wid => newworkflow.id, :node => "root", :command => "", :status => STATUS_NEW, :instances => 0, :maxinstances => 0, :workdir => '')
+    rootjob.save
+    newworkflow.parse('root',rootjob.id)
+    jobmsg = '{ "id" : "'+newworkflow.id.to_s+'", "root" : "'+rootjob.id.to_s+'"}'
+    job = Job.new(:wid => newworkflow.id, :node => "fake", :command => "", :status => STATUS_FAKE, :instances => 0, :workdir => '')
+    job.sendmessage(OP_START,jobmsg)
+    return newworkflow.id
+  end
+
+  # Launch a new workflow based on input workflow object
+  # wfile: workflow input file
+  #
+  # uid: optional user id
+  # debug: activate debug mode, do not execute the commands
+  # @return workflow id
+  def self.launch(wfile, var = [], uid='admin', bucket='manband' , debug=false)
+    workflow = self.add(wfile,var,uid,bucket,debug)
+    if workflow==nil
+      return nil
+    end
+    @@log.debug "Start workflow "+workflow.to_s
+    self.start(workflow,debug)
+    return workflow
+  end
+
+end

data/lib/manband/flowconfig.rb

@@ -0,0 +1,87 @@
+require 'uuid'
+require 'data_mapper'
+require 'dm-migrations'
+
+STATUS_SKIP = -2
+STATUS_FAKE = -1
+STATUS_NEW = 0
+STATUS_RUNNING = 1
+STATUS_OVER = 2
+STATUS_SUSPEND = 3 # SUSPEND mode for a job means a job is over and paused
+STATUS_ERROR = 4
+
+STORE_NO = -1
+STORE_DO = 0
+STORE_RUN = 1
+STORE_OVER = 2
+STORE_ERROR = 4
+
+OP_NEW = "new"
+OP_START = "start"
+OP_FINISH = "finish"
+OP_ERROR = "error"
+OP_WSUSPEND = "workflowsuspend"
+OP_JSUSPEND = "jobsuspend"
+OP_RUN = "run"
+OP_WRESUME = "workflowresume"
+OP_JRESUME = "jobresume"
+OP_SKIP = "skip" # For if conditions, go through jobs but skip execution
+OP_CLEAN = "clean"  # Delete work dirs of workflow
+OP_DESTROY = "destroy"  # Delete workflow and its work dirs
+OP_STORE = "store" # Store result to S3
+
+# Base config
+class FlowConfig
+   @@workdir='/tmp'
+   @@s3host = 'genokvm4.genouest.org'
+   @@s3port = '8773'
+   @@s3path = '/services/Walrus'
+
+   @@uploaddir = '/tmp/upload'
+
+   def self.uploaddir
+     return @@uploaddir
+   end
+
+   def self.s3host
+     return @@s3host
+   end
+
+   def self.sets3(host,port= '8773',path='/services/Walrus')
+     @@s3host = host
+     @@s3port = port
+     @@s3path = path
+   end
+  
+   def self.s3port
+     return @@s3port
+   end
+
+   def self.s3path
+     return @@s3path
+   end
+
+   def self.workdir
+     return @@workdir
+   end
+
+   def self.setuploaddir(directory)
+     @@uploaddir = directory
+   end
+
+   def self.setworkdir(directory)
+     @@workdir=directory
+   end
+
+   def self.getjobdir(workflowdir = nil)
+     uuid = UUID.new
+     if workflowdir == nil
+       return @@workdir+"/"+uuid.generate.to_s
+     else
+       return workflowdir+"/"+uuid.generate.to_s 
+     end
+   end
+
+
+end
+

data/lib/manband/job.rb

@@ -0,0 +1,282 @@
+require 'data_mapper'
+require 'dm-migrations'
+require 'yaml'
+require 'manband/workflow.rb'
+require 'manband/flowconfig.rb'
+require 'mb-minion'
+require 'logger'
+
+include Minion
+
+#DataMapper.setup(:default, 'sqlite:///home/osallou/Desktop/genflow/project.db')
+DataMapper.setup(:default, ENV['MYSQL_URL'])
+
+#TODO manage IF operation and resutl.
+# Exit code = 0 selects first node, other code select second node
+# Should inform with a finish "code: exitcode"  and manage it in workflowhandler to RUN selected node and SKIP other node
+
+class JobLink
+  include DataMapper::Resource
+
+  property :id, Serial
+  property :wid, Integer # Workflow id
+  property :from, Integer # Job origin id
+  property :to, Integer # Job destination id
+
+end
+
+class Job
+  include DataMapper::Resource
+
+  @@debug = false
+
+  def self.debug(mode)
+    @@debug = mode
+  end
+
+  @@log = Logger.new(STDOUT)
+  @@log.level = Logger::DEBUG
+
+  property :id, Serial
+  property :wid, Integer # id of the worflow
+  property :node, String  # id of the node
+  property :command, Text # Array of command lines to execute (1 per matching regexp)
+  property :status, Integer  # is job in execution, finished or failed?
+  property :instances , Integer # Current number of job instance for this command according to pattern
+  property :maxinstances, Integer # Number of instances launched
+  property :workdir, Text # working dir for this job
+  property :handler, String # job handler managing this job
+  property :queue, String, :default => ""  # Queue to handle the job
+  property :error, Text, :default => "[]" # JSON Array of error, per instance
+  property :store, Integer, :default => STORE_NO # Storage status
+
+  # Update job status and launch the command
+  def run(curhandler,instance=0)
+    # Send run command, possibly multiple ones according to pattern
+    # Command would send a message when over
+    curjob = Job.get(@id)
+    if @status != STATUS_SUSPEND
+      @status=STATUS_RUNNING
+      curjob.update(:status => STATUS_RUNNING, :handler => curhandler.to_s)
+    else
+      curjob.update(:handler => curhandler.to_s)
+    end
+    workdir = curjob.workdir
+    if instance>0
+      workdir = workdir + "/node" + instance.to_s
+    end
+    err = runcommand(workdir,instance)
+    if err == nil || err == false
+      # An error occured
+      jobmsg = '{ "workflow" : "'+@wid.to_s+'" ,  "node" : "'+@node+'", "id" : "'+@id.to_s+'", "handler" : "'+curhandler.to_s+'", "instance" : "'+instance.to_s+'" }'
+      sendmessage(OP_ERROR,jobmsg)
+    else
+      if curjob.store == STORE_DO || curjob.store == STORE_ERROR
+        workflow = WorkFlow.get(curjob.wid)
+        jobmsg = '{ "id" : "'+curjob.id.to_s+'", "bucket" : "'+workflow.bucket+'" }'
+        sendmessage(OP_STORE,jobmsg)
+      end
+      jobmsg = '{ "workflow" : "'+@wid.to_s+'" ,  "node" : "'+@node+'", "id" : "'+@id.to_s+'", "handler" : "'+curhandler.to_s+'", "instance" : "'+instance.to_s+'" }'
+      sendmessage(OP_FINISH,jobmsg)
+    end
+  end
+
+  # Skip treatment, just answer
+  def skip(curhandler)
+    curjob = Job.get(@id)
+    curjob.update(:handler => curhandler.to_s)
+    jobmsg = '{ "workflow" : "'+@wid.to_s+'" ,  "node" : "'+@node+'", "id" : "'+@id.to_s+'", "handler" : "'+curhandler.to_s+'" }'
+    sendmessage(OP_FINISH,jobmsg)
+  end
+
+  # Execute locally the command
+  def runcommand(workdir,instance)
+    initcmd = "AMQP_URL="" && MYSQL_URL="" && mkdir -p "+workdir+" && cd "+workdir+" && WORKDIR="+workdir+" && " 
+    curjob = Job.get(@id)
+    command = JSON.parse(curjob.command)
+    if command.length > 1
+      system(initcmd+command[0])
+    else
+      system(initcmd+command[instance-1])
+    end
+  end
+
+  # Change instance counter
+  # If workflow is in suspend status, suspend the job at the end of its treatment
+  def finish
+    if @status == STATUS_SKIP
+      return
+    end
+    workflow = WorkFlow.get(@wid)
+    if workflow.status == STATUS_SUSPEND
+      @status= STATUS_SUSPEND
+      DataMapper.repository(:default).adapter.execute('UPDATE jobs SET instances = instances + 1, status = '+STATUS_SUSPEND+' WHERE id='+@id.to_s);
+      curjob = Job.get(@id)
+      #curjob = Job.get(@id)
+      #instancesplusone = curjob.instances + 1
+      #curjob.update(:status => STATUS_SUSPEND, :instances => instancesplusone)
+    elsif @status == STATUS_SUSPEND
+      # Job set in suspend mode (breakpoint), keep in this status
+      #instancesplusone = curjob.instances + 1
+      #curjob.update(:instances => instancesplusone) 
+      DataMapper.repository(:default).adapter.execute('UPDATE jobs SET instances = instances + 1 WHERE id='+@id.to_s);
+      curjob = Job.get(@id)
+    else
+      @status= STATUS_OVER
+      DataMapper.repository(:default).adapter.execute('UPDATE jobs SET instances = instances + 1  WHERE id='+@id.to_s);
+      # TODO add a lock for update
+      curjob = Job.get(@id)
+      if curjob.instances >= curjob.maxinstances
+        curjob.update(:status => STATUS_OVER)
+      else
+      end
+    end
+  end
+
+  # Compare sintance counter to max instances
+  def isover?
+    if @status == STATUS_OVER
+      return true
+    elsif @status == STATUS_SKIP
+      return true
+    else
+      return false
+    end
+  end
+
+  # Resume a suspended job
+  # Update its status and run next jobs
+  def resume
+    workflow = WorkFlow.get(@wid)
+    # if status = suspend, continue to next
+    # if status = error, restart job
+    if @status == STATUS_SUSPEND
+      @@log.debug "Resume from suspended job "+@id.to_s+" for workflow "+@wid.to_s
+      @status= STATUS_OVER
+      curjob = Job.get(@id)
+      curjob.update(:status => STATUS_OVER)
+      runnext
+    elsif @status == STATUS_ERROR
+      @@log.debug "Resume from error job "+@id.to_s+" for workflow "+@wid.to_s
+      @status = STATUS_RUNNING
+      curjob = Job.get(@id)
+      commands = workflow.getnodecommand(curjob.node)
+      if commands == nil
+        @@log.error "Could not get command for node "+curjob.node
+        return
+      end
+      # Restart whole job, use new tmp dir, reset errors
+      curjob.update(:status => STATUS_RUNNING, :command => commands.to_json, :instances => 0, :workdir => FlowConfig.getjobdir(workflow.workdir), :error => "[]", :maxinstances => commands.length)
+      i = 0
+      if commands.length > 1
+        i = 1
+      end
+      commands.each do |command|
+        jobmsg = '{ "workflow" : "'+@wid.to_s+'" ,  "node" : "'+@node+'", "id" : "'+@id.to_s+'", "instance" : "'+i.to_s+'" }'
+        i += 1
+        # send message
+        sendmessage(OP_RUN,jobmsg,curjob.queue)
+      end
+    end
+  end
+
+  def error!(instance = 0)
+    @status= STATUS_ERROR
+    curjob = Job.get(@id)
+    err = JSON.parse(curjob.error)
+    err.push(instance)
+    curjob.update(:status => STATUS_ERROR, :error => err.to_json)
+    workflow = WorkFlow.get(@wid)
+    workflow.update(:status => STATUS_ERROR)
+  end
+
+  # Run next jobs
+  def runnext(skip=false)
+    workflow = WorkFlow.get(@wid)
+    # Are all previous jobs over?
+    previous = JobLink.all(:to => @id)
+    #nexts = workflow.getnextjobs(@node)
+    if previous!=nil
+      @@log.debug "Checking previous jobs: "+previous.length.to_s
+    end
+    if previous == nil || previous.length==0
+      # This is fine, must be a root node
+    elsif previous.length==1
+      # Only 1 parent. As we got the message, previous is over
+    else
+      # Check each job
+      previous.each do |link|
+        pjob = Job.get(link.from)
+        if pjob.status != STATUS_OVER
+          @@log.debug "At least one previous job is not over ("+pjob.id.to_s+"), wait for next message"
+          return false
+        end
+      end
+      @@log.debug "All previous jobs are over, continue...."
+    end
+
+    # Look at next jobs
+    nexts = JobLink.all(:from => @id)
+    #nexts = workflow.getnextjobs(@node)
+    if nexts == nil || nexts.length==0
+      workflow.isover?
+      return true
+    end
+    nexts.each do |link|
+    job = Job.get(link.to)
+    #nexts.each do |nextnode|
+      #nextnode.strip!
+      #queue = workflow.getnodequeue(nextnode)
+      if skip==true  || @status == STATUS_SKIP
+        jobstatus = STATUS_SKIP
+        operation = OP_SKIP
+        commands = [ "skipcommand"]
+      else
+        jobstatus = STATUS_NEW
+        operation = OP_RUN
+        commands = workflow.getnodecommand(job.node) 
+      end
+      if commands == nil
+        @@log.error "Could not get command for node "+job.id.to_s
+        job.update(:command => "", :status => STATUS_ERROR, :instances => 0, :maxinstances => 0, :workdir => FlowConfig.getjobdir(workflow.workdir))
+        workflow.update(:status => STATUS_ERROR)
+        return
+      end
+      @@log.debug "New job command: "+commands.to_s
+      job.update(:command => commands.to_json, :status => jobstatus, :instances => 0, :maxinstances => commands.length, :workdir => FlowConfig.getjobdir(workflow.workdir))
+      #err = job.save
+      i=0
+      # If multiple instances, differenciate and start at 1
+      if commands.length > 1
+        i = 1
+      end
+      commands.each do |command| 
+        jobmsg = '{ "workflow" : "'+job.wid.to_s+'" ,  "node" : "'+job.node+'", "id" : "'+job.id.to_s+'", "instance" : "'+i.to_s+'" }'
+        i += 1
+        # send message
+        sendmessage(operation,jobmsg,job.queue)
+      end
+    end
+  end
+
+   def sendmessage(operation,msg,jobqueue='')
+     queue = "manband.master"
+     if operation == OP_RUN || operation == OP_SKIP || operation == OP_DESTROY || operation == OP_CLEAN|| operation == OP_STORE
+       queue = "manband.node"+jobqueue
+     end
+     if queue != nil
+       Minion.enqueue(queue, { "operation" => operation, "msg" => msg })
+       if @@debug == true
+         bmsg = BandMessage.new(:wid => @wid, :message => '{ "operation" => '+operation+', "msg" => '+msg+' }' )
+         bmsg.save
+       end
+     end
+   end
+
+end
+
+
+
+DataMapper.finalize
+DataMapper.auto_upgrade!
+

data/lib/manband/store.rb

@@ -0,0 +1,77 @@
+require 'aws/s3.rb'
+require 'logger'
+require 'zip/zip'
+require 'find'
+require 'fileutils'
+
+require 'manband/flowconfig.rb'
+require 'manband/user.rb'
+
+
+class Storeband
+
+  @@log = Logger.new(STDOUT)
+  @@log.level = Logger::DEBUG
+
+  # Compress a directory
+  def compress(dir,name)
+    Zip::ZipFile.open(name, Zip::ZipFile::CREATE)do |zipfile|
+      Find.find(dir) do |path|
+        Find.prune if File.basename(path)[0] == ?.
+        dest = /#{dir}\/(\w.*)/.match(path)
+        # Skip files if they exists
+        begin
+          zipfile.add(dest[1],path) if dest
+        rescue Zip::ZipEntryExistsError
+        end
+      end
+    end
+  end
+
+  # Store a job workdir to S3
+  def store(job,uid,bucket="manband")
+    user = User.get(uid)
+    if user == nil
+      job.update(:store => STORE_ERROR)
+      return false
+    end
+    if user.s3_access.nil? || user.s3_secret.nil?
+      job.update(:store => STORE_ERROR)
+      return false
+    end
+    job.update(:store => STORE_RUN)
+    zipfile = "manband-"+job.wid.to_s+"-"+job.node+".zip"
+    if File.exists?(FlowConfig.workdir+"/"+zipfile)
+      File.delete(FlowConfig.workdir+"/"+zipfile)
+    end
+    begin
+      # compress
+      @@log.debug "Compress to "+FlowConfig.workdir+"/"+zipfile
+      compress(job.workdir, FlowConfig.workdir+"/"+zipfile)
+      # Send file
+      sends3(FlowConfig.workdir+"/"+zipfile,bucket,user.s3_access,user.s3_secret)
+    rescue
+      @@log.error "An error occured during S3 operation: "+job.id.to_s
+      job.update(:store => STORE_ERROR)
+      return false
+    end
+    job.update(:store => STORE_OVER)
+  end
+
+  def sends3(name,bucket,access,secret)
+    @@log.debug "connect to s3"
+    AWS::S3::Base.establish_connection!(
+      :access_key_id     => access,
+      :secret_access_key => secret,
+      :server => FlowConfig.s3host,
+      :port => FlowConfig.s3port,
+      :path => FlowConfig.s3path
+    )
+    fbase = File.basename(name)
+
+    AWS::S3::S3Object.store(fbase, open(name), bucket)
+    @@log.debug "object stored"
+  end
+
+
+end

data/lib/manband/user.rb

@@ -0,0 +1,47 @@
+require 'uuid'
+require 'data_mapper'
+require 'dm-migrations'
+
+require 'manband/flowconfig.rb'
+
+#DataMapper.setup(:default, 'sqlite:///tmp/project.db')
+DataMapper.setup(:default, ENV['MYSQL_URL'])
+
+class User
+  include DataMapper::Resource
+
+  @@admin = 'admin'
+
+  property :login, String, :key => true # User login
+  property :password, String # USer password
+  property :s3_access, String # User id key
+  property :s3_secret, String # Secret key
+
+  def self.init
+    admin = User.get(@@admin)
+    if admin == nil
+      uuid = UUID.new
+      admin = User.new(:login => @@admin, :password => Digest::SHA1.hexdigest('admin'))
+      admin.save
+    end
+  end
+
+  def authenticate(password)
+    shapwd = Digest::SHA1.hexdigest(password) 
+    if self.password == shapwd
+      return true
+    end
+    return false; 
+  end
+
+  def self.admin
+    return @@admin
+  end
+
+end
+
+DataMapper.finalize
+DataMapper.auto_upgrade!
+
+User.init()
+