manband 0.4.0

data/lib/manband/workflow.rb

@@ -0,0 +1,274 @@
+require 'data_mapper'
+require 'dm-migrations'
+require 'yaml'
+require 'logger'
+require 'fileutils'
+require 'json'
+require 'manband/flowconfig.rb'
+require 'manband/bandmanager.rb'
+require 'manband/job.rb'
+
+#DataMapper.setup(:default, 'sqlite:///home/osallou/Desktop/genflow/project.db')
+DataMapper.setup(:default, ENV['MYSQL_URL'])
+
+
+
+class WorkFlow
+  include DataMapper::Resource
+
+  @@log = Logger.new(STDOUT)
+  @@log.level = Logger::DEBUG
+
+  property :id,         Serial    # An auto-increment integer key
+  property :uid,   Text  # user id
+  property :name,      String    # A varchar type string, for short strings
+  property :description,       Text      # A text block, for longer string data.
+  property :file, String   # path to the worflow file definition
+  property :created_at, DateTime  # A DateTime, for any date you might like.
+  property :terminated_at, DateTime # Termination date
+  property :terminals, Integer
+  property :status, Integer  # Running, Suspended, Error, Finished
+  property :workdir, Text # Workflow work dir
+  property :vars, Text, :default => "{}" # Runtime vars
+  property :bucket, String, :default => "manband" # Bucket name for S3 storage
+  property :parent, Integer, :default => 0  # Parent workflow id, none by default
+  property :instances, Integer, :default => 0 # Number of sub workflows, if any
+
+  def isover?
+    # decrement terminals
+    @terminals = @terminals - 1
+    curw = nil
+    # Use lock if MYSQL
+    if ENV['MYSQL_URL'].include?("mysql")
+      DataMapper.repository(:default).adapter.execute("UPDATE work_flows SET terminals = terminals - 1 WHERE id="+@id.to_s);
+      curw = WorkFlow.get(@id)
+    else
+      curw = WorkFlow.get(@id)
+      curw.update(:terminals => @terminals)
+    end
+    if curw.terminals <=0
+      @@log.info "Workflow "+@id.to_s+" is over"
+      curw.update(:terminated_at => Time.now, :status => STATUS_OVER)
+      return true
+    end
+    return false
+  end
+
+  # Return an array of node names
+  def getnextjobs(curnode)
+    #fworkflow = YAML.load_file(@file)
+    fworkflow = BandManager.load(@file)
+    if fworkflow==nil
+      return nil
+    end
+    if fworkflow["workflow"][curnode]["next"] == nil
+      @@log.debug "no next node, this branch is over"
+      #isover?
+      return nil
+    end
+    nexts = fworkflow["workflow"][curnode]["next"].split(',')
+    if nexts[0].empty?
+      @@log.debug "no next node, this branch is over"
+      #isover?
+      return nil
+    end
+    return nexts
+  end
+
+  def parse(curnode, id = nil)
+    #fworkflow = YAML.load_file(@file)
+    fworkflow = BandManager.load(@file)
+    if fworkflow==nil
+      return nil
+    end
+    jobs = getnextjobs(curnode)
+    if jobs == nil
+      return
+    end
+    jobs.each do |job|
+      if Job.count(:wid => @id, :node => job) == 0
+        queue = ""
+        if fworkflow["workflow"][job]["queue"]!=nil
+          queue = fworkflow["workflow"][job]["queue"]
+        end
+        status = STATUS_NEW
+        if fworkflow["workflow"][job]["breakpoint"]!=nil
+          @@log.debug "Node "+job+" has a breakpoint set"
+          status = STATUS_SUSPEND
+        end
+        store = STORE_NO
+        if (!fworkflow["options"].nil? && fworkflow["options"]["store"] == "all") || fworkflow["workflow"][job]["store"] == true
+          @@log.debug "Add store option for job "+job
+          store = STORE_DO
+        end
+        workdir = FlowConfig.getjobdir(@workdir)
+        if curnode == "root"
+          workdir = self.workdir + "/root";
+        end
+        newjob = Job.new(:wid => @id, :node => job, :command => "", :status => status, :instances => 0, :maxinstances => 0, :queue => queue, :workdir => FlowConfig.getjobdir(@workdir), :store => store)
+        newjob.save
+        if id != nil
+          @@log.debug "Add link "+id.to_s+"->"+newjob.id.to_s+","+newjob.node
+          link = JobLink.new(:wid => @id, :from => id, :to => newjob.id)
+          link.save
+        end
+        parse(job,newjob.id)
+      else
+        if id!=nil
+          # Already declared, just add link
+          linkedjob = Job.first(:wid => @id, :node => job)
+          @@log.debug "Add link "+id.to_s+"->"+linkedjob.id.to_s+","+linkedjob.node
+          link = JobLink.new(:wid => @id, :from => id, :to => linkedjob.id)
+          link.save
+        end
+      end
+    end
+  end
+
+  # Update in command the runtime vars
+  # return modified command
+  def setruntimevars(varexpr,command)
+    newcommand = String.new(command)
+    runtimevars = JSON.parse(self.vars)
+    if runtimevars[varexpr]!=nil
+      @@log.debug "Replace in command #var."+varexpr+"# by "+runtimevars[varexpr]
+      newcommand["#var."+varexpr+"#"]= runtimevars[varexpr]
+    else
+      # runtime var is node defined
+      return nil
+    end
+    return newcommand
+  end
+
+  # Return command for the node in the workflow
+  def getnodecommand(curnode)
+    #fworkflow = YAML.load_file(@file)
+    fworkflow = BandManager.load(@file)
+    if fworkflow==nil
+      return nil
+    end
+    maincommand = fworkflow["workflow"][curnode]["command"]
+    # Manage node regexp
+    exprs = maincommand.scan(/#(node|var)\.(.*?)#/)
+    if exprs.length == 0
+      return [ fworkflow["workflow"][curnode]["command"] ]
+    end
+    subnodefilelist = Hash.new
+    multinode=nil
+    for reg in 0..exprs.length-1
+      @@log.debug "Expr "+": "+exprs[reg][0]+" "+exprs[reg][1]
+      if exprs[reg][0] == "var"
+        maincommand = setruntimevars(exprs[reg][1],maincommand)
+        if maincommand == nil
+          @@log.error "Runtime var "+exprs[reg][1]+" is not defined for  workflow "+@id.to_s
+          return nil
+        end
+      end
+      if exprs[reg][0] == "node"
+        # Get regexp for this node
+        subnode = exprs[reg][1]
+        # If regexp is empty, we jsut want the directory
+        if fworkflow["workflow"][curnode][subnode]==nil
+          return nil
+        end
+        if fworkflow["workflow"][curnode][subnode]['regexp'].strip == ''
+          subnoderegexp = nil
+        else
+          subnoderegexp = Regexp.new(fworkflow["workflow"][curnode][subnode]['regexp'])
+        end
+        # List all files for this node regexp
+        if subnode.match(/local/)
+          # Local files reference
+          nodepath = fworkflow["workflow"][curnode][subnode]['url']
+        else
+          # An other node reference
+          if subnode == 'root'
+            nodepath = self.workdir+'/root';
+          else
+            subjob = Job.first(:wid => @id, :node => subnode)
+            nodepath = subjob.workdir
+          end
+        end
+        
+        filelist = Array.new
+        if subnoderegexp == nil
+          filelist.push(nodepath+"/")
+        else
+          if !File.exists?(nodepath)
+            @@log.error("path does not exists!")
+            return nil
+          end
+          Dir.new(nodepath).entries.each do |n|
+            if subnoderegexp.match(n)
+              filelist.push(nodepath+"/"+n)
+            end
+          end
+        end
+        @@log.debug "File list "+subnode+": "+filelist.to_s
+        # update file list per node regexp in the command
+        subnodefilelist[subnode]=filelist 
+        if filelist.length>1
+          if multinode!=nil
+            # We do not support multiple lists in same command (N*N*N*....)
+            return nil
+          else
+            multinode = subnode
+          end
+        end
+      end
+    end
+
+    # TODO manage local URI regexp (or remote)
+    # If remote: list then download
+
+    # Now create an array of command with file lists substitution
+    commands = Array.new
+    subnodefilelist.each do |key,slist|
+      if slist[0] == nil
+        return nil
+      end
+      if key != multinode
+        # replace, one match only allowed here
+        maincommand["#node."+key+"#"]= slist[0]
+      end
+    end
+    if multinode!=nil
+      @@log.debug "Multinode: "+multinode
+      subnodefilelist[multinode].each do |file|
+        newcommand = String.new(maincommand)
+        @@log.debug "Command: "+newcommand
+        newcommand["#node."+multinode+"#"]= file
+        commands.push(newcommand)
+      end
+    else
+      commands.push(maincommand)
+    end
+    @@log.debug "Commands: "+commands.to_s
+    return commands
+  end
+
+  def clean
+    workflow = WorkFlow.get(@id)
+    if workflow.workdir == nil
+      return
+    end
+    if File.directory? workflow.workdir
+      FileUtils.rm_rf(workflow.workdir)
+    end
+  end
+
+end
+
+# Table hosting workflow handler messages
+class BandMessage
+  include DataMapper::Resource
+
+  property :id,         Serial    # An auto-increment integer key
+  property :wid,     Integer     # Workflow id
+  property :message,  Text # Message text
+
+end
+
+DataMapper.finalize
+DataMapper.auto_upgrade!
+

metadata

@@ -0,0 +1,175 @@
+--- !ruby/object:Gem::Specification 
+name: manband
+version: !ruby/object:Gem::Version 
+  hash: 15
+  prerelease: 
+  segments: 
+  - 0
+  - 4
+  - 0
+  version: 0.4.0
+platform: ruby
+authors: 
+- Olivier Sallou
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-05-10 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: amqp
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: json
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: data_mapper
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: dm-mysql-adapter
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: zip
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: mb-minion
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id006
+- !ruby/object:Gem::Dependency 
+  name: mb-aws-s3
+  prerelease: false
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id007
+description: 
+email: olivier.sallou@irisa.fr
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README
+- LICENSE
+files: 
+- bin/manband.rb
+- bin/remoteband.rb
+- bin/jobhandler.rb
+- bin/workflowhandler.rb
+- bin/userband.rb
+- lib/manband/store.rb
+- lib/manband/bandmanager.rb
+- lib/manband/user.rb
+- lib/manband/workflow.rb
+- lib/manband/job.rb
+- lib/manband/flowconfig.rb
+- README
+- LICENSE
+homepage: http://gforge.inria.fr/projects/manband
+licenses: 
+- CeCILL-C
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.15
+signing_key: 
+specification_version: 3
+summary: Workflow orchestrator based on RabbitMQ
+test_files: []
+