manband 0.4.0 → 0.5.0

data/bin/jobhandler.rb

@@ -7,6 +7,13 @@ require 'mb-minion'
 
 include Minion
 
+# The program manages job commands. Master nodes (workflow handler) sends
+# messages to jobhandler instances. Each instance manages one and
+# only one message at a time. Jobhnadlers are also in charge of storage to S3.
+# Author: Olivier Sallou <olivier.sallou@irisa.fr>
+# Copyright: 2012, IRISA
+
+
 if ENV["MYSQL_URL"]==nil
   puts "MYSQL_URL environment variable is not set exiting..."
   exit 1

data/bin/manband.rb

@@ -14,6 +14,10 @@ require 'manband/flowconfig.rb'
 require 'manband/job.rb'
 require 'manband/bandmanager.rb'
 
+# The program sends a workflow execution request to manband orchestrator.
+# Author: Olivier Sallou <olivier.sallou@irisa.fr>
+# Copyright: 2012, IRISA
+
 log = Logger.new(STDOUT)
 log.level = Logger::INFO
 

data/bin/remoteband.rb

@@ -7,6 +7,11 @@ require 'mb-minion'
 
 include Minion
 
+# The program sends a workflow execution request to manband orchestrator.
+# This program does not need database credentials but does not get workflow id in return
+# Author: Olivier Sallou <olivier.sallou@irisa.fr>
+# Copyright: 2012, IRISA
+
 
 log = Logger.new(STDOUT)
 log.level = Logger::INFO

data/bin/userband.rb

@@ -8,6 +8,12 @@ $:.push File.expand_path("../lib")
 
 require 'manband/flowconfig.rb'
 
+# The program is used to manage users (add, update) in the database.
+# Users are used for web interface authentification and S3 credentials.
+# Author: Olivier Sallou <olivier.sallou@irisa.fr>
+# Copyright: 2012, IRISA
+
+
 log = Logger.new(STDOUT)
 log.level = Logger::INFO
 

data/bin/workflowhandler.rb

@@ -7,6 +7,15 @@ require 'mb-minion'
 
 include Minion
 
+# The program s the main orchestrator of the workflow.
+# It starts with a workflow execution request and sends messages to job
+# handlers according to the input workflow.
+# It is possible to have multipe workflow handler, sharing messages for 
+# the same workflow, or to handler a high number of workflow requests.
+# Author: Olivier Sallou <olivier.sallou@irisa.fr>
+# Copyright: 2012, IRISA
+
+
 if ENV["MYSQL_URL"]==nil
   puts "MYSQL_URL environment variable is not set exiting..."
   exit 1
@@ -52,7 +61,7 @@ optparse.parse!
 
 userconf = File.expand_path("~")+"/.manband"
 if @@options[:conf]==nil && File.exists?(userconf)
-  options[:conf]=userconf
+  @@options[:conf]=userconf
 end
 
 if @@options[:conf]!=nil

data/lib/manband/bandmanager.rb

@@ -11,14 +11,14 @@ include Minion
 require 'manband/workflow.rb'
 require 'manband/flowconfig.rb'
 
-# Library to launch new workflows
+# This class is used to launch new workflows
 class BandManager
 
   @@log = Logger.new(STDOUT)
   @@log.level = Logger::DEBUG
 
 
-  # Load workflow file
+  # Loads a workflow file
   def self.load(wfile) 
     if !File.exists?(wfile)
       @@log.error "Workflow file "+wfile+" does not exist!"
@@ -143,7 +143,7 @@ class BandManager
     end
   end
 
-  # Execute a workflow from an other one
+  # Execute a workflow from an other one (clone it)
   def self.launchclone(id)
     workflow = WorkFlow.get(id)
     newworkflow = WorkFlow.new(:uid => workflow.uid , :name => workflow.name, :description => workflow.description, :created_at => Time.now, :file => workflow.file, :terminals => workflow.terminals, :status => STATUS_NEW, :workdir => FlowConfig.getjobdir(), :vars => workflow.vars, :bucket => workflow.bucket)

data/lib/manband/flowconfig.rb

@@ -2,6 +2,9 @@ require 'uuid'
 require 'data_mapper'
 require 'dm-migrations'
 
+# Configuration library
+
+# Determines the workflow and node status
 STATUS_SKIP = -2
 STATUS_FAKE = -1
 STATUS_NEW = 0
@@ -10,12 +13,14 @@ STATUS_OVER = 2
 STATUS_SUSPEND = 3 # SUSPEND mode for a job means a job is over and paused
 STATUS_ERROR = 4
 
+# Determines if a job must be stored to S3, and its current status
 STORE_NO = -1
 STORE_DO = 0
 STORE_RUN = 1
 STORE_OVER = 2
 STORE_ERROR = 4
 
+# List of message operations
 OP_NEW = "new"
 OP_START = "start"
 OP_FINISH = "finish"
@@ -30,10 +35,10 @@ OP_CLEAN = "clean"  # Delete work dirs of workflow
 OP_DESTROY = "destroy"  # Delete workflow and its work dirs
 OP_STORE = "store" # Store result to S3
 
-# Base config
+# This class holds the base config
 class FlowConfig
    @@workdir='/tmp'
-   @@s3host = 'genokvm4.genouest.org'
+   @@s3host = 'localhost'
    @@s3port = '8773'
    @@s3path = '/services/Walrus'
 
@@ -47,6 +52,7 @@ class FlowConfig
      return @@s3host
    end
 
+   # Sets S3 storage parameters
    def self.sets3(host,port= '8773',path='/services/Walrus')
      @@s3host = host
      @@s3port = port
@@ -65,14 +71,21 @@ class FlowConfig
      return @@workdir
    end
 
+   # defines upload directory for webband
+   # It must be accessible by the handlers
    def self.setuploaddir(directory)
      @@uploaddir = directory
    end
 
+   # Defines the work directory. It must be shared between job
+   # and workflow handlers.
    def self.setworkdir(directory)
      @@workdir=directory
    end
 
+
+   # Returns a work directory for a job. Directory
+   # is based on a unique identifier.
    def self.getjobdir(workflowdir = nil)
      uuid = UUID.new
      if workflowdir == nil

data/lib/manband/job.rb

@@ -11,10 +11,7 @@ include Minion
 #DataMapper.setup(:default, 'sqlite:///home/osallou/Desktop/genflow/project.db')
 DataMapper.setup(:default, ENV['MYSQL_URL'])
 
-#TODO manage IF operation and resutl.
-# Exit code = 0 selects first node, other code select second node
-# Should inform with a finish "code: exitcode"  and manage it in workflowhandler to RUN selected node and SKIP other node
-
+# This class defines the link between the nodes in the workflow
 class JobLink
   include DataMapper::Resource
 
@@ -25,11 +22,14 @@ class JobLink
 
 end
 
+# This class  manages the job execution and its status on a job handler.
 class Job
   include DataMapper::Resource
 
   @@debug = false
 
+  # Sets debug mode
+  # mode: boolean
   def self.debug(mode)
     @@debug = mode
   end
@@ -50,7 +50,10 @@ class Job
   property :error, Text, :default => "[]" # JSON Array of error, per instance
   property :store, Integer, :default => STORE_NO # Storage status
 
-  # Update job status and launch the command
+  # Update job status and launch the command locally
+  # Sends a finish message or an error message according to the job status.
+  # if storage is needed, job will send a finish AND a store message. Storage
+  # will occur in parallel of the rest of the workflow.
   def run(curhandler,instance=0)
     # Send run command, possibly multiple ones according to pattern
     # Command would send a message when over
@@ -81,7 +84,7 @@ class Job
     end
   end
 
-  # Skip treatment, just answer
+  # Skip treatment, just answer, for debug
   def skip(curhandler)
     curjob = Job.get(@id)
     curjob.update(:handler => curhandler.to_s)
@@ -89,7 +92,10 @@ class Job
     sendmessage(OP_FINISH,jobmsg)
   end
 
-  # Execute locally the command
+  # Execute locally the command, creating directories and setting environment
+  # variables to empty string for security.
+  # wordir: job working directory
+  # instance: instance number in the list of commands
   def runcommand(workdir,instance)
     initcmd = "AMQP_URL="" && MYSQL_URL="" && mkdir -p "+workdir+" && cd "+workdir+" && WORKDIR="+workdir+" && " 
     curjob = Job.get(@id)
@@ -102,7 +108,8 @@ class Job
   end
 
   # Change instance counter
-  # If workflow is in suspend status, suspend the job at the end of its treatment
+  # If workflow is in suspend status, suspend the job at
+  # the end of its treatment
   def finish
     if @status == STATUS_SKIP
       return
@@ -133,7 +140,7 @@ class Job
     end
   end
 
-  # Compare sintance counter to max instances
+  # Compares intance counter to max instances to determine if job is over.
   def isover?
     if @status == STATUS_OVER
       return true
@@ -179,7 +186,9 @@ class Job
       end
     end
   end
-
+  
+  # Sets a job in ERROR status.
+  # instance: job instance number in fault
   def error!(instance = 0)
     @status= STATUS_ERROR
     curjob = Job.get(@id)
@@ -258,7 +267,12 @@ class Job
       end
     end
   end
-
+ 
+   # Sends a message. According to the operation, message will be 
+   # sent to master or node queues.
+   # operation: kind of message
+   # msg: message to send
+   # jobqueue: optional specific queue 
    def sendmessage(operation,msg,jobqueue='')
      queue = "manband.master"
      if operation == OP_RUN || operation == OP_SKIP || operation == OP_DESTROY || operation == OP_CLEAN|| operation == OP_STORE

data/lib/manband/store.rb

@@ -7,7 +7,7 @@ require 'fileutils'
 require 'manband/flowconfig.rb'
 require 'manband/user.rb'
 
-
+# this class manage the storage of a job directory to S3
 class Storeband
 
   @@log = Logger.new(STDOUT)
@@ -29,6 +29,11 @@ class Storeband
   end
 
   # Store a job workdir to S3
+  #
+  # job: current job
+  # uid: user identifier
+  # bucket: bucket name to store the zip file
+  # @return false in case of failure
   def store(job,uid,bucket="manband")
     user = User.get(uid)
     if user == nil
@@ -50,16 +55,21 @@ class Storeband
       compress(job.workdir, FlowConfig.workdir+"/"+zipfile)
       # Send file
       sends3(FlowConfig.workdir+"/"+zipfile,bucket,user.s3_access,user.s3_secret)
-    rescue
-      @@log.error "An error occured during S3 operation: "+job.id.to_s
+    rescue Exception => e
+      @@log.error "An error occured during S3 operation: "+job.id.to_s+" "+e.message
       job.update(:store => STORE_ERROR)
       return false
     end
     job.update(:store => STORE_OVER)
   end
 
+  # Sends the file to the S3 bucket
+  # name: file name
+  # bucket: destination bucket name
+  # access: user credential access
+  # secret: user credential secret
   def sends3(name,bucket,access,secret)
-    @@log.debug "connect to s3"
+    @@log.debug "connect to s3: "+FlowConfig.s3host+":"+FlowConfig.s3port.to_s+FlowConfig.s3path
     AWS::S3::Base.establish_connection!(
       :access_key_id     => access,
       :secret_access_key => secret,

data/lib/manband/user.rb

@@ -7,6 +7,7 @@ require 'manband/flowconfig.rb'
 #DataMapper.setup(:default, 'sqlite:///tmp/project.db')
 DataMapper.setup(:default, ENV['MYSQL_URL'])
 
+# This class manages users in the database
 class User
   include DataMapper::Resource
 
@@ -17,6 +18,8 @@ class User
   property :s3_access, String # User id key
   property :s3_secret, String # Secret key
 
+  # Creates a default admin account if it does not exists.
+  # Default password is <b>admin</b>
   def self.init
     admin = User.get(@@admin)
     if admin == nil
@@ -26,6 +29,9 @@ class User
     end
   end
 
+  # Check passwords
+  # password: user password
+  # @return: true if authentication is correct
   def authenticate(password)
     shapwd = Digest::SHA1.hexdigest(password) 
     if self.password == shapwd

data/lib/manband/workflow.rb

@@ -12,7 +12,9 @@ require 'manband/job.rb'
 DataMapper.setup(:default, ENV['MYSQL_URL'])
 
 
-
+# This class orchestrator the workflow status and the workflow file
+# analysis. It determines if workflow is over, what are the next jobs
+# to execute, ...
 class WorkFlow
   include DataMapper::Resource
 
@@ -34,6 +36,8 @@ class WorkFlow
   property :parent, Integer, :default => 0  # Parent workflow id, none by default
   property :instances, Integer, :default => 0 # Number of sub workflows, if any
 
+  # Checks if a workflow is over, e.g. we have reached all the
+  # terminal nodes (leafs).
   def isover?
     # decrement terminals
     @terminals = @terminals - 1
@@ -54,7 +58,8 @@ class WorkFlow
     return false
   end
 
-  # Return an array of node names
+  # Get the list of jobs to be run after current node
+  # @return an array of node names
   def getnextjobs(curnode)
     #fworkflow = YAML.load_file(@file)
     fworkflow = BandManager.load(@file)
@@ -75,6 +80,9 @@ class WorkFlow
     return nexts
   end
 
+  # Parse workflow file and create jobs and links in the database
+  # curnode: current node
+  # id: id of the node as link originator
   def parse(curnode, id = nil)
     #fworkflow = YAML.load_file(@file)
     fworkflow = BandManager.load(@file)
@@ -140,7 +148,9 @@ class WorkFlow
     return newcommand
   end
 
-  # Return command for the node in the workflow
+  # Return a list of commands for the node in the workflow
+  # There is one command per input file matching regular expresssions,
+  # if any. Default is 1 command.
   def getnodecommand(curnode)
     #fworkflow = YAML.load_file(@file)
     fworkflow = BandManager.load(@file)
@@ -247,6 +257,7 @@ class WorkFlow
     return commands
   end
 
+  # Clean a workflow directory
   def clean
     workflow = WorkFlow.get(@id)
     if workflow.workdir == nil

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: manband
 version: !ruby/object:Gem::Version 
-  hash: 15
+  hash: 11
   prerelease: 
   segments: 
   - 0
-  - 4
+  - 5
   - 0
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors: 
 - Olivier Sallou