manband 0.4.0

data/LICENSE

@@ -0,0 +1,10 @@
+Files: *
+Copyright: 2012, Olivier Sallou <olivier.sallou@irisa.fr>
+License: CeCILL-C v2
+
+Files: minion/*
+https://github.com/jgwmaxwell/minion
+Copyright: 2010, Orion Henry
+           2010, Adam Wiggins, Kyle Drake
+           2012, John Maxwell
+License: MIT

data/README

@@ -0,0 +1,108 @@
+# Requirements
+
+RabbitMQ, MYSQL with one user and one database
+
+gem install amqp
+gem install json
+gem install data_mapper
+gem install dm-myql-adapter
+gem install dm-migrations
+gem install zip
+gem install mb-minion
+gem install mb-aws-s3
+
+For debug, one can use dm-sqlite-adapter. SQLite does not fit for production and concurrency issues may occur. To avoid this, only 1 workflow handler and 1 job handler should be used.
+MySQL handles correctly the concurrency with lock mechanisms.
+
+# Installation
+
+Export or clone repository git@github.com:osallou/aws-s3.git
+Copy lib/aws directory to manband lib directory (lib/manband/aws)
+S3 host information can be set via jobhandler command-line
+
+# Configuration
+
+Some configuration can be set via a config file with the --conf option.
+Program will also search for a file ".manband" in user home directory.
+If none is used, program will use defaults.
+An example conf file is available in test directory (conf.yaml).
+
+For storage in S3, user credentials must be stored in database.
+If user has access to the database (non shared install), the userband script will help setting up the user (see User Management).
+In a shared install, one MUST use the web interface to setup credentials and launch workflows as there is no control on user id.
+
+# Security
+
+Each workflow is ran in a specific set of queues in a virtual host in RabbitMQ. This access is controlled via AMQP_URL credentials, restricting access only to a user or a set of users.
+Scripts using database access (manband, userband) must not be used in a shared install. One should use remoteband to launch workflows (warning: workflow file must be accessible by nodes).
+Access to user credentials (s3,..) should be allowed only via the web interface or a custom interface.
+
+# Execution
+
+Before executing the programs, one must set in environment:
+
+export AMQP_URL="amqp://johndoe:abc123@localhost/my_vhost"
+export MYSQL_URL="mysql://USER:PASSWORD@localhost/MYDB"
+
+All programs, including the web interface need access to the database and RabbitMQ.
+In a cloud environment, where each user or group of user have to launch his own instances, two solutions:
+1) A master instance holds the web interface and the database. The master instance could then launch new instances (workflowhandler or jobhandler) with MYSQL_URL refering to master instance
+2) For each user, provide with the AMQP queue a dedicated MySQL database. Then user can refer to this database for his handlers.
+
+In all cases, cloud instances would need to get at boot time those 2 variables before executing the handlers or the web interface.
+
+## Workflow Handler
+
+It is possible to run multiple workflow handler to handler the message load
+
+    ruby -rubygems workflowhandler.rb
+
+Use -d for message tracking.
+
+## Job Handler
+
+It is possible to run multiple job handlers to handler job commands
+
+    ruby -rubygems jobhandler.rb -i myhandlername
+
+## Main program
+
+For usage:
+
+    ruby -rubygems manband -h
+
+
+To start a new workflow:
+
+    ruby manband.rb  -w test/samplew.yaml
+
+If user do not have MYSQL credentials, user can use:
+
+    ruby remoteband.rb -w test/samplew.yaml
+
+remoteband initiate a "new" message to start a workflow without MYSQL credentials need.
+
+
+## Web interface
+
+For production mode, on port 4567:
+
+    rackup  -E production -p 4567
+
+## User management
+
+By default, admin account has password "admin". This should be modified after installation.
+
+userband.rb can help to add or update users without the web interface
+
+## S3 storage
+
+By default, bucket used is "manband". It can be specified in command-line.
+
+## Gem creation
+
+Execute:
+    rack package
+
+Gem is created in pkg directory
+Webband may use this gem, or use current files with same hierarchy

data/bin/jobhandler.rb

@@ -0,0 +1,135 @@
+require 'rubygems'
+require 'yaml'
+require 'optparse'
+require 'amqp'
+require 'json'
+require 'mb-minion'
+
+include Minion
+
+if ENV["MYSQL_URL"]==nil
+  puts "MYSQL_URL environment variable is not set exiting..."
+  exit 1
+end
+if ENV["AMQP_URL"]==nil
+  puts "AMQP_URL environment variable is not set, using defaults."
+end
+
+$:.push File.expand_path("../lib")
+
+require 'manband/job.rb'
+require 'manband/store.rb'
+
+handler = Time.now.to_s
+
+log = Logger.new(STDOUT)
+log.level = Logger::INFO
+
+options = {}
+
+optparse = OptionParser.new do|opts|
+  
+  # This displays the help screen, all programs are
+  # assumed to have this option.
+  opts.on( '-h', '--help', 'Display this screen' ) do
+    puts opts
+    exit
+  end
+
+  opts.on( '-i', '--id ID', 'Handler id' ) do |id|
+    handler = id.to_s
+  end
+
+  options[:queue] = ""
+  opts.on( '-q', '--queue QUEUE', 'Queue registration filter' ) do |queue|
+    options[:queue] = "."+queue
+  end
+
+  options[:store] = nil
+  opts.on( '-s', '--store S3HOST', 'S3 host information host:port:path, e.g. localhost:8773:/services/Walrus' ) do |store|
+    options[:store] = store
+  end
+
+  options[:conf] = nil
+  opts.on( '-c', '--conf CONFIG', 'Use config file' ) do |config|
+    options[:conf] = config
+  end
+
+end
+
+optparse.parse!
+
+userconf = File.expand_path("~")+"/.manband"
+if options[:conf]==nil && File.exists?(userconf)
+  options[:conf]=userconf
+end
+
+if options[:conf]!=nil
+  if !File.exists?(options[:conf])
+    puts "Config file does not exists"
+    exit 1 
+  end
+  conf = YAML.load_file(options[:conf])
+  puts "Using configuration file "+options[:conf]
+  FlowConfig.sets3(conf["s3"]["host"],conf["s3"]["port"],conf["s3"]["path"])
+  FlowConfig.setworkdir(conf["workdir"])
+end
+
+if options[:store]!=nil
+  elts = options[:store].split(':')
+  FlowConfig.sets3(elts[0],elts[1],elts[2])
+end
+
+log.info "Start handler "+handler
+
+job "manband.node"+options[:queue]  do |args|
+    msg = JSON.parse(args["msg"])
+    if args["operation"] == OP_DESTROY
+      myworkflow = WorkFlow.get(msg["id"])
+      if myworkflow == nil
+        log.error "Error: workflow does not exists "+msg["id"]
+      else 
+        myworkflow.clean
+        jobs = Job.all(:wid => msg[:id])
+        if jobs!=nil
+          jobs.destroy
+        end
+        links = JobLink.all(:wid => msg[:id])
+        if links!=nil
+          links.destroy
+        end
+        myworkflow.destroy
+        messages = BandMessage.all(:wid => msg[:id])
+        if messages != nil
+          messages.destroy
+        end
+      end
+    elsif args["operation"] == OP_CLEAN
+      myworkflow = WorkFlow.get(msg["id"])
+      if myworkflow == nil
+        log.error "Error: workflow does not exists "+msg["id"]
+      else
+        myworkflow.clean
+      end
+    else
+      myjob = Job.get(msg["id"])
+      if myjob == nil
+        log.error "Error: job does not exists "+msg["id"]
+      else 
+        if args["operation"] == OP_RUN
+          if msg["instance"]!=nil
+            myjob.run(handler, msg["instance"].to_i)
+          else
+            myjob.run(handler)
+          end
+        elsif args["operation"] == OP_SKIP
+          myjob.skip(handler)
+        elsif args["operation"] == OP_STORE
+          storage = Storeband.new
+          uid = WorkFlow.get(myjob.wid).uid
+          storage.store(myjob,uid,msg["bucket"])
+        end
+      end
+    end
+end
+

data/bin/manband.rb

@@ -0,0 +1,108 @@
+require 'rubygems'
+require 'yaml'
+require 'optparse'
+require 'amqp'
+require 'json'
+require 'mb-minion'
+
+include Minion
+
+$:.push File.expand_path("../lib")
+
+require 'manband/workflow.rb'
+require 'manband/flowconfig.rb'
+require 'manband/job.rb'
+require 'manband/bandmanager.rb'
+
+log = Logger.new(STDOUT)
+log.level = Logger::INFO
+
+options = {}
+
+optparse = OptionParser.new do|opts|
+  
+  # This displays the help screen, all programs are
+  # assumed to have this option.
+  opts.on( '-h', '--help', 'Display this screen' ) do
+    puts opts
+    exit
+  end
+
+  opts.on( '-s', '--show WORKFLOW', 'show workflow with id WORKFLOW') do  |workflow|
+    workflow = WorkFlow.get(workflow)
+    puts "Workflow "+workflow.id.to_s
+    puts " -- start: "+workflow.created_at.to_s
+    puts " -- end  : "+workflow.terminated_at.to_s
+    jobs = Job.all(:wid => workflow.id)
+    jobs.each do |job|
+      if job.node != "root"
+        puts " ---- "+job.id.to_s+","+job.node+","+job.status.to_s
+        puts " ------ Workdir: "+job.workdir   unless job.workdir == nil
+        puts " ------ Handler: "+job.handler   unless job.handler == nil
+      end
+    end
+  end
+
+
+  options[:workflow] = nil
+  opts.on( '-w', '--workflow FILE', 'Use input worflow') do |file|
+    puts "using "+file
+    options[:workflow] = file
+  end
+
+  options[:conf] = nil
+  opts.on( '-c', '--conf CONFIG', 'Use config file' ) do |config|
+    options[:conf] = config
+  end
+
+
+  options[:uid] = 'admin'
+  opts.on( '-u', '--user UID', 'User id') do |uid|
+    options[:uid] = uid
+  end
+
+  options[:bucket] = 'manband'
+  opts.on( '-b', '--bucket BUCKET', 'S3 bucket if storage is used') do |bucket|
+    options[:bucket] = bucket
+  end
+
+  options[:var] = Array.new
+  opts.on( '-v', '--var KEY1=VAL1,KEY2=VAL2,KEY3=VAL3', Array, 'Runtime variables') do |runtimevars|
+    options[:var] = runtimevars
+  end
+
+  options[:debug] = false
+  opts.on( '-d', '--debug', 'Debug mode, do not execute commands') do
+    options[:debug] = true
+  end
+end
+
+optparse.parse!
+
+if options[:workflow]==nil
+  exit
+end
+
+userconf = File.expand_path("~")+"/.manband"
+if options[:conf]==nil && File.exists?(userconf)
+  options[:conf]=userconf
+end
+
+if options[:conf]!=nil
+  if !File.exists?(options[:conf])
+    puts "Config file does not exists"
+    exit 1
+  end
+  conf = YAML.load_file(options[:conf])
+  puts "Using configuration file "+options[:conf]
+  FlowConfig.setworkdir(conf["workdir"])
+end
+
+
+wid = BandManager.launch(options[:workflow],options[:var],options[:uid],options[:bucket],options[:debug])
+
+if wid == nil
+  log.error "Could not launch the workflow"
+else
+  log.info "Workflow "+wid.to_s+" starting..."
+end

data/bin/remoteband.rb

@@ -0,0 +1,76 @@
+require 'rubygems'
+require 'yaml'
+require 'optparse'
+require 'amqp'
+require 'json'
+require 'mb-minion'
+
+include Minion
+
+
+log = Logger.new(STDOUT)
+log.level = Logger::INFO
+
+options = {}
+
+optparse = OptionParser.new do|opts|
+  
+  # This displays the help screen, all programs are
+  # assumed to have this option.
+  opts.on( '-h', '--help', 'Display this screen' ) do
+    puts opts
+    exit
+  end
+
+  opts.on( '-s', '--show WORKFLOW', 'show workflow with id WORKFLOW') do  |workflow|
+    workflow = WorkFlow.get(workflow)
+    puts "Workflow "+workflow.id.to_s
+    puts " -- start: "+workflow.created_at.to_s
+    puts " -- end  : "+workflow.terminated_at.to_s
+    jobs = Job.all(:wid => workflow.id)
+    jobs.each do |job|
+      if job.node != "root"
+        puts " ---- "+job.id.to_s+","+job.node+","+job.status.to_s
+        puts " ------ Workdir: "+job.workdir   unless job.workdir == nil
+        puts " ------ Handler: "+job.handler   unless job.handler == nil
+      end
+    end
+  end
+
+
+  options[:workflow] = nil
+  opts.on( '-w', '--workflow FILE', 'Use input worflow') do |file|
+    puts "using "+file
+    options[:workflow] = file
+  end
+
+  options[:uid] = 'admin'
+  opts.on( '-u', '--user UID', 'User id') do |uid|
+    options[:uid] = uid
+  end
+
+  options[:var] = Array.new
+  opts.on( '-v', '--var KEY1=VAL1,KEY2=VAL2,KEY3=VAL3', Array, 'Runtime variables') do |runtimevars|
+    options[:var] = runtimevars
+  end
+
+  options[:bucket] = 'manband'
+  opts.on( '-b', '--bucket BUCKET', 'S3 bucket if storage is used') do |bucket|
+    options[:bucket] = bucket
+  end
+
+
+end
+
+optparse.parse!
+
+if options[:workflow]==nil
+  exit
+end
+
+msg  = Hash.new
+msg["wfile"]=options[:workflow]
+msg["var"]=options[:var]
+msg["uid"]=options[:uid]
+msg["bucket"]=options[:bucket]
+Minion.enqueue("manband.master", { "operation" => "new", "msg" => msg.to_json })

data/bin/userband.rb

@@ -0,0 +1,75 @@
+require 'rubygems'
+require 'yaml'
+require 'optparse'
+require 'json'
+require 'logger'
+
+$:.push File.expand_path("../lib")
+
+require 'manband/flowconfig.rb'
+
+log = Logger.new(STDOUT)
+log.level = Logger::INFO
+
+user = nil
+
+options = {}
+
+optparse = OptionParser.new do|opts|
+  
+  # This displays the help screen, all programs are
+  # assumed to have this option.
+  opts.on( '-h', '--help', 'Display this screen' ) do
+    puts opts
+    exit
+  end
+
+  options[:login] = nil
+  opts.on( '-u', '--user UID', 'User id') do |uid|
+    options[:login] = uid
+  end
+
+  options[:password] = nil
+  opts.on( '-p', '--password PASSWORD', 'User password') do |passwd|
+    options[:password] = passwd
+  end
+
+  options[:access] = nil
+  opts.on( '-a', '--access S3_ACCESS', 'S3 Access key') do |access|
+    options[:access] = access
+  end
+
+  options[:secret] = nil
+  opts.on( '-s', '--secret S3_SECRET', 'S3 secret key') do |secret|
+    options[:secret] = secret
+  end
+end
+
+optparse.parse!
+
+if options[:login]==nil
+  puts "User id is mandatory"
+  exit
+end
+
+user = User.get(options[:login])
+
+if user == nil
+  user = User.new(:login => options[:login], :password => Digest::SHA1.hexdigest(options[:password]), :s3_access => options[:access], :s3_secret => options[:secret] )
+  user.save
+  log.info "User "+user.id+" created"
+else
+  if options[:password] == nil
+    options[:password] = user.password
+  end
+  if options[:access] == nil
+    options[:access] = user.s3_access
+  end
+  if options[:secret] == nil
+    options[:secret] = user.s3_secret
+  end
+  user.update(:password => Digest::SHA1.hexdigest(options[:password]), :s3_access => options[:access], :s3_secret => options[:secret])
+  log.info "User "+user.id.to_s+" updated"
+end
+
+

data/bin/workflowhandler.rb

@@ -0,0 +1,182 @@
+require 'rubygems'
+require 'yaml'
+require 'optparse'
+require 'amqp'
+require 'json'
+require 'mb-minion'
+
+include Minion
+
+if ENV["MYSQL_URL"]==nil
+  puts "MYSQL_URL environment variable is not set exiting..."
+  exit 1
+end
+if ENV["AMQP_URL"]==nil
+  puts "AMQP_URL environment variable is not set, using defaults."
+end
+
+$:.push File.expand_path("../lib")
+
+require 'manband/workflow.rb'
+require 'manband/job.rb'
+require 'manband/bandmanager.rb'
+
+log = Logger.new(STDOUT)
+log.level = Logger::INFO
+
+@@options = {}
+
+optparse = OptionParser.new do|opts|
+  
+  # This displays the help screen, all programs are
+  # assumed to have this option.
+  opts.on( '-h', '--help', 'Display this screen' ) do
+    puts opts
+    exit
+  end
+
+  @@options[:conf] = nil
+  opts.on( '-c', '--conf CONFIG', 'Use config file' ) do |config|
+    @@options[:conf] = config
+  end
+
+
+  @@options[:debug]=false
+  opts.on( '-d', '--debug', 'Debug mode, store all messages in database') do
+    @@options[:debug]=true
+    Job.debug(true)
+  end
+end
+
+optparse.parse!
+
+userconf = File.expand_path("~")+"/.manband"
+if @@options[:conf]==nil && File.exists?(userconf)
+  options[:conf]=userconf
+end
+
+if @@options[:conf]!=nil
+  if !File.exists?(@@options[:conf])
+    puts "Config file does not exists"
+    exit 1
+  end
+  conf = YAML.load_file(@@options[:conf])
+  puts "Using configuration file "+@@options[:conf]
+  FlowConfig.sets3(conf["s3"]["host"],conf["s3"]["port"],conf["s3"]["path"])
+  FlowConfig.setworkdir(conf["workdir"])
+end
+
+job "manband.master"  do |args|
+    msg = JSON.parse(args["msg"])
+    if args["operation"] == OP_NEW
+      BandManager.launch(msg["wfile"],msg["var"],msg["uid"],msg["bucket"])
+    end
+    if args["operation"] == OP_START
+      savemessage(msg["id"],args["operation"],args["msg"])
+      workflow = WorkFlow.get(msg["id"])
+      if workflow == nil
+        log.error "Error: workflow does not exists "+msg["id"] 
+      else
+        workflow.update(:status => STATUS_RUNNING)
+        myjob = Job.get(msg["root"])
+        myjob.runnext
+      end
+    end
+    if args["operation"] == OP_SKIP
+      savemessage(msg["id"],args["operation"],args["msg"])
+      workflow = WorkFlow.get(msg["id"])
+      if workflow == nil
+        log.error "Error: workflow does not exists "+msg["id"]
+      else
+        workflow.update(:status => STATUS_RUNNING)
+        myjob = Job.get(msg["root"])
+        myjob.runnext(true)
+      end
+    end
+    if args["operation"] == OP_ERROR
+      savemessage(msg["workflow"],args["operation"],args["msg"])
+      myjob = Job.get(msg["id"])
+      if myjob == nil
+        log.error "Error: job does not exists "+msg["id"]
+      else
+        log.debug "Job "+msg["id"]+" in error"
+        instance = 0
+        if msg["instance"]!=nil
+          instance = msg["instance"]
+        end
+        myjob.error!(instance)
+      end
+    end
+    if args["operation"] == OP_FINISH
+      savemessage(msg["workflow"],args["operation"],args["msg"])
+      myjob = Job.get(msg["id"])
+      if myjob == nil
+        log.error "Error: job does not exists "+msg["id"]
+      else
+        if msg["handler"]!=nil && myjob.handler != msg["handler"]
+          log.debug "Finish received from a different handler "+myjob.handler+" vs "+msg["handler"]+", skipping message"
+        else
+          myjob.finish
+          if myjob.isover?
+            myjob.runnext
+          end
+        end
+      end
+    end
+    if args["operation"] == OP_WSUSPEND
+      savemessage(msg["id"],args["operation"],args["msg"])
+      workflow = WorkFlow.get(msg["id"])
+      if workflow == nil
+        log.error "Error: workflow does not exists "+msg["id"]
+      else
+        workflow.update(:status => STATUS_SUSPEND)
+      end
+    end
+    if args["operation"] == OP_WRESUME
+      savemessage(msg["id"],args["operation"],args["msg"])
+      workflow = WorkFlow.get(msg["id"])
+      if workflow == nil
+        log.error "Error: workflow does not exists "+msg["id"]
+      else
+        log.info "Resuming workflow "+msg["id"]
+        workflow.update(:status => STATUS_RUNNING)
+        suspendedjobs = Job.all(:status => STATUS_SUSPEND, :wid => workflow.id)
+        suspendedjobs.each do |job|
+          job.resume
+        end
+        errorjobs = Job.all(:status => STATUS_ERROR, :wid => workflow.id)
+        errorjobs.each do  |job|
+          job.resume
+        end
+      end
+    end
+    if args["operation"] == OP_JSUSPEND
+      savemessage(msg["workflow"],args["operation"],args["msg"])
+      curjob = Job.get(msg["id"])
+      if curjob == nil
+        log.error "Error: Job does not exists "+msg["id"]
+      else
+        if curjob.status != STATUS_SKIP
+          curjob.update(:status => STATUS_SUSPEND)
+        end
+      end
+    end
+    if args["operation"] == OP_JRESUME
+      savemessage(msg["workflow"],args["operation"],args["msg"])
+      curjob = Job.get(msg["id"])
+      if curjob == nil
+        log.error "Error: Job does not exists "+msg["id"]
+      else
+        log.info "Resuming job "+msg["id"]
+        curjob.resume
+      end
+    end
+end
+
+def savemessage(id,operation,msg)
+   if @@options[:debug] == false
+     return
+   end
+   bmsg = BandMessage.new(:wid => id, :message => '{ "operation" => '+operation+', "msg" => '+msg+' }' )
+   bmsg.save
+end