scbi_queue_system 0.0.2

data/History.txt

@@ -0,0 +1,8 @@
+=== 0.0.2 2011-07-12
+
+Fixed daemon installation
+
+=== 0.0.1 2011-06-28
+
+* 1 major enhancement:
+  * Initial release

data/Manifest.txt

@@ -0,0 +1,23 @@
+bin/queue_manager.rb
+bin/sqstat
+bin/sqsub
+bin/sqs_install_daemon
+History.txt
+lib/scbi_queue_system/autolaunch/com.scbi_queue_system.plist
+lib/scbi_queue_system/autolaunch/sqsd-linux
+lib/scbi_queue_system/done_job_list.rb
+lib/scbi_queue_system/internal_config/internal_config.json
+lib/scbi_queue_system/job_list.rb
+lib/scbi_queue_system/queued_job_list.rb
+lib/scbi_queue_system/running_job_list.rb
+lib/scbi_queue_system.rb
+Manifest.txt
+PostInstall.txt
+Rakefile
+README.rdoc
+script/console
+script/destroy
+script/generate
+test/submit_script.sh
+test/test_helper.rb
+test/test_scbi_queue_system.rb

data/PostInstall.txt

@@ -0,0 +1,7 @@
+
+For more information on scbi_queue_system, see http://scbi_queue_system.rubyforge.org
+
+NOTE: Change this information in PostInstall.txt 
+You can also delete it if you don't want it.
+
+

data/README.rdoc

@@ -0,0 +1,138 @@
+= scbi_queue_system
+
+* http://www.scbi.uma.es/downloads
+
+== DESCRIPTION:
+
+scbi_queue_system (SQS) handles a simple queue of jobs executions over multiple machines (clustered installation) or your own personal computer.
+
+== FEATURES/PROBLEMS:
+
+* SQS can be used as very simple batch queue system for personal multicore computers as well as for small clusters
+* It handles machines with different number of cores/CPUs
+* It believes on well intentioned users, and because of that it doesn't kill jobs if they are using more CPUs than requested
+* All jobs run under the same user (the one that SQS manager run under)
+
+== SYNOPSIS:
+
+Once SQS is installed and queue manager is running, you can start using it:
+
+=== To submit a new job:
+
+  sqsub file.sh
+
+where file.sh is a script file where you run your programs. Example file issuing ls and hostname commands:
+
+  $> cat file.sh 
+  #!/usr/bin/env bash
+
+  ls
+  hostname
+
+=== To submit a new job using 4 cpus:
+
+  sqsub file.sh 4
+
+You can also set the cpu count inside the submit script this way:
+
+  $> cat file.sh 
+  #!/usr/bin/env bash
+  # CPUS = 4
+
+  ls
+  hostname
+
+=== To view queue status
+
+  sqstat
+  
+To view already done jobs:
+  
+  sqstat -d
+  
+
+== REQUIREMENTS:
+
+* OS X / linux operating systems.
+
+== INSTALL:
+
+=== 1.- Personal/standalone computer:
+
+In a personal installation, all pieces of the queue system are installed on the same computer. To install the queue manager on one computer:
+
+  gem install scbi_queue_system
+
+Once installed, you should start the queue manager:
+
+  queue_manager.rb
+
+By default, it is configured to use localhost with 1 cpu as testing
+  
+<b>NOTE:</b> See automatic startup
+
+
+=== 2.-Cluster:
+
+On a clustered installation you need to choose a node to act as the queue manager. 
+
+==== -On the manager machine
+
+Install SQS on the manager machine:
+
+  gem install scbi_queue_system
+
+Start the queue manager:
+
+  queue_manager.rb
+
+Share your QUEUED folder with frontend machines and give write-only permission to others (chmod 772)
+
+==== -On the frontends machines:
+
+Later on, you should install the sqsub command on your cluster's frontend machines (the one/ones from where you are going to submit jobs).
+
+  gem install scbi_queue_system
+
+=== 3.- Environment variables:
+
+* SQS_BASE_PATH : sets the location of sqs directoies. 
+
+* SQS_QUEUED_PATH : sets the location of QUEUED directory. This location must be shared and with write only permissions for all SQS clients (those that need to use sqsub).
+
+* SQS_LOG_FILE: sets the location of log files.
+
+=== 4.- Automatic startup
+
+Once the gem is installed you can start the queue manager (it is recommended to setup automatic startup so you don't have to start it manually after each reboot). The queue manager only should be started up in the manager machine, and not on frontends.
+
+==== -On Linux
+
+
+==== -On Mac OS X
+
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2011 Dario guerrero
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,26 @@
+require 'rubygems'
+gem 'hoe', '>= 2.1.0'
+require 'hoe'
+require 'fileutils'
+require './lib/scbi_queue_system'
+
+Hoe.plugin :newgem
+# Hoe.plugin :website
+# Hoe.plugin :cucumberfeatures
+
+# Generate all the Rake tasks
+# Run 'rake -T' to see list of generated tasks (from gem root directory)
+$hoe = Hoe.spec 'scbi_queue_system' do
+  self.developer 'Dario Guerrero', 'dariogf@gmail.com'
+  self.post_install_message = 'PostInstall.txt' # TODO remove if post-install message not required
+  self.rubyforge_name       = self.name # TODO this is default value
+  self.extra_deps         = [['json','>= 1.5.3']]
+
+end
+
+require 'newgem/tasks'
+Dir['tasks/**/*.rake'].each { |t| load t }
+
+# TODO - want other tests/tasks run by default? Add them to the list
+# remove_task :default
+# task :default => [:spec, :features]

data/bin/queue_manager.rb

@@ -0,0 +1,209 @@
+#!/usr/bin/env ruby
+
+# $: << File.join(File.dirname(__FILE__),'..','lib')
+
+require 'scbi_queue_system'
+require 'logger'
+
+require 'running_job_list'
+require 'queued_job_list'
+require 'done_job_list'
+
+system_log_message="SCBI SQS queue at: #{QUEUED_PATH}"
+`logger "#{system_log_message}"`
+
+user=`whoami`
+system_log_message="SCBI SQS user: #{user}"
+`logger "#{system_log_message}"`
+
+
+$LOG = Logger.new(LOG_FILE, 10, 1024000)
+
+# $LOG = Logger.new(STDOUT, 10, 1024000)
+
+$LOG.level = Logger::INFO
+
+$LOG.info 'Starting up SQS'
+
+# LOAD CONFIG FILES
+
+def load_config
+  
+  $LOG.info 'Reading config files'
+
+  if File.exists?(CONFIG_FILE)
+    config_txt=File.read(CONFIG_FILE)
+
+    # remove comments
+    config_txt_cleaned=config_txt.split("\n").select{|l| !(l=~ /^\s*\#/)}.join("\n")
+    # puts config_txt_cleaned
+    config=JSON::parse(config_txt_cleaned,:symbolize_names => true)
+    
+  else
+    # default config
+    config={}
+
+    config[:polling_time] = 10
+    config[:machine_list] = []
+    
+    cpus=1
+    
+    if RUBY_PLATFORM.downcase.include?("darwin")
+      cpus=`hwprefs -cpu_count`.chomp.to_i
+    else
+      cpus=`grep processor /proc/cpuinfo |wc -l`.chomp.to_i
+    end
+    
+    machine={:name=>'localhost', :cpus => cpus}
+    
+    config[:machine_list] << machine
+    # config[:sqs_user] = 'dariogf'
+
+    f=File.open(CONFIG_FILE,'w')
+    f.puts "# You can add comment lines to this config file"
+    
+    f.puts JSON::pretty_generate(config)
+    f.close
+    $LOG.debug "Generating default config. You can modify it by editing #{CONFIG_FILE}"
+  end
+
+  $LOG.debug "Config loaded #{config.to_json}"
+
+
+  # # create machine slots
+  config[:machine_list].each do |machine|
+    FileUtils.mkdir_p(File.join(RUNNING_PATH,machine[:name]))
+    FileUtils.mkdir_p(File.join(SENT_PATH,machine[:name]))
+  end
+  return config
+end
+
+def select_next_jobs(machines,queued,running)
+  # select next job for running
+  res = {}
+
+  if !queued.jobs.empty?
+
+    # iterate over machines
+    machines.each do |machine|
+
+      # find free_cpus
+      running_cpus=running.running_cpus_in_machine(machine[:name])
+      free_cpus = machine[:cpus]-running_cpus
+
+      $LOG.debug("Free space on #{machine[:name]}: #{free_cpus}")
+
+      # use free cpus
+      while (free_cpus > 0) do
+
+          next_job=queued.jobs.find{|job| job[:cpus] <= free_cpus}
+
+          if next_job
+            # delete job from queue
+            queued.jobs.delete(next_job)
+
+            # discount cpus
+            free_cpus -= next_job[:cpus]
+
+            # add job to be run on current machine
+            res[machine[:name]]=[] if res[machine[:name]].nil?
+            res[machine[:name]] << next_job
+          else
+            break
+          end
+
+        end
+      end
+    end
+
+    return res
+  end
+
+  def update_running_jobs(machines,running)
+
+    machines.each do |machine|
+
+      running_in_machine=running.running_jobs_in_machine(machine[:name])
+      if machine[:name].upcase.index('LOCALHOST')
+        cmd= machine[:ps_command] || "ps ax -o command"
+      else
+        ps_cmd=machine[:ps_command] || "ps ax -o command"
+        cmd="ssh #{machine[:name]} \"#{ps_cmd}\""
+      end
+      # puts cmd
+      res=`#{cmd}`
+
+      running_in_machine.each do |job|
+        sqs_job_file=File.join(RUNNING_PATH,machine[:name],job[:run_script])
+        user_job_file=File.join(job[:cwd],job[:run_script])
+        if File.exists?(sqs_job_file)
+
+          # if script is not running
+          if !res.index(user_job_file)
+            # remove it from folder
+            File.delete(sqs_job_file)
+            FileUtils.mv(File.join(SENT_PATH,machine[:name],job[:run_script]),DONE_PATH)
+            $LOG.info("Job finished: #{job[:run_script]}")
+          end
+        end
+      end
+
+      # puts "#{machine[:name]}, #{res}"
+
+    end
+
+  end
+
+  config=load_config
+      
+  exit_loop=false
+
+  Signal.trap("INT") do
+    puts "Terminating SQS manager..."
+    puts "Please wait #{config[:polling_time]} seconds..."
+    exit_loop=true
+  end
+
+  # event loop
+  begin
+    
+    config=load_config
+    # clear screen
+    # print "\e[2J\e[f"
+
+    # check files
+    $LOG.debug "Checking queue and machine status"
+
+    queued=QueuedJobList.new
+    running=RunningJobList.new
+
+    # puts queued.to_s
+    # puts running.to_s
+
+
+    update_running_jobs(config[:machine_list],running)
+
+    # puts "#{queued.count} queued jobs"
+    # puts "QUEUED:\n#{queued.map{|q| q.to_json}.join("\n")}"
+
+    next_jobs=select_next_jobs(config[:machine_list],queued,running)
+
+    if !next_jobs.empty?
+      $LOG.debug("Running next jobs")
+      next_jobs.each do |machine,jobs|
+        jobs.each do |job|
+          QueuedJobList.execute_job(job,machine)
+        end
+      end
+    end
+
+    # queued=QueuedJobList.new
+    # running=RunningJobList.new
+    #
+    # puts queued.stats_header
+    # puts queued.to_s
+    # puts running.to_s
+
+
+    sleep config[:polling_time]
+  end while !exit_loop

data/bin/sqs_install_daemon

@@ -0,0 +1,67 @@
+#!/usr/bin/env ruby
+
+require 'fileutils'
+
+user_name=ARGV.shift
+
+if user_name.nil? || user_name.empty? || user_name=='root'
+  puts "You must provide an user name to run SQS with that user instead of root"
+  exit -1
+end
+
+if RUBY_PLATFORM.downcase.include?("darwin")
+  daemon_path=File.join(File.dirname(__FILE__),'..','lib','scbi_queue_system','autolaunch','com.scbi_queue_system.plist')
+  destination_path=File.expand_path(File.join('/','Library','LaunchDaemons','com.scbi_queue_system.plist'))
+  
+  if system('launchctl list com.scbi_queue_system.plist')
+    puts "Daemon already installed, uninstalling it. Run again to install"
+    `sudo -u #{user_name} launchctl unload #{destination_path}`
+    FileUtils.rm destination_path
+  else
+    puts "Installing daemon"
+    text=File.read(daemon_path)
+    
+    text.gsub!('USER_NAME',user_name)
+    
+    f=File.open(destination_path,'w')
+    f.puts text
+    f.close
+    
+    FileUtils.chown user_name, nil, destination_path
+    
+    FileUtils.chmod 0755, destination_path
+    
+    # FileUtils.cp dameon_path, destination_path
+    `sudo -u #{user_name} launchctl load #{destination_path}`
+  end
+else
+  daemon_path=File.join(File.dirname(__FILE__),'..','lib','scbi_queue_system','autolaunch','sqsd-linux')
+    destination_path=File.join('/','etc','init.d','sqsd-linux')
+    check=`chkconfig -l |grep -c sqsd-linux`.chomp
+    puts "Checking #{check}"
+
+    if check!='0'
+      puts "Daemon already installed, uninstalling it. Run again to install"
+
+      `#{destination_path} stop`
+      `chkconfig -d sqsd-linux`
+      FileUtils.rm destination_path
+    else
+
+      puts "Installing daemon #{daemon_path}"
+
+      text=File.read(daemon_path)
+      text.gsub!('USER_NAME',user_name)
+
+      f=File.open(destination_path,'w')
+      f.puts text
+      f.close
+
+      FileUtils.chown user_name, nil, destination_path
+      FileUtils.chmod 0755, destination_path
+      `#{destination_path} start`
+      `chkconfig -a sqsd-linux`
+    end
+end
+
+puts "Finished"

data/bin/sqstat

@@ -0,0 +1,74 @@
+#!/usr/bin/env ruby
+
+# $: << File.join(File.dirname(__FILE__),'..','lib')
+
+require 'scbi_queue_system'
+require 'optparse'
+
+DEFAULT_INTERVAL=10
+# bucle=ARGV.shift
+
+
+options = {}
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{File.basename($0)} [options]"
+
+  options[:loop]=false
+  options[:interval]=DEFAULT_INTERVAL
+  
+  opts.on("-l", "--loop [INTERVAL]", "Run continuously") do |i|
+    options[:loop] = true
+    options[:interval] = i.to_i
+    if options[:interval]==0
+      options[:interval]=DEFAULT_INTERVAL
+    end
+  end
+
+  opts.on("-d", "--done", "Also show done jobs") do |d|
+    options[:done] = d
+  end
+
+end.parse!
+
+
+
+# if ARGV.count == 1
+#   puts "#{File.basename($0)} submit_script"
+#   exit -1
+# end
+
+def stat(with_done_jobs)
+  queued=QueuedJobList.new
+  running=RunningJobList.new
+
+  puts queued.stats_header
+  puts queued.to_s
+  puts running.to_s
+  puts
+  
+  if with_done_jobs
+    done=DoneJobList.new
+    puts "Recently DONE jobs:"
+    puts done.to_s
+  end
+
+end
+
+if options[:loop]
+
+  exit_loop=false
+
+  Signal.trap("INT") do
+    puts "Terminating, wait #{options[:interval]} seconds"
+    exit_loop=true
+  end
+
+  begin
+    print "\e[2J\e[f"
+    stat(options[:done])
+    sleep options[:interval]
+  end while !exit_loop
+else
+  
+  stat(options[:done])
+end

data/bin/sqsub

@@ -0,0 +1,52 @@
+#!/usr/bin/env ruby
+
+# $: << File.join(File.dirname(__FILE__),'..','lib')
+
+require 'scbi_queue_system'
+
+if ARGV.count == 0
+  puts "#{File.basename($0)} submit_script"
+  exit -1
+end
+submit_script=ARGV.shift
+
+if !File.exists?(submit_script)
+  puts "File #{submit_script} doesn't exists"
+  exit -1
+end
+
+cpus=1
+
+begin
+  cpus=ARGV.shift.to_i
+  if cpus==0 
+    cpus=1
+  end
+rescue
+  cpus=1
+end
+
+t=Time.now
+filename = ("%04d%02d%02d_%02d%02d%02d_%07d" % [t.year,t.month,t.day,t.hour,t.min,t.sec,t.usec])
+
+filename += ('_' + File.basename(submit_script))
+
+# copy file to queued
+original_script=File.read(submit_script)
+
+# FileUtils.cp(submit_script,File.join(QUEUED_PATH,filename))
+
+f=File.open(File.join(QUEUED_PATH,filename),'a')
+
+f.puts '#!/usr/bin/env bash'
+f.puts "cd \"#{Dir.pwd}\""
+f.puts "# CWD = #{Dir.pwd}"
+f.puts "# CPUS = #{cpus}"
+f.puts "# SCRIPT = #{File.expand_path(submit_script)}"
+f.puts "# RUN_SCRIPT = #{filename}"
+f.puts original_script
+f.close
+
+`chmod +x #{File.join(QUEUED_PATH,filename)}`
+
+puts "File #{filename} submitted"

data/lib/scbi_queue_system.rb

@@ -0,0 +1,57 @@
+$:.unshift(File.dirname(__FILE__)) unless
+  $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+$:<<File.join(File.dirname(__FILE__), File.basename(__FILE__,File.extname(__FILE__)))
+
+
+require 'fileutils'
+require 'json'
+
+# load internal config
+INTERNAL_CONFIG_PATH=File.join(File.dirname(__FILE__), File.basename(__FILE__,File.extname(__FILE__)),'internal_config','internal_config.json')
+
+if File.exists?(INTERNAL_CONFIG_PATH)
+  config_txt=File.read(INTERNAL_CONFIG_PATH)
+  internal_config=JSON::parse(config_txt, :symbolize_names => true)
+  if ENV['SQS_BASE_PATH']
+    internal_config[:base_path] = ENV['SQS_BASE_PATH']
+  end
+else
+  # default config
+  internal_config={}
+  internal_config[:base_path] = ENV['SQS_BASE_PATH'] || '~/.sqs'
+  
+  f=File.open(INTERNAL_CONFIG_PATH,'w')
+  f.puts JSON::pretty_generate(internal_config)
+  f.close
+end
+
+JOBS_PATH=File.join(internal_config[:base_path],'sqs','jobs')
+
+CONFIG_PATH=File.join(internal_config[:base_path],'sqs','config')
+
+LOGS_PATH=File.join(internal_config[:base_path],'sqs','logs')
+
+CONFIG_FILE=File.join(CONFIG_PATH,'config.json')
+
+LOG_FILE=ENV['SQS_LOG_PATH'] || File.join(LOGS_PATH,'sqs.log')
+
+# create paths
+FileUtils.mkdir_p([CONFIG_PATH,LOGS_PATH])
+
+QUEUED_PATH=ENV['SQS_QUEUED_PATH'] || File.join(JOBS_PATH,'queued')
+DONE_PATH=File.join(JOBS_PATH,'done')
+RUNNING_PATH=File.join(JOBS_PATH,'running')
+SENT_PATH=File.join(JOBS_PATH,'sent')
+
+# create paths
+FileUtils.mkdir_p([QUEUED_PATH,DONE_PATH,SENT_PATH,RUNNING_PATH])
+
+require 'running_job_list'
+require 'queued_job_list'
+require 'done_job_list'
+
+
+module ScbiQueueSystem
+   VERSION = '0.0.2'
+end

data/lib/scbi_queue_system/autolaunch/com.scbi_queue_system.plist

@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>KeepAlive</key>
+	<true/>
+	<key>Label</key>
+	<string>com.scbi_queue_system.plist</string>
+	<key>OnDemand</key>
+	<false/>
+	<key>ProgramArguments</key>
+	<array>
+		<string>queue_manager.rb</string>
+	</array>
+	<key>RunAtLoad</key>
+	<true/>
+	<key>UserName</key>
+	<string>USER_NAME</string>
+</dict>
+</plist>

data/lib/scbi_queue_system/autolaunch/sqsd-linux

@@ -0,0 +1,84 @@
+#!/bin/sh
+
+### BEGIN INIT INFO
+# Provides:          SCBI queue system service
+# Required-Start:    $syslog
+# Required-Stop:     $syslog
+# Default-Start:     3 5
+# Default-Stop:      0 1 2 4 6
+# Short-Description: SCBI queue system service
+# Description: SCBI queue system service
+#
+### END INIT INFO
+
+SCBI_BIN=/usr/bin/queue_manager.rb
+SQS_USER=USER_NAME
+
+# Shell functions sourced from /etc/rc.status:
+#      rc_check         check and set local and overall rc status
+#      rc_status        check and set local and overall rc status
+#      rc_status -v     be verbose in local rc status and clear it afterwards
+#      rc_status -v -r  ditto and clear both the local and overall rc status
+#      rc_status -s     display "skipped" and exit with status 3
+#      rc_status -u     display "unused" and exit with status 3
+#      rc_failed        set local and overall rc status to failed
+#      rc_failed <num>  set local and overall rc status to <num>
+#      rc_reset         clear both the local and overall rc status
+#      rc_exit          exit appropriate to overall rc status
+#      rc_active        checks whether a service is activated by symlinks
+. /etc/rc.status
+
+rc_reset
+case "$1" in
+    start)
+	echo -n "Starting SQS "
+	## Start daemon with startproc(8). If this fails
+	## the return value is set appropriately by startproc.
+	/sbin/startproc -u $SQS_USER $SCBI_BIN
+
+	# Remember status and be verbose
+	rc_status -v
+	;;
+    stop)
+	echo -n "Shutting down SQS "
+	## Stop daemon with killproc(8) and if this fails
+	## killproc sets the return value according to LSB.
+	/sbin/killproc -TERM $SCBI_BIN
+
+	# Remember status and be verbose
+	rc_status -v
+	;;
+    restart)
+	## Stop the service and regardless of whether it was
+	## running or not, start it again.
+	$0 stop
+	$0 start
+
+	# Remember status and be quiet
+	rc_status
+	;;
+    status)
+	echo -n "Checking for service SQS "
+	## Check status with checkproc(8), if process is running
+	## checkproc will return with exit status 0.
+
+	# Return value is slightly different for the status command:
+	# 0 - service up and running
+	# 1 - service dead, but /var/run/  pid  file exists
+	# 2 - service dead, but /var/lock/ lock file exists
+	# 3 - service not running (unused)
+	# 4 - service status unknown :-(
+	# 5--199 reserved (5--99 LSB, 100--149 distro, 150--199 appl.)
+	
+	# NOTE: checkproc returns LSB compliant status values.
+	/sbin/checkproc $SCBI_BIN
+	# NOTE: rc_status knows that we called this init script with
+	# "status" option and adapts its messages accordingly.
+	rc_status -v
+	;;
+    *)
+	echo "Usage: $0 {start|stop|status|restart}"
+	exit 1
+	;;
+esac
+rc_exit

data/lib/scbi_queue_system/done_job_list.rb

@@ -0,0 +1,10 @@
+require 'job_list'
+
+class DoneJobList < JobList
+  
+  def initialize
+    
+    super(DONE_PATH)
+  end
+  
+end

data/lib/scbi_queue_system/internal_config/internal_config.json

@@ -0,0 +1,3 @@
+{
+  "base_path": "/drives"
+}

data/lib/scbi_queue_system/job_list.rb

@@ -0,0 +1,112 @@
+class JobList
+  
+  attr_accessor :jobs
+  
+  def initialize(folder)
+    
+    @jobs = list_files(folder)
+  end
+
+  def stats_header
+      res=['']
+
+      res << "#{'Job name'.ljust(40)}\tCPUs\tStatus"
+
+      return res.join("\n")
+
+  end
+
+  def to_s
+    res=[]
+
+    res <<  "="*80
+    
+    @jobs.each do |job|
+      # puts job
+      res << "#{job[:name]}\t#{job[:cpus]}\t#{self.class.to_s[0]}"
+    end
+    
+    return res.join("\n")
+    
+  end
+  
+  
+private
+
+  def parse_job_file(job)
+    h={}
+    
+    h[:file] = job
+    text=File.read(job)
+
+    cpus = 1
+    cwd=''
+    script=''
+    run_script=''
+
+    # parse script
+    text.each_line do |line|
+      if line.upcase =~ /^\s*#\s*CPUS\s*=\s*(\d+)/
+        begin
+          cpus = $1.to_i
+        rescue
+          cpus = 1
+        end
+      end
+
+      if line.upcase =~ /^\s*#\s*CWD\s*=\s*(.+)/
+        begin
+          # use without upcase for path
+          line =~ /^\s*#\s*CWD\s*=\s*(.+)/
+          cwd = $1
+        rescue
+          cwd = ''
+        end
+      end
+      
+      if line.upcase =~ /^\s*#\s*SCRIPT\s*=\s*(.+)/
+        begin
+          # use without upcase for path
+          line =~ /^\s*#\s*SCRIPT\s*=\s*(.+)/
+          script = $1
+        rescue
+          script = ''
+        end
+      end
+      
+      if line.upcase =~ /^\s*#\s*RUN_SCRIPT\s*=\s*(.+)/
+        begin
+          # use without upcase for path
+          line =~ /^\s*#\s*RUN_SCRIPT\s*=\s*(.+)/
+          run_script = $1
+        rescue
+          run_script = ''
+        end
+      end
+      
+      
+    end
+
+    h[:name] = File.basename(job)
+    h[:cpus] = cpus
+    h[:cwd] = cwd
+    h[:script] = script
+    h[:run_script] = run_script
+
+    return h
+  end
+
+  def list_files(folder)
+    res =[]
+    # puts folder
+    Dir.glob(File.join(folder,'*')).entries.each do |job|
+      h=parse_job_file(job)
+      res << h
+    end
+
+    return res
+  end
+  
+
+  
+end

data/lib/scbi_queue_system/queued_job_list.rb

@@ -0,0 +1,57 @@
+require 'job_list'
+
+
+class QueuedJobList < JobList
+
+  def initialize
+
+    super(QUEUED_PATH)
+
+  end
+
+  def self.execute_job(job,machine)
+
+    $LOG.info("Sending job #{job[:name]} to #{machine}")
+
+    job_path=File.join(QUEUED_PATH,job[:name])
+
+    machine_path = File.join(SENT_PATH,machine)
+
+    FileUtils.cp(job_path , File.join(RUNNING_PATH,machine))
+    FileUtils.mv(job_path , machine_path)
+
+    # launch_cmd="ssh #{machine} \"bash #{File.join(machine_path,job[:name])}\""
+
+
+    output_file=File.join(job[:cwd],job[:name]+'.out')
+    error_file=File.join(job[:cwd],job[:name]+'.error')
+
+    # output_file=File.join('/tmp',job[:name]+'.out')
+    # error_file=File.join('/tmp',job[:name]+'.error')
+    copy_cmd = "scp #{File.join(machine_path,job[:name])} #{machine}:#{job[:cwd]}"
+    
+    if system(copy_cmd)
+      # launch_cmd = "ssh #{machine} \"nohup bash < #{job[:script]} > #{output_file} 2> #{error_file} & \""
+      script_on_machine = File.join(job[:cwd],job[:name])
+
+      if machine.upcase.index('LOCALHOST')
+        launch_cmd = "nohup #{script_on_machine} </dev/null > #{output_file} 2> #{error_file} &"
+      else
+        launch_cmd = "ssh #{machine} \"nohup #{script_on_machine} </dev/null > #{output_file} 2> #{error_file} &\""
+      end
+
+      $LOG.info("Launch cmd: #{launch_cmd}")
+      if system(launch_cmd)
+        $LOG.info "LAUNCH DONE"
+      else
+        $LOG.error "LAUNCH FAILED: #{launch_cmd}"
+      end
+    else
+      $LOG.error "SCP FAILED: #{copy_cmd}"
+    end
+
+
+
+  end
+
+end

data/lib/scbi_queue_system/running_job_list.rb

@@ -0,0 +1,67 @@
+require 'job_list'
+
+class RunningJobList < JobList
+
+  def initialize
+    super(RUNNING_PATH)
+  end
+
+
+  def running_jobs_in_machine(machine)
+    return list_machine_files(RUNNING_PATH,machine)
+  end
+
+  def running_cpus_in_machine(machine)
+    $LOG.debug(machine)
+    res=0
+
+    running_jobs_in_machine(machine).each do |job|
+      res+=job[:cpus]
+    end
+
+    return res
+  end
+
+  def to_s
+    res=[]
+
+    @jobs.each do |machine,jobs|
+      # puts job
+      jobs.each do |job|
+        res<< "#{job[:name]}\t#{job[:cpus]}\tRUNNING\t #{machine}"
+      end
+    end
+
+    return res.join("\n")
+
+  end
+
+  private
+
+    def list_machine_files(folder, machine_id)
+      res = []
+      Dir.glob(File.join(folder,machine_id,'*')).entries.each do |job|
+
+        h=parse_job_file(job)
+        
+        res << h
+      end
+
+      return res
+    end
+
+    # process currently running files
+    def list_files(folder)
+      res = {}
+
+      Dir.glob(File.join(folder,'*')).entries.each do |machine|
+        machine_id=File.basename(machine)
+
+        res[machine_id]= list_machine_files(folder,machine_id)
+      end
+
+      return res
+    end
+
+
+end

data/script/console

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+# File: script/console
+irb = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb'
+
+libs =  " -r irb/completion"
+# Perhaps use a console_lib to store any extra methods I may want available in the cosole
+# libs << " -r #{File.dirname(__FILE__) + '/../lib/console_lib/console_logger.rb'}"
+libs <<  " -r #{File.dirname(__FILE__) + '/../lib/scbi_queue_system.rb'}"
+puts "Loading scbi_queue_system gem"
+exec "#{irb} #{libs} --simple-prompt"

data/script/destroy

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/destroy'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
+RubiGen::Scripts::Destroy.new.run(ARGV)

data/script/generate

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/generate'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
+RubiGen::Scripts::Generate.new.run(ARGV)

data/test/submit_script.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+ls
+sleep 30
+ls
+

data/test/test_helper.rb

@@ -0,0 +1,3 @@
+require 'stringio'
+require 'test/unit'
+require File.dirname(__FILE__) + '/../lib/scbi_queue_system'

data/test/test_scbi_queue_system.rb

@@ -0,0 +1,11 @@
+require File.dirname(__FILE__) + '/test_helper.rb'
+
+class TestScbiQueueSystem < Test::Unit::TestCase
+
+  def setup
+  end
+  
+  def test_truth
+    assert true
+  end
+end

metadata

@@ -0,0 +1,105 @@
+--- !ruby/object:Gem::Specification 
+name: scbi_queue_system
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.0.2
+platform: ruby
+authors: 
+- Dario Guerrero
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-07-12 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: json
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.5.3
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 2.8.0
+  type: :development
+  version_requirements: *id002
+description: scbi_queue_system (SQS) handles a simple queue of jobs executions over multiple machines (clustered installation) or your own personal computer.
+email: 
+- dariogf@gmail.com
+executables: 
+- queue_manager.rb
+- sqstat
+- sqsub
+- sqs_install_daemon
+extensions: []
+
+extra_rdoc_files: 
+- History.txt
+- Manifest.txt
+- PostInstall.txt
+files: 
+- bin/queue_manager.rb
+- bin/sqstat
+- bin/sqsub
+- bin/sqs_install_daemon
+- History.txt
+- lib/scbi_queue_system/autolaunch/com.scbi_queue_system.plist
+- lib/scbi_queue_system/autolaunch/sqsd-linux
+- lib/scbi_queue_system/done_job_list.rb
+- lib/scbi_queue_system/internal_config/internal_config.json
+- lib/scbi_queue_system/job_list.rb
+- lib/scbi_queue_system/queued_job_list.rb
+- lib/scbi_queue_system/running_job_list.rb
+- lib/scbi_queue_system.rb
+- Manifest.txt
+- PostInstall.txt
+- Rakefile
+- README.rdoc
+- script/console
+- script/destroy
+- script/generate
+- test/submit_script.sh
+- test/test_helper.rb
+- test/test_scbi_queue_system.rb
+homepage: http://www.scbi.uma.es/downloads
+licenses: []
+
+post_install_message: PostInstall.txt
+rdoc_options: 
+- --main
+- README.rdoc
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+requirements: []
+
+rubyforge_project: scbi_queue_system
+rubygems_version: 1.7.2
+signing_key: 
+specification_version: 3
+summary: scbi_queue_system (SQS) handles a simple queue of jobs executions over multiple machines (clustered installation) or your own personal computer.
+test_files: 
+- test/test_helper.rb
+- test/test_scbi_queue_system.rb