torque_rm 0.1.0

data/Rakefile

@@ -0,0 +1,52 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "torque_rm"
+  gem.homepage = "http://github.com/helios/torque_rm"
+  gem.license = "MIT"
+  gem.summary = %Q{TORQUE Resource Manager wrapper for Ruby}
+  gem.description = %Q{TORQUE Resource Manager for Ruby. Submit, check and control your job directly from Ruby.}
+  gem.email = "ilpuccio.febo@gmail.com"
+  gem.authors = ["Raoul Jean Pierre Bonnal"]
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+
+RSpec::Core::RakeTask.new(:rcov) do |spec|
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+require 'cucumber/rake/task'
+Cucumber::Rake::Task.new(:features)
+
+task :default => :spec
+
+require 'rdoc/task'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "torque_rm #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/bin/torque_rm_rest

@@ -0,0 +1,147 @@
+#!/usr/bin/env ruby
+
+require 'haml'
+require 'sinatra'
+require 'torque_rm'
+require 'sinatra/twitter-bootstrap'
+require 'sinatra/json'
+require 'sinatra/respond_with'
+# require 'ap'
+require "#{File.dirname(__FILE__)}/../web/helpers/qstat.rb"
+
+
+set :run, true
+enable :run
+
+register Sinatra::Twitter::Bootstrap::Assets
+
+set :root, File.join(File.dirname(__FILE__),"..","web")
+
+
+helpers TORQUE::Qstat::Sinatra::Helpers
+
+get '/' do
+  'Welcome to our gorgeous TORQUE/PBS web interface.'.to_json
+end
+
+
+get '/qstat', :provides => [:json, :html] do
+  jobs = TORQUE::Qstat.new.query
+  respond_to do |format|
+    format.json do
+      jobs.to_json
+    end
+    format.html { haml :qstat, :format => :html5, :locals => {:jobs => jobs}}
+ end
+end
+
+get '/qstat/fields', :provides => [:json] do
+
+  json TORQUE::Qstat::FIELDS
+end
+
+get '/qstat/all/:field', :provides => [:json] do |field|
+  if TORQUE::Qstat::FIELDS.include? field
+    jobs = TORQUE::Qstat.new.query
+    json jobs.map{|job| job[field.to_sym]}
+  else
+    return status 404
+  end
+end
+
+
+### JOBS
+
+get '/qstat/:job_id', :provides => [:json] do |job_id|
+  respond_to do |format|
+    format.json do
+    	job = TORQUE::Qstat.new.query job_id: job_id
+      # q = TORQUE::Qstat.new
+      # p = TORQUE::Qstat::Parser.new
+      # t = TORQUE::Qstat::Trans.new
+
+      # s= "Job Id: 2770.deep.space.nine\n    Job_Name = STDIN\n    Job_Owner = johndoe@deep.space.nine\n    resources_used.cput = 00:00:00\n    resources_used.mem = 3148kb\n    resources_used.vmem = 32528kb\n    resources_used.walltime = 00:28:05\n    job_state = R\n    queue = bio\n    server = deep.space.nine\n    Checkpoint = u\n    ctime = Mon Sep  2 18:43:10 2013\n    Error_Path = deep.space.nine:/mnt/bio/ngs/data/home/johndoe/STDIN.e2770\n    exec_host = scrapper/0\n    exec_port = 15003\n    Hold_Types = n\n    Join_Path = n\n    Keep_Files = n\n    Mail_Points = a\n    mtime = Mon Sep  2 18:43:11 2013\n    Output_Path = deep.space.nine:/mnt/bio/ngs/data/home/johndoe/STDIN.o2770\n\t\n    Priority = 0\n    qtime = Mon Sep  2 18:43:10 2013\n    Rerunable = True\n    Resource_List.ncpus = 2\n    Resource_List.nice = 0\n    session_id = 16963\n    Variable_List = PBS_O_QUEUE=bio,PBS_O_HOST=deep.space.nine,\n\tPBS_O_HOME=/mnt/bio/ngs/data/home/johndoe,PBS_O_LANG=it_IT.UTF-8,\n\tPBS_O_LOGNAME=johndoe,\n\tPBS_O_PATH=/mnt/bio/ngs/data/opt/fastx_toolkit/bin:/mnt/bio/ngs/data/\n\topt/bedtools/bin:/mnt/bio/ngs/data/opt/fuseki:/mnt/bio/ngs/data/opt/je\n\tna/bin:/mnt/bio/ngs/data/opt/CASAVA/bin:/mnt/bio/ngs/data/bin/cloaked-\n\thipster/utils:/mnt/bio/ngs/data/bin/cloaked-hipster/utils/ngs/pipeline\n\ts:/mnt/bio/ngs/data/bin/cloaked-hipster/utils/db:/mnt/bio/ngs/data/opt\n\t/trimmomatic:/mnt/bio/ngs/data/opt/PHYLOCSF/hmmer-3.0-linux-intel-x86_\n\t64/binaries:/mnt/bio/ngs/data/opt/PHYLOCSF/mlin-PhyloCSF-983a652/:/mnt\n\t/bio/ngs/data/bin/cloaked-hipster/ncrnapp/PBS_pipeline/:/mnt/bio/ngs/d\n\tata/opt/bowtie-0.12.9:/mnt/bio/ngs/data/opt/bowtie_current:/mnt/bio/ng\n\ts/data/opt/STAR_current:/mnt/bio/ngs/data/opt/RSeQC-2.3.5/RQC/usr/loca\n\tl/bin:/mnt/bio/ngs/data/opt/tophat_current:/mnt/bio/ngs/data/opt/PASA/\n\tscripts:/mnt/bio/ngs/data/opt/PASA/bin:/mnt/bio/ngs/data/opt/fasta/bin\n\t:/mnt/bio/ngs/data/opt/cufflinks:/mnt/bio/ngs/data/opt/samtools:/mnt/b\n\tio/ngs/data/opt/samtools/bcftools:/mnt/bio/ngs/data/opt/samtools/misc:\n\t/mnt/bio/ngs/data/opt/STAR_current/bin:/mnt/bio/ngs/data/opt/gmap/bin:\n\t/mnt/bio/ngs/data/opt/velvet:/mnt/bio/ngs/data/opt/velvet/contrib/colu\n\tmbus_scripts:/opt/blat:/mnt/bio/ngs/data/opt/oases/:/mnt/bio/ngs/data/\n\topt/oases/scripts:/mnt/bio/ngs/data/opt/trinityrnaseq:/mnt/bio/ngs/dat\n\ta/opt/trinityrnaseq/util:/usr/local/bin/:/mnt/bio/ngs/data/home/bonnal\n\traoul/.rbenv/bin:/mnt/bio/ngs/data/home/johndoe/.rbenv/shims:/mnt/\n\tbio/ngs/data/home/johndoe/.rbenv/bin:/usr/local/sbin:/usr/local/bi\n\tn:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games,\n\tPBS_O_MAIL=/var/mail/johndoe,PBS_O_SHELL=/bin/bash,\n\tPBS_SERVER=spark,PBS_O_WORKDIR=/mnt/bio/ngs/data/home/johndoe\n    etime = Mon Sep  2 18:43:10 2013\n    submit_args = -l ncpus=2 -l nice=0\n    start_time = Mon Sep  2 18:43:11 2013\n    start_count = 1\n    fault_tolerant = False\n    submit_host = deep.space.nine"
+      # job = job_id == "2770" ? q.mock(t.apply(p.parse(s.gsub(/\n\t/,'')))) : []
+      if job.empty?
+        return status 404
+      else
+        json :job => job.first
+      end
+    end
+  end
+	# if job.empty?
+ #    "There is no job running with id #{job_id}"
+	# else
+
+ #  	haml :qstat_job, :format => :html5, :locals => {:job => job.first} #get the first and unique elment if the array is not empty
+ #  end
+end
+
+get '/qstat/job/fields', :provides => [:json] do
+  json :job_fields => TORQUE::Qstat::Job.fields
+end
+
+get '/qstat/:job_id/:field', :provides => [:json] do |job_id, field|
+
+
+  # puts TORQUE::Qstat::Job.fields
+  if TORQUE::Qstat::Job.fields.include? field
+    # q = TORQUE::Qstat.new
+    # p = TORQUE::Qstat::Parser.new
+    # t = TORQUE::Qstat::Trans.new
+    # s= "Job Id: 2770.deep.space.nine\n    Job_Name = STDIN\n    Job_Owner = johndoe@deep.space.nine\n    resources_used.cput = 00:00:00\n    resources_used.mem = 3148kb\n    resources_used.vmem = 32528kb\n    resources_used.walltime = 00:28:05\n    job_state = R\n    queue = bio\n    server = deep.space.nine\n    Checkpoint = u\n    ctime = Mon Sep  2 18:43:10 2013\n    Error_Path = deep.space.nine:/mnt/bio/ngs/data/home/johndoe/STDIN.e2770\n    exec_host = scrapper/0\n    exec_port = 15003\n    Hold_Types = n\n    Join_Path = n\n    Keep_Files = n\n    Mail_Points = a\n    mtime = Mon Sep  2 18:43:11 2013\n    Output_Path = deep.space.nine:/mnt/bio/ngs/data/home/johndoe/STDIN.o2770\n\t\n    Priority = 0\n    qtime = Mon Sep  2 18:43:10 2013\n    Rerunable = True\n    Resource_List.ncpus = 2\n    Resource_List.nice = 0\n    session_id = 16963\n    Variable_List = PBS_O_QUEUE=bio,PBS_O_HOST=deep.space.nine,\n\tPBS_O_HOME=/mnt/bio/ngs/data/home/johndoe,PBS_O_LANG=it_IT.UTF-8,\n\tPBS_O_LOGNAME=johndoe,\n\tPBS_O_PATH=/mnt/bio/ngs/data/opt/fastx_toolkit/bin:/mnt/bio/ngs/data/\n\topt/bedtools/bin:/mnt/bio/ngs/data/opt/fuseki:/mnt/bio/ngs/data/opt/je\n\tna/bin:/mnt/bio/ngs/data/opt/CASAVA/bin:/mnt/bio/ngs/data/bin/cloaked-\n\thipster/utils:/mnt/bio/ngs/data/bin/cloaked-hipster/utils/ngs/pipeline\n\ts:/mnt/bio/ngs/data/bin/cloaked-hipster/utils/db:/mnt/bio/ngs/data/opt\n\t/trimmomatic:/mnt/bio/ngs/data/opt/PHYLOCSF/hmmer-3.0-linux-intel-x86_\n\t64/binaries:/mnt/bio/ngs/data/opt/PHYLOCSF/mlin-PhyloCSF-983a652/:/mnt\n\t/bio/ngs/data/bin/cloaked-hipster/ncrnapp/PBS_pipeline/:/mnt/bio/ngs/d\n\tata/opt/bowtie-0.12.9:/mnt/bio/ngs/data/opt/bowtie_current:/mnt/bio/ng\n\ts/data/opt/STAR_current:/mnt/bio/ngs/data/opt/RSeQC-2.3.5/RQC/usr/loca\n\tl/bin:/mnt/bio/ngs/data/opt/tophat_current:/mnt/bio/ngs/data/opt/PASA/\n\tscripts:/mnt/bio/ngs/data/opt/PASA/bin:/mnt/bio/ngs/data/opt/fasta/bin\n\t:/mnt/bio/ngs/data/opt/cufflinks:/mnt/bio/ngs/data/opt/samtools:/mnt/b\n\tio/ngs/data/opt/samtools/bcftools:/mnt/bio/ngs/data/opt/samtools/misc:\n\t/mnt/bio/ngs/data/opt/STAR_current/bin:/mnt/bio/ngs/data/opt/gmap/bin:\n\t/mnt/bio/ngs/data/opt/velvet:/mnt/bio/ngs/data/opt/velvet/contrib/colu\n\tmbus_scripts:/opt/blat:/mnt/bio/ngs/data/opt/oases/:/mnt/bio/ngs/data/\n\topt/oases/scripts:/mnt/bio/ngs/data/opt/trinityrnaseq:/mnt/bio/ngs/dat\n\ta/opt/trinityrnaseq/util:/usr/local/bin/:/mnt/bio/ngs/data/home/bonnal\n\traoul/.rbenv/bin:/mnt/bio/ngs/data/home/johndoe/.rbenv/shims:/mnt/\n\tbio/ngs/data/home/johndoe/.rbenv/bin:/usr/local/sbin:/usr/local/bi\n\tn:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games,\n\tPBS_O_MAIL=/var/mail/johndoe,PBS_O_SHELL=/bin/bash,\n\tPBS_SERVER=spark,PBS_O_WORKDIR=/mnt/bio/ngs/data/home/johndoe\n    etime = Mon Sep  2 18:43:10 2013\n    submit_args = -l ncpus=2 -l nice=0\n    start_time = Mon Sep  2 18:43:11 2013\n    start_count = 1\n    fault_tolerant = False\n    submit_host = deep.space.nine"
+    job = TORQUE::Qstat.new.query job_id: job_id
+    # job = q.mock(t.apply(p.parse(query.gsub(/\n\t/,''))))
+    json field.to_sym => job.first[field.to_sym]
+  else
+    return status 404
+  end
+  # if job.empty?
+ #    "There is no job running with id #{job_id}"
+  # else
+
+ #    haml :qstat_job, :format => :html5, :locals => {:job => job.first} #get the first and unique elment if the array is not empty
+ #  end
+end
+
+post '/qstat/:job_id/del', :provides => [:json] do |job_id|
+  job = TORQUE::Qstat.new.query job_id: job_id
+  if job.empty?
+    return status 404
+  else
+    json job.first.del
+  end
+ 
+end
+
+
+
+__END__
+
+@@ layout
+%html
+  %head
+    = bootstrap_assets
+  %body
+    - container :fluid do
+      .navbar
+        .navbar-inner
+          %a{class:"brand", href:"#"} TORQUE PBS
+          %ul{class:"nav"}
+            %li{class:"active"}
+              %a{href:"#"} Getting Started
+            %li 
+              %a{href:"#"} Documentation
+            %li 
+              %a{href:"#"} REST Api
+            %li 
+              %a{href:"#"} About
+            %li 
+              %a{href:"#"} Contact
+ 
+
+      - row do
+        - span8 do
+          = yield
+        - span4 :offset => 4 do
+          footer

data/features/step_definitions/common_steps.rb

@@ -0,0 +1,21 @@
+Given(/^the server "(.*?)"$/) do |server_name|
+  TORQUE.server = server_name # express the regexp above with the code you wish you had
+end
+
+
+When(/^I request the server name without specify the hostname$/) do
+  @server = TORQUE.server
+end
+
+When(/^I request the server name$/) do
+  @server = TORQUE.server # express the regexp above with the code you wish you had
+end
+
+# Then(/^I should get a "(.*?)"$/) do |reply|
+#   @server.should be_an_instance_of(Rye::Box) # express the regexp above with the code you wish you had
+# end
+
+Then(/^I should get a "(.*?)" on "(.*?)"$/) do |ssh_manager_object, hostname|
+  @server.should be_an_instance_of(Object.const_get(ssh_manager_object)) # express the regexp above with the code you wish you had
+  @server.host.should == hostname
+end

data/features/step_definitions/qsub_steps.rb

@@ -0,0 +1,47 @@
+Before do
+  @qsub = TORQUE::Qsub.new
+end
+
+Given /^I want to be notified if the job begins$/ do
+  @qsub.m << "b"
+end
+
+Given /^I want to be notified if the job ends$/ do
+  @qsub.m << "e"
+end
+
+Given /^I want to be notified if the job aborts$/ do
+  @qsub.m << "a"
+end
+
+Given(/^the name "(.*?)"$/) do |job_name|
+  @qsub.name = job_name
+end
+
+
+Given(/^I want to use the "(.*?)" shell$/) do |job_shell|
+  @qsub.shell = job_shell
+end
+
+Given(/^I want to set the "(.*?)" to "(.*?)"$/) do |qsub_parameter, value|
+  @qsub.send "#{qsub_parameter}=", value # express the regexp above with the code you wish you had
+end
+
+Given(/^I want to use (\d+) cpus for a single node$/) do |n_cpus|
+  @qsub.cpus = n_cpus # express the regexp above with the code you wish you had
+end
+
+
+Given(/^the command "(.*?)"$/) do |job_command|
+  @qsub.script = job_command # express the regexp above with the code you wish you had
+end
+
+Then(/^I should get a configuration like$/) do |pbs_script|
+  @qsub.to_s.should == pbs_script # express the regexp above with the code you wish you had
+end
+
+ Then(/^I should get the pbs script$/) do |pbs_script|
+  @qsub.to_s.should == pbs_script # express the regexp above with the code you wish you had
+end
+
+

data/features/support/env.rb

@@ -0,0 +1,13 @@
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+
+$LOAD_PATH.unshift(File.dirname(__FILE__) + '/../../lib')
+require 'torque_rm'
+
+require 'rspec/expectations'

data/features/torque_rm.feature

@@ -0,0 +1,9 @@
+#Feature: something something
+#  In order to something something
+#  A user something something
+#  something something something
+
+#  Scenario: something something
+#    Given inspiration
+#    When I create a sweet new gem
+#    Then everyone should see how awesome I am

data/features/torque_rm/common.feature

@@ -0,0 +1,22 @@
+Feature: Configure torque_rm
+  In order to interact with the Torque server
+  As a user I must configure the system
+
+  Scenario: Torque is the localhost
+    When I request the server name without specify the hostname
+    Then I should get a "Rye::Box" on "localhost"
+
+  Scenario: Torque is a remote machine
+    Given the server "torque.remote.net"
+    When I request the server name
+    Then I should get a "Rye::Box" on "torque.remote.net"
+
+  Scenario: Torque binary are installed in the default location
+    Given the server "torque.remote.net"
+    When I try to get a qstat
+    Then the server tell me it can execute the command
+
+  Scenario: Torque binary are not installed in the default location
+    Given the server "torque.remote.net"
+    When I try to get a qstat
+    Then the server tell me it can not execute the command

data/features/torque_rm/qsub.feature

@@ -0,0 +1,52 @@
+Feature: Submit jobs
+  In order to submit a job
+  A user should be able to create a batch job
+  prividing multiple PBS options
+  attaching a shell script
+
+  Scenario: Configure a batch job
+    Given the name "MyBatch"
+    And I want to use the "/bin/bash" shell    
+    And I want to be notified if the job begins
+    And I want to be notified if the job ends
+    And I want to be notified if the job aborts
+    Then I should get a configuration like
+      """
+      #PBS -m bea
+      #PBS -N MyBatch
+      #PBS -S /bin/bash
+
+      """
+
+Scenario: Prepare a script to get the hostname from a node
+    Given the name "MyBatch"
+    And I want to use the "/bin/bash" shell    
+    And I want to be notified if the job begins
+    And I want to be notified if the job ends
+    And I want to be notified if the job aborts
+    And I want to set the "working_directory" to "/home/submitter/test_cluster"
+    And I want to set the "stdout" to "/home/submitter/test_cluster"
+    And I want to set the "stderr" to "/home/submitter/test_cluster"
+    And I want to use 4 cpus for a single node
+    And the command "echo `hostname`"
+    Then I should get the pbs script
+      """
+      #PBS -d /home/submitter/test_cluster
+      #PBS -e /home/submitter/test_cluster
+      #PBS -l nodes=1:ppn=4
+      #PBS -m bea
+      #PBS -N MyBatch
+      #PBS -o /home/submitter/test_cluster
+      #PBS -S /bin/bash
+      echo `hostname`
+      """
+
+
+  Scenario: Get the hostname from a node
+    Given the name "MyBatch"
+    And I want to use the "/bin/bash" shell    
+    And I want to be notified if the job begins
+    And I want to be notified if the job ends
+    And I want to be notified if the job aborts
+    And the command "echo `hostname`"
+    Then I should get the name of the execution node

data/lib/torque_rm.rb

@@ -0,0 +1,11 @@
+require 'parslet'
+require 'yaml'
+require 'colorize'
+require 'terminal-table'
+require 'torque_rm/common'
+require 'torque_rm/qsub'
+require 'torque_rm/qstat'
+require 'torque_rm/qdel'
+
+# Try to laod the default configuration
+TORQUE.load_config

data/lib/torque_rm/common.rb

@@ -0,0 +1,74 @@
+require 'rye'
+require 'etc'
+
+module TORQUE
+  @@username = Etc.getlogin
+	@@qcommands_path = '/usr/bin'
+  @@master = Rye::Box.new("localhost")
+  @@conf = {}
+  def self.server=(hostname)
+  	if hostname
+  		@@master = Rye::Box.new(hostname, @@conf)
+  	end
+  end
+
+  def self.server
+  	@@master
+  end
+
+	def self.username=(username)
+		@@username = username
+	end
+
+
+  def self.qcommands_path=(path)
+  	@@qcommands_path = path
+  	%w(qstat qsub qdel).each do |command|
+  	  Rye::Cmd.remove_command command if Rye::Cmd.can? command
+  	  Rye::Cmd.add_command command, File.join(qcommands_path, command)
+  	end
+  end
+
+  def self.qcommands_path
+  	@@qcommands_path
+  end
+
+  def self.path
+  	self.qcommands_path
+  end
+
+  def self.read_config(file)
+  	if File.exists?(file)
+  	  conf = YAML::load( File.open( file) )
+      @@conf = conf.dup
+      @@conf.delete(:hostname)
+      @@conf.delete(:path)
+  	  self.qcommands_path = conf[:path]
+    	self.username = conf[:user] unless conf[:user].nil?
+      # self.port = conf[:port] unless conf[:port].nil?
+      # self.password = conf[:password] unless conf[:password].nil?
+  	  self.server = conf[:hostname]
+		end
+  end
+
+  # Load configuration, default from file in user home with name .toruqe_rm.yaml
+  def self.load_config(file=nil)
+  	self.read_config File.expand_path(file.nil? ? "~/.torque_rm.yaml" : file)
+  	self
+  end
+
+  # Save configuration, default in user home with name .toruqe_rm.yaml
+  def self.save_config(file=nil)
+  	File.write File.expand_path(file.nil? ? "~/.torque_rm.yaml" : file), {hostname: @@master.host, path: @@qcommands_path, user: @@username}.to_yaml
+  end
+
+  # Get the host name/ip of the local/remote server user as submitter/interface to PBS
+  def self.host
+  	self.server.host
+  end
+
+  # Get the hostname, this may require an internet connection and fully qualified name
+  def self.hostname
+  	self.server.hostname
+  end
+end