condor 1.1.0 → 2.0.0

data/README

@@ -8,19 +8,27 @@ It consists of a wrapper for a Condor submit file and a program, <tt>condor-exec
 
 queues up a job for the program +prog+ with arguments <tt>--option arg1 arg2</tt>.  To pass options to <tt>condor-exec</tt> itself, place them before +prog+.  For example:
 
-	condor-exec --universe=MPI prog --option arg1 arg2
+	condor-exec --output=myoutput prog --option arg1 arg2
 
-Alternately arguments may be specified in a separate arguments file, and one
-Condor job will be queued for each line in the file
+Alternately, arguments may be specified in a separate arguments file, and one
+Condor job will be queued for each line in the file.
 
 	condor-exec --argfile=args prog
 
 See <tt>condor-exec --help</tt> for more information.
 
+An <tt>/etc/ruby-condor.yaml</tt> file may be used to specify default attributes for all processes.  This is a YAML encoding a of hash.  For example, the following will specify the Universe, Getenv, and Log values:
+
+	--- 
+	Universe: vanilla
+	Getenv: "true"
+	Log: /tmp/$ENV(USER).condor.$(Cluster)
+
 = History
 
 1.0.0:: Basic submit file wrapper; <tt>condor-exec</tt> program
 1.1.0:: Additional options in <tt>condor-exec</tt>, including argument files; Add logging support
+2.0.0:: Load defaults from configuration file; change command line of condor-exec
 
 = Copyright
 

data/bin/condor-exec

@@ -55,6 +55,9 @@ Switch options appearing before the first non-switch argument are passed to
 '--universe=vanilla' and not '--universe vanilla'). Switch options appearing
 after the first non-switch argument are treated as arguments to the command.
 
+The config file option is used to add arbitrary Condor attributes to the header
+of the submit file. The configuration file should be a YAML encoding of a hash.
+
 If an argument file is specified, one Condor job is created for each line in
 the file, using the text of the line as arguments.  These arguments are
 appended after any specified on the command line.
@@ -65,6 +68,11 @@ EOS
     options[:argfile] = value
   end
 
+  opts.on("-cVALUE", "--configfile VALUE",
+          "Configuration filename") do |value|
+    options[:configfile] = value
+  end
+
   opts.on("-l", "--logging LEVEL", "Logging level") do |level|
     Condor.set_log_level(eval("Logger::#{level.upcase}"))
   end
@@ -77,23 +85,20 @@ EOS
     options[:error] = value
   end
 
-  opts.on("-gVALUE", "--getenv VALUE", "Get environment") do |value|
-    options[:getenv] = value
-  end
-
-  opts.on("-tVALUE", "--transfer-executable VALUE", "Transfer executable") do |value|
-    options[:transfer_executable] = value
-  end
-
-  opts.on("-uVALUE", "--universe VALUE", "Universe") do |value|
-    options[:universe] = value
-  end
-  
-  opts.on("-s", "--[no-]submit", "no-submit generates the submit file but does not call condor_submit") do |value|
+  opts.on("-s", "--[no-]submit",
+          "no-submit generates the submit file but does not call condor_submit") do |value|
     options[:submit] = value
   end
 
 end
 
 command_line = p.parse_command_line
-Condor.submit(command_line.shift, command_line, options)
+begin
+  Condor.submit(command_line.shift, command_line, options)
+rescue Condor::ExecutableException => e
+  puts e
+  if e.instance_of?(Condor::NoExecutableException) and
+     not options == {:submit => true}
+    puts "Did you leave a space in a condor-exec option?  (e.g. --output myoutput instead of --output=myoutput)"
+  end
+end

data/lib/condor.rb

@@ -16,13 +16,13 @@
 # editalign; if not, write to the Free Software Foundation, Inc., 51 Franklin
 # St, Fifth Floor, Boston, MA 02110-1301 USA
 
-require "erb"
 require "logger"
+require "pathname"
 require "tempfile"
 
 # Wrapper for the Condor distributed computing system.
 module Condor
-  VERSION = "1.1.0"
+  VERSION = "2.0.0"
 
   # Create the logger and set its default log level to ERROR.  This function
   # is called when the module is loaded.
@@ -46,71 +46,100 @@ module Condor
     Condor::LOGGER.level = level
   end
 
-  # The text of a file that can be sent to the condor_submit program.
-  class SubmitFile < ERB
-    # Specify a single executable and a list of argument sets.  Each element
-    # of argument_list is a set of arguments.  One job will be queued up for
-    # each set of arguments.
-    def initialize(executable, argument_list, options = {})
-      # Get defaults for any unspecified options.
-      @options = {
-        :getenv => "true",
-        :transfer_executable => "false",
-        :universe => "vanilla"}.merge!(options)
-      # Create the submit file text.
+  # Parent class for exceptions raised by this module because of bad
+  # executable files.
+  class ExecutableException < Exception
+    attr_reader :executable
+
+    def initialize(executable, message)
       @executable = executable
-      @argument_list = argument_list
-      @log = File.join(Dir::tmpdir, "#{ENV['USER']}.condor.$(Cluster).$(Process)")
-      @output = options.fetch(:output, current_dir_log_file("output"))
-      @error = options.fetch(:error, current_dir_log_file("error"))
+      super(message)
+    end
+  end
+
+  # Exception thrown by SubmitFile when the specified program does not exist.
+  class NoExecutableException < ExecutableException
+    def initialize(executable)
+      super(executable, "Program '#{executable}' does not exist.")
+    end
+  end
+
+  # Exception thrown by SubmitFile when the specified program does not have
+  # executable permissions set.
+  class ExecutePermissionException < ExecutableException
+    def initialize(executable)
+      super(executable, "Program '#{executable}' is not executable.")
+    end
+  end
+
+  # The text of a file that can be sent to the <tt>condor_submit</tt> program.
+  class SubmitFile
+    # Specify a single executable and a list of argument sets.  Each element
+    # of <tt>argument_list</tt> is a set of arguments.  One job will be queued
+    # up for each set of arguments.
+    def initialize(executable, arguments, options = {},
+                   output = nil, error = nil)
+      executable.strip!
+      # Read default options from a system-wide configuratation file if one
+      # exists.
+      options = configuration_options.merge(options)
       # Create the header common to all the jobs.
-      template = <<-EOTEXT
-Universe   = <%= @options[:universe] %>
-getenv     = <%= @options[:getenv] %>
-transfer_executable = <%= @options[:transfer_executable] %>
-Log        = <%= @log %>
-Executable = <%= @executable %>
-      EOTEXT
+      @submit = (options.keys.select do |attribute|
+        not options[attribute].nil?
+      end.collect do |attribute|
+        "#{attribute} = #{options[attribute]}"
+      end + ["executable = #{executable}"]).join("\n") + "\n"
       # Queue up the arguments for each individual job.
-      argument_list.each_with_index do |arguments, i|
-        template += <<-EOTEXT
+      output = current_dir_log_file(executable, "output") if output.nil?
+      error = current_dir_log_file(executable, "error") if error.nil?
+      arguments.each do |job_arguments|
+        @submit += <<-EOTEXT
 
-arguments  = <%= @argument_list[#{i}].join(" ") %>
-output     = <%= @output %>
-error      = <%= @error %>
-Queue
+\targuments  = #{job_arguments.join(' ')}
+\toutput     = #{output}
+\terror      = #{error}
+\tQueue
         EOTEXT
       end
-      super(template)
-    end
-
-    # Allocate the name of a temporary file in the current directory.
-    def current_dir_log_file(name)
-      "#{File.basename(@executable).strip}.$(Cluster).$(Process).#{name}"
     end
 
     # Return a string that can serve as a Condor submit file.
     def to_s
-      result(binding)
+      @submit
     end
-    
-    private :current_dir_log_file
+
+    # Read system-wide options from a YAML configuration file.  Return an
+    # empty hash if the file does not exist.
+    def configuration_options
+      config = Pathname.new("/etc/ruby-condor.yaml")
+      config.file? ? open(config) {|f| YAML.load(f)} : {}
+    end
+
+    # The name of a temporary file in the current directory.
+    def current_dir_log_file(executable, name)
+      "#{File.basename(executable)}.$(Cluster).$(Process).#{name}"
+    end
+
+    private :current_dir_log_file, :configuration_options
   end
 
 
-  # This runs the specified executable and arguments as a Condor command.  The
-  # function creates a condor_submit.nnn file in the current directory. STDOUT
-  # and STDERR are routed to *.output and *.error files with the names of the
-  # executable and the process ID in the name.  The Condor log file is written
-  # to the system's temporary directory.
+  # This function runs the specified executable and arguments as a Condor job.
+  # It creates a condor_submit.nnn file in the current directory.
   def Condor.submit(executable, command_line_arguments, options = {})
     if executable.nil? or executable.empty?
       raise "You must specify a command to run"
     end
-    # Condor needs the full path to the command.
-    # BUGBUG This is UNIX specific.
-    executable = `which #{executable} 2> /dev/null`.strip
-    not executable.empty? or raise "#{executable} is not executable."
+    # Condor needs a full path to a executable program.  Raise an error if
+    # this cannot be determined.
+    executable.strip!
+    full_exec_paths = File.files_on_path(executable)
+    raise NoExecutableException.new(executable) if full_exec_paths.empty?
+    executables = full_exec_paths.select{|f| File.executable?(f)}
+    if executables.empty?
+      raise ExecutePermissionException.new(full_exec_paths.join(","))
+    end
+    executable = executables.first
     # Create a unique submit file name of the form condor_submit.nnn.
     n = 0
     while n < 1000
@@ -132,12 +161,35 @@ Queue
     else
       [command_line_arguments]
     end
+    # Optionally read configuration from a file.
+    config = if options.has_key?(:configfile)
+      open(options[:configfile]) {|f| YAML.load(f)}
+    else
+      {}
+    end
     # Write Condor information to the submit file.
     File.open(submit_name, "w") do |file|
-      file << Condor::SubmitFile.new(executable, arguments_list, options)
+      file << Condor::SubmitFile.new(executable, arguments_list, config,
+                                     options[:output], options[:error])
     end
     # Optionally submit the Condor job.
     `condor_submit #{submit_name}` if options[:submit]
-  end
+  end # Condor.submit
 
 end # Condor
+
+# Extensions to the File class.
+class File
+  # Does the specified full filename have a path component?
+  def File.path_specified?(fullname)
+    not basename(fullname) == fullname
+  end
+
+  # Find all instances of the specified file on the path.
+  def File.files_on_path(name, path = ENV["PATH"])
+    return [name] if path_specified?(name)
+    path.split(File::PATH_SEPARATOR).collect do |p|
+      File.join(p, name)
+    end.select {|f| File.exists?(f)}
+  end
+end

data/test/test_condor.rb

@@ -27,9 +27,108 @@
 require "test/unit"
 require "condor"
 
+# A class the simulates the absence of a /etc/ruby-condor.yaml file.
+class SubmitFileWithoutConfig < Condor::SubmitFile
+  def configuration_options
+    {}
+  end
+end
+
+# A class the simulates a /etc/ruby-condor.yaml file.
+class SubmitFileWithConfig < Condor::SubmitFile
+  def configuration_options
+    {"Universe" => "vanilla"}
+  end
+end
 
 class CondorTestCase < Test::Unit::TestCase
-  def test_submit_file
-    s = Condor::SubmitFile.new("ruby", [["arg1", "arg2"]])
+  def test_noconfig_submit_file
+    s = SubmitFileWithoutConfig.new("ruby", [["arg1", "arg2"]])
+    expected = <<-EOTEXT
+executable = ruby
+
+\targuments  = arg1 arg2
+\toutput     = ruby.$(Cluster).$(Process).output
+\terror      = ruby.$(Cluster).$(Process).error
+\tQueue
+    EOTEXT
+    assert_equal expected, s.to_s
   end
+
+  def test_config_submit_file
+    s = SubmitFileWithConfig.new("ruby", [["arg1", "arg2"]])
+    expected = <<-EOTEXT
+Universe = vanilla
+executable = ruby
+
+\targuments  = arg1 arg2
+\toutput     = ruby.$(Cluster).$(Process).output
+\terror      = ruby.$(Cluster).$(Process).error
+\tQueue
+    EOTEXT
+    assert_equal expected, s.to_s
+  end
+
+  def test_options
+    s = SubmitFileWithoutConfig.new("ruby", [["arg1", "arg2"]],
+                                    {"Universe" => "vanilla"})
+    expected = <<-EOTEXT
+Universe = vanilla
+executable = ruby
+
+\targuments  = arg1 arg2
+\toutput     = ruby.$(Cluster).$(Process).output
+\terror      = ruby.$(Cluster).$(Process).error
+\tQueue
+    EOTEXT
+    assert_equal expected, s.to_s    
+  end
+
+  def test_input_output
+    s = SubmitFileWithoutConfig.new("ruby", [["arg1", "arg2"]], {},
+                                    "output", "error")
+    expected = <<-EOTEXT
+executable = ruby
+
+\targuments  = arg1 arg2
+\toutput     = output
+\terror      = error
+\tQueue
+    EOTEXT
+    assert_equal expected, s.to_s    
+  end
+
+  def test_multiple_arguments
+    s = SubmitFileWithoutConfig.new("ruby", [["arg1", "arg2"],
+                                             ["arg3", "arg4"]])
+    expected = <<-EOTEXT
+executable = ruby
+
+\targuments  = arg1 arg2
+\toutput     = ruby.$(Cluster).$(Process).output
+\terror      = ruby.$(Cluster).$(Process).error
+\tQueue
+
+\targuments  = arg3 arg4
+\toutput     = ruby.$(Cluster).$(Process).output
+\terror      = ruby.$(Cluster).$(Process).error
+\tQueue
+    EOTEXT
+    assert_equal expected, s.to_s
+  end
+
+  def test_overwrite_config
+    s = SubmitFileWithConfig.new("ruby", [["arg1", "arg2"]],
+                                 {"Universe" => nil})
+    expected = <<-EOTEXT
+executable = ruby
+
+\targuments  = arg1 arg2
+\toutput     = ruby.$(Cluster).$(Process).output
+\terror      = ruby.$(Cluster).$(Process).error
+\tQueue
+    EOTEXT
+    assert_equal expected, s.to_s
+  end
+
 end # CondorTestCase

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.2
 specification_version: 1
 name: condor
 version: !ruby/object:Gem::Version 
-  version: 1.1.0
-date: 2009-02-12 00:00:00 -08:00
+  version: 2.0.0
+date: 2009-03-03 00:00:00 -08:00
 summary: Ruby wrapper for the Condor distributed computing system
 require_paths: 
 - lib