workflow_manager 0.0.6 → 0.0.7

data/bin/wfm_job_list

@@ -1,10 +1,19 @@
 #!/usr/bin/env ruby
 # encoding: utf-8
 # 20121112 masa workflow manager client
-Version = '20131104-192211'
+Version = '20131108-112017'
 
 require 'drb/drb' 
 require 'workflow_manager/optparse_ex'
+require 'pstore'
+class PStore
+  def each
+    self.roots.each do |key|
+      yield(key, self[key])
+    end
+  end
+end
+
 #require File.join((File.expand_path('../../lib',__FILE__)), 'optparse_ex.rb')
 
 opt = OptionParser.new do |o|

data/bin/workflow_manager

@@ -24,7 +24,6 @@ if opt.mode
   unless File.exist?(config_file)
     app_dir = File.expand_path('..', __FILE__)
     default_config_dir = File.join(app_dir, "../config/environments")
-    p default_config_dir
     default_config_file = File.join(default_config_dir, opt.mode+".rb")
     if File.exist?(default_config_file)
       FileUtils.cp(default_config_file, config_file)

data/lib/workflow_manager/server.rb

@@ -3,17 +3,28 @@
 
 require 'drb/drb' 
 require 'fileutils'
-require 'kyotocabinet'
+begin
+  require 'kyotocabinet'
+  NO_KYOTO = false
+rescue LoadError
+  require 'pstore'
+  class PStore
+    def each
+      self.roots.each do |key|
+        yield(key, self[key])
+      end
+    end
+  end
+  NO_KYOTO = true
+end
 
 module WorkflowManager
   # default parameters
-  #LOG_DIR  = '/srv/GT/analysis/workflow_manager_logs'
   LOG_DIR  = 'logs'
   DB_DIR   = 'dbs'
   INTERVAL = 30
   RESUBMIT = 0
 
-  #class WorkflowManager
   class Server
     @@config = nil
     class Config
@@ -45,26 +56,47 @@ module WorkflowManager
       end
       @@config
     end
-#  end
 
-  #class WorkflowManager
-#  class Server
+    class KyotoDB
+      def initialize(db_file)
+        @file = db_file
+        @db = KyotoCabinet::DB.new
+      end
+      def transaction
+        @db.open(@file)
+        yield(@db)
+        @db.close
+      end
+    end
+    class PStoreDB 
+      def initialize(db_file)
+        @db = PStore.new(db_file)
+      end
+      def transaction
+        @db.transaction do 
+          yield(@db)
+        end
+      end
+    end
+
     def initialize
-          @interval = config.interval
-          @resubmit = config.resubmit
+      @interval = config.interval
+      @resubmit = config.resubmit
       @db_stat = File.join(config.db_dir, 'statuses.kch')
       @db_logs  = File.join(config.db_dir, 'logs.kch')
 
-          @log_dir = File.expand_path(config.log_dir)
+      @log_dir = File.expand_path(config.log_dir)
       @db_dir  = File.expand_path(config.db_dir)
-          FileUtils.mkdir_p @log_dir unless File.exist?(@log_dir)
-          FileUtils.mkdir_p @db_dir unless File.exist?(@db_dir)
-      @statuses = KyotoCabinet::DB.new
-          @logs = KyotoCabinet::DB.new 
-          @system_log = File.join(@log_dir, "system.log")
-          @mutex = Mutex.new
+      FileUtils.mkdir_p @log_dir unless File.exist?(@log_dir)
+      FileUtils.mkdir_p @db_dir unless File.exist?(@db_dir)
+      #@statuses = KyotoCabinet::DB.new
+      @statuses = NO_KYOTO ? PStoreDB.new(@db_stat) : KyotoDB.new(@db_stat)
+      #@logs = KyotoCabinet::DB.new 
+      @logs = NO_KYOTO ? PStoreDB.new(@db_logs) : KyotoDB.new(@db_logs)
+      @system_log = File.join(@log_dir, "system.log")
+      @mutex = Mutex.new
       @cluster = config.cluster
-          log_puts("Server starts")
+      log_puts("Server starts")
     end
     def hello
       'hello, '+ @cluster.name
@@ -73,163 +105,175 @@ module WorkflowManager
       @cluster.copy_commands(org_dir, dest_parent_dir)
     end
     def log_puts(str)
-        time = Time.now.strftime("[%Y.%m.%d %H:%M:%S]")
-        @mutex.synchronize do
-            open(@system_log, "a") do |out|
-                out.print time + " " + str + "\n"
-            end
-        end
+      time = Time.now.strftime("[%Y.%m.%d %H:%M:%S]")
+      @mutex.synchronize do
+         open(@system_log, "a") do |out|
+           out.print time + " " + str + "\n"
+         end
+      end
     end
     def start_monitoring(submit_command, user = 'sushi lover', resubmit = 0, script = '', project_number = 0, sge_options='', log_dir = '')
-        log_puts("monitoring: script=" + submit_command + " user=" + user + " resubmit=" + resubmit.to_s + " project=" + project_number.to_s + " sge option=" + sge_options + " log dir=" + log_dir.to_s)
+      log_puts("monitoring: script=" + submit_command + " user=" + user + " resubmit=" + resubmit.to_s + " project=" + project_number.to_s + " sge option=" + sge_options + " log dir=" + log_dir.to_s)
 
-        #warn submit_command
-        #
-        # TODO: analyze arguments
-        #
-        job_id, log_file, command = @cluster.submit_job(submit_command, script, sge_options)
-    log_puts("submit: " + job_id + " " + command)
+      #warn submit_command
+      #
+      # TODO: analyze arguments
+      #
+      job_id, log_file, command = @cluster.submit_job(submit_command, script, sge_options)
+      log_puts("submit: " + job_id + " " + command)
 
-        #
-        # monitor worker
-        #
-        if job_id and log_file
-            monitor_worker = Thread.new(job_id, log_file, submit_command, user, resubmit, script, project_number, sge_options, log_dir) do |t_job_id, t_log_file, t_submit_command, t_user, t_resubmit, t_script, t_project_number, t_sge_options, t_log_dir|
-                loop do
-                    status = success_or_fail(t_job_id, t_log_file) 
-          script_name = File.basename(submit_command).split(/-/).first
-          @statuses.open(@db_stat)
-          start_time = if stat = @statuses[t_job_id] and stat = stat.split(/,/) and time = stat[2]
-                         time
+      #
+      # monitor worker
+      #
+      if job_id and log_file
+        monitor_worker = Thread.new(job_id, log_file, submit_command, user, resubmit, script, project_number, sge_options, log_dir) do |t_job_id, t_log_file, t_submit_command, t_user, t_resubmit, t_script, t_project_number, t_sge_options, t_log_dir|
+          loop do
+            status = success_or_fail(t_job_id, t_log_file) 
+            script_name = File.basename(submit_command).split(/-/).first
+            #@statuses.open(@db_stat)
+            @statuses.transaction do |statuses|
+            #start_time = if stat = @statuses[t_job_id] and stat = stat.split(/,/) and time = stat[2]
+              start_time = if stat = statuses[t_job_id] and stat = stat.split(/,/) and time = stat[2]
+                             time
+                           end
+              time = if start_time 
+                       if status == 'success' or status == 'fail'
+                         start_time + '/' + Time.now.strftime("%Y-%m-%d %H:%M:%S")
+                       else
+                         start_time
                        end
-          time = if start_time 
-                   if status == 'success' or status == 'fail'
-                     start_time + '/' + Time.now.strftime("%Y-%m-%d %H:%M:%S")
-                   else
-                     start_time
-                   end
-                 else
-                   Time.now.strftime("%Y-%m-%d %H:%M:%S")
-                 end
-                @statuses[t_job_id] = [status, script_name, time, user, project_number].join(',')
-          @statuses.close
-          @logs.open(@db_logs)
-                    @logs[t_job_id] = t_log_file
-          @logs.close
-                    #warn t_job_id + " " + status
-                    if status == 'success'
-                        log_puts(status + ": " + t_job_id)
-            unless t_log_dir.empty?
-              copy_commands(t_log_file, t_log_dir).each do |command|
-                log_puts(command)
-                system command
-              end
-              err_file = t_log_file.gsub('_o.log','_e.log')
-              copy_commands(err_file, t_log_dir).each do |command|
-                log_puts(command)
-                system command
-              end
+                     else
+                       Time.now.strftime("%Y-%m-%d %H:%M:%S")
+                     end
+            #@statuses[t_job_id] = [status, script_name, time, user, project_number].join(',')
+              statuses[t_job_id] = [status, script_name, time, user, project_number].join(',')
+            #@statuses.close
+            end
+            @logs.transaction do |logs|
+              logs[t_job_id] = t_log_file
             end
-                        Thread.current.kill
-                    elsif status == 'fail'
-                        log_puts(status + ": " + t_job_id)
-                        #
-                        # TODO: re-submit
-                        #
-                        if t_resubmit < RESUBMIT
-                            log_puts("resubmit: " + t_job_id)
-                            resubmit_job_id = start_monitoring(t_submit_command, t_user, t_resubmit + 1, t_script, t_project_number, t_sge_options)
-              script_name = File.basename(submit_command).split(/-/).first
-              @statuses.open(@db_stat)
-                            @statuses[t_job_id] = ["resubmit: " + resubmit_job_id.to_s, script_name, Time.now.strftime("%Y-%m-%d %H:%M:%S"), t_user, t_project_number].join(',')
-              @statuses.close
-                        else
-                            log_puts("fail: " + t_job_id)
-                        end
-            unless t_log_dir.empty?
-              copy_commands(t_log_file, t_log_dir).each do |command|
-                log_puts(command)
-                system command
+            #warn t_job_id + " " + status
+            if status == 'success'
+              log_puts(status + ": " + t_job_id)
+              unless t_log_dir.empty?
+                copy_commands(t_log_file, t_log_dir).each do |command|
+                  log_puts(command)
+                  system command
+                end
+                err_file = t_log_file.gsub('_o.log','_e.log')
+                copy_commands(err_file, t_log_dir).each do |command|
+                  log_puts(command)
+                  system command
+                end
               end
-              err_file = t_log_file.gsub('_o.log','_e.log')
-              copy_commands(err_file, t_log_dir).each do |command|
-                log_puts(command)
-                system command
+              Thread.current.kill
+            elsif status == 'fail'
+              log_puts(status + ": " + t_job_id)
+              #
+              # TODO: re-submit
+              #
+              if t_resubmit < RESUBMIT
+                log_puts("resubmit: " + t_job_id)
+                resubmit_job_id = start_monitoring(t_submit_command, t_user, t_resubmit + 1, t_script, t_project_number, t_sge_options)
+                script_name = File.basename(submit_command).split(/-/).first
+                #@statuses.open(@db_stat)
+                @statuses.transaction do |statuses|
+                  statuses[t_job_id] = ["resubmit: " + resubmit_job_id.to_s, script_name, Time.now.strftime("%Y-%m-%d %H:%M:%S"), t_user, t_project_number].join(',')
+                #@statuses.close
+                end
+              else
+                log_puts("fail: " + t_job_id)
               end
+              unless t_log_dir.empty?
+                copy_commands(t_log_file, t_log_dir).each do |command|
+                  log_puts(command)
+                  system command
+                end
+                err_file = t_log_file.gsub('_o.log','_e.log')
+                copy_commands(err_file, t_log_dir).each do |command|
+                  log_puts(command)
+                  system command
+                end
+              end
+              Thread.current.kill
             end
-                        Thread.current.kill
-                    end
-                    sleep @interval
-                end	
-            end
-            job_id.to_i
+            sleep @interval
+          end	
         end
+        job_id.to_i
+      end
     end
     def status(job_id)
-    stat = nil
-    @statuses.open(@db_stat)
-        stat = @statuses[job_id.to_s]
-    @statuses.close
-    stat
+      stat = nil
+      #@statuses.open(@db_stat)
+      @statuses.transaction do |statuses|
+        stat = statuses[job_id.to_s]
+      #@statuses.close
+      end
+      stat
     end
     def job_list(with_results=false, project_number=nil)
       s = []
-      @statuses.open(@db_stat)
-      @statuses.each do |key, value|
-        if project_number 
-          if x = value.split(/,/)[4].to_i==project_number.to_i
+      #@statuses.open(@db_stat)
+      @statuses.transaction do |statuses|
+        statuses.each do |key, value|
+          if project_number 
+            if x = value.split(/,/)[4].to_i==project_number.to_i
+              s << [key, value]
+            end
+          else
             s << [key, value]
           end
-        else
-          s << [key, value]
         end
+      #@statuses.close
       end
-      @statuses.close
       s.sort.reverse.map{|v| v.join(',')}.join("\n")
     end
     def get_log(job_id, with_err=false)
-    @logs.open(@db_logs)
-        log_file = @logs[job_id.to_s]
-    @logs.close
-        log_data = if log_file and File.exist?(log_file)
-            "__STDOUT LOG__\n\n" + File.read(log_file)
-        else
-            'no log file'
+      log_file = nil
+      @logs.transaction do |logs|
+        log_file = logs[job_id.to_s]
+      end
+      log_data = if log_file and File.exist?(log_file)
+                   "__STDOUT LOG__\n\n" + File.read(log_file)
+                 else
+                   'no log file'
+                 end
+      if with_err
+        err_file = log_file.gsub(/_o\.log/,'_e.log')
+        if err_file and File.exist?(err_file)
+          log_data << "\n\n__STDERR LOG__\n\n"
+          log_data << File.read(err_file)
         end
-    if with_err
-      err_file = log_file.gsub(/_o\.log/,'_e.log')
-     if err_file and File.exist?(err_file)
-        log_data << "\n\n__STDERR LOG__\n\n"
-        log_data << File.read(err_file)
       end
-    end
-        log_data
+      log_data
     end
     def get_script(job_id)
-    @logs.open(@db_logs)
-        script_file = @logs[job_id.to_s]
-    @logs.close
-    if script_file
-      script_file = script_file.gsub(/_o\.log/,'')
-    end
-        script = if script_file and File.exist?(script_file)
-            File.read(script_file)
-        else
-            'no script file'
-        end
-        script
+      script_file = nil
+      @logs.transaction do |logs|
+        script_file = logs[job_id.to_s]
+      end
+      if script_file
+        script_file = script_file.gsub(/_o\.log/,'')
+      end
+      script = if script_file and File.exist?(script_file)
+                 File.read(script_file)
+               else
+                 'no script file'
+               end
+      script
     end
     def success_or_fail(job_id, log_file)
-    job_running = @cluster.job_running?(job_id)
-    job_ends = @cluster.job_ends?(log_file)
-        msg = if job_running
-            'running'
-        elsif job_ends
-            'success'
-        else
-            'fail'
-        end
-        msg
+      job_running = @cluster.job_running?(job_id)
+      job_ends = @cluster.job_ends?(log_file)
+      msg = if job_running
+              'running'
+            elsif job_ends
+              'success'
+            else
+              'fail'
+            end
+      msg
     end
   end
 end

data/lib/workflow_manager/version.rb

@@ -1,3 +1,3 @@
 module WorkflowManager
-  VERSION = "0.0.6"
+  VERSION = "0.0.7"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: workflow_manager
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
   prerelease: 
 platform: ruby
 authors: