workflow_manager 0.6.0 → 0.7.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fe50e7ccd102e4cd531cf2afb47e1222ca0b06e434ab34fdfff6a0f3a0fd35c3
-  data.tar.gz: fd07c0b863627a1a8065e7f710931ba99c0609e5ef9e9e82f107c35eee678b68
+  metadata.gz: 49b41b8057a7487ed992b96a9f0b81a155bef54d2fe9b63cde6c2503a84d2257
+  data.tar.gz: 373b56284df027f151a0a77f7d28fc0cd8331b51f9e2ef22e4484b0db8732154
 SHA512:
-  metadata.gz: 6a9701699b79fd61f889b86dbc3060fe63f6d5e8b6e2532dabb0635b5c3f37f335563dfeda5ba677328ff88e1c2197df60c84e9c9526b4a316eac751b9e3b939
-  data.tar.gz: d42a9d888a80e7f8b549a64595d13e3c268499d4f78d00805c070c6d60feb9febd20f335e7e8af398f910f6e1eade7b60cf200e83d99451230a4e0f37e134e0f
+  metadata.gz: df2c18b28e56ee900c2845b0ae43ee826a59fbab3b39ec6f93b5f62880b3bf2fcbc7ab4f3804899a6234e0ae5f9f7abebacd77fd24900515da31cf4d6b1215e9
+  data.tar.gz: e6ca40a3e1428284c73f578e09be5028e55bd630ce673699288d8ce3b76ccf107e5e265eba3a731401e38b7b1e01fcb5fa5e6152aeec642333cc35f681131074

data/.gitignore

@@ -15,3 +15,4 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
+command.log

data/Gemfile

@@ -1,4 +1,6 @@
 source 'https://rubygems.org'
 
-# Specify your gem's dependencies in workflow_manager.gemspec
-gemspec
+gem 'redis'
+gem 'sidekiq'
+#gem 'workflow_manager', :path => '/srv/GT/analysis/masaomi/FGCZ/prototype_workflow_manager_with_sidekiq_20210122/workflow_manager/'
+gem 'workflow_manager'

data/bin/wfm_monitoring

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 # encoding: utf-8
 # 20121112 masa workflow manager client
-Version = '20200722-161135'
+Version = '20210625-165025'
 
 require 'drb/drb' 
 require 'workflow_manager/optparse_ex'
@@ -54,4 +54,5 @@ sge_options << "-n #{opt.nodes}" if opt.nodes
 script_content = File.read(script_file)
 workflow_manager = DRbObject.new_with_uri(uri)
 #puts workflow_manager.start_monitoring(script_file, user, 0, script_content, project_number, sge_options.join(' '), opt.log)
-puts workflow_manager.start_monitoring2(script_file, script_content, user, project_number, sge_options.join(' '), opt.log)
+#puts workflow_manager.start_monitoring2(script_file, script_content, user, project_number, sge_options.join(' '), opt.log)
+puts workflow_manager.start_monitoring3(script_file, script_content, user, project_number, sge_options.join(' '), opt.log)

data/bin/workflow_manager

@@ -8,7 +8,7 @@ Version = WorkflowManager::VERSION
 opt = OptionParser.new do |o|
   o.banner = "Version: #{Version}\nUsage:\n #{File.basename(__FILE__)} -d [druby://host:port] -m [development|production]"
   o.on(:server, 'druby://localhost:12345', '-d server', '--server', 'workflow manager URI (default: druby://localhost:12345)')
-  o.on(:mode, 'development', '-m mode', '--mode', 'development|production (default: development)')
+  o.on(:mode, 'production', '-m mode', '--mode', 'development|production (default: production)')
   o.parse!(ARGV)
 end
 
@@ -33,7 +33,9 @@ if opt.mode
     default_config_file = File.join(default_config_dir, opt.mode+".rb")
     if File.exist?(default_config_file)
       default_redis_config_file = File.join(default_config_dir, "redis.conf")
+      default_sidekiq_config_file = File.join(default_config_dir, "sidekiq.yml")
       FileUtils.cp(default_redis_config_file, config_dir)
+      FileUtils.cp(default_sidekiq_config_file, config_dir)
       FileUtils.cp(default_config_file, config_file)
     else
       raise "Configure file does not exist: #{config_file}"
@@ -43,4 +45,22 @@ if opt.mode
 end
 DRb.start_service(uri, WorkflowManager::Server.new) 
 puts DRb.uri
-DRb.thread.join
+#DRb.thread.join
+
+sleep 1
+
+sidekiq_pid = fork do
+  app_dir = File.expand_path('..', __FILE__)
+  job_checker = File.join(app_dir, "../lib/job_checker.rb")
+  exec("sidekiq -C config/environments/sidekiq.yml -r #{job_checker}")
+end
+
+begin
+  DRb.thread.join
+  puts "__END__"
+rescue SignalException
+  Process.kill("HUP", sidekiq_pid)
+  sleep 1
+  puts "__CORRECTLY_END__"
+end
+

data/config/environments/production.rb

@@ -7,6 +7,6 @@ WorkflowManager::Server.configure do |config|
   config.interval = 30
   config.resubmit = 0
   config.redis_conf = "config/environments/redis.conf"
-  config.cluster = WorkflowManager::FGCZCluster.new('FGCZCluster')
+  config.cluster = WorkflowManager::FGCZDebian10Cluster.new('FGCZDebian10Cluster')
 end
 

data/config/environments/redis.conf

@@ -1,9 +1,9 @@
 loglevel debug
 logfile ./logs/redis.log
-databases 2
+databases 4
 save 300 10
 rdbcompression yes
 dir ./dbs
 dbfilename redis.rdb
 maxmemory 10gb
-port 6379
+port 6380

data/config/environments/sidekiq.yml

@@ -0,0 +1,8 @@
+:verbose: false
+:pidfile: ./tmp/pids/sidekiq.pid
+:logfile: ./log/sidekiq.log
+:concurrency: 100
+:queues:
+  - default
+  - test
+

data/lib/job_checker.rb

@@ -0,0 +1,81 @@
+require 'sidekiq'
+require 'redis'
+
+WORKER_INTERVAL = 10 # [s]
+REDIS_CONF = File.expand_path("../../config/environments/redis.conf", __FILE__)
+PORT = if File.exist?(REDIS_CONF)
+         redis_conf = Hash[*File.readlines(REDIS_CONF).map{|line| line.chomp.split}.map{|e| [e[0], e[1,100].join(",")]}.flatten]
+         redis_conf["port"].to_i
+       else
+         6379
+       end
+SIDEKIQ_URL = "redis://localhost:#{PORT}/3"
+warn "redis.conf: #{REDIS_CONF}"
+warn "Redis port: #{PORT}"
+warn "Sidekiq URL: #{SIDEKIQ_URL}"
+
+Sidekiq.configure_server do |config|
+  config.redis = { url: SIDEKIQ_URL }
+end
+
+Sidekiq.configure_client do |config|
+  config.redis = { url: SIDEKIQ_URL }
+end
+
+class Redis
+  alias_method :[], :get
+  alias_method :[]=, :set
+end
+
+class JobChecker
+  include Sidekiq::Worker
+  sidekiq_options queue: :default, retry: 5
+
+  def generate_new_job_script(log_dir, script_basename, script_content)
+    new_job_script = File.basename(script_basename) + "_" + Time.now.strftime("%Y%m%d%H%M%S%L")
+    new_job_script = File.join(log_dir, new_job_script)
+    open(new_job_script, 'w') do |out|
+      out.print script_content
+      out.print "\necho __SCRIPT END__\n"
+    end
+    new_job_script
+  end
+  def update_time_status(status, script_basename, user, project_number)
+    unless @start_time
+      @start_time = Time.now.strftime("%Y-%m-%d %H:%M:%S")
+    end
+    time = Time.now.strftime("%Y-%m-%d %H:%M:%S")
+    [status, script_basename, [@start_time, time].join("/"), user, project_number].join(',')
+  end
+
+  def perform(job_id, script_basename, log_file, user, project_id)
+    puts "JobID (in JobChecker): #{job_id}"
+    db0 = Redis.new(port: PORT, db: 0) # state + alpha DB
+    db1 = Redis.new(port: PORT, db: 1) # log DB
+    db2 = Redis.new(port: PORT, db: 2) # project jobs DB
+    db1[job_id] = log_file
+    pre_state = nil
+    @start_time = nil
+    begin
+      command = "sacct --jobs=#{job_id} --format=state"
+      #puts command
+      ret = `#{command}`
+      #print ret
+      state = ret.split(/\n/).last.strip
+      #puts "state: #{state}"
+      db0[job_id] = update_time_status(state, script_basename, user, project_id)
+
+      unless state == pre_state
+        db0[job_id] = update_time_status(state, script_basename, user, project_id)
+        project_jobs = eval((db2[project_id]||[]).to_s)
+        project_jobs = Hash[*project_jobs]
+        project_jobs[job_id] = state
+        #p project_jobs
+        db2[project_id] = project_jobs.to_a.flatten.last(200).to_s
+      end
+      pre_state = state
+      sleep WORKER_INTERVAL
+    end while state =~ /RUNNING/  or state =~ /PENDING/ or state =~ /---/
+  end
+end
+

data/lib/worker4.rb

@@ -0,0 +1,80 @@
+require 'sidekiq'
+require 'redis'
+
+WORKER_INTERVAL = 10 # [s]
+
+Sidekiq.configure_server do |config|
+  config.redis = { url: 'redis://localhost:6380/3' }
+end
+
+Sidekiq.configure_client do |config|
+  config.redis = { url: 'redis://localhost:6380/3' }
+end
+
+class Redis
+  alias_method :[], :get
+  alias_method :[]=, :set
+end
+
+class JobWorker
+  include Sidekiq::Worker
+  sidekiq_options queue: :default, retry: 5
+
+  def generate_new_job_script(log_dir, script_basename, script_content)
+    new_job_script = File.basename(script_basename) + "_" + Time.now.strftime("%Y%m%d%H%M%S%L")
+    new_job_script = File.join(log_dir, new_job_script)
+    open(new_job_script, 'w') do |out|
+      out.print script_content
+      out.print "\necho __SCRIPT END__\n"
+    end
+    new_job_script
+  end
+  def update_time_status(status, script_basename, user, project_number)
+    unless @start_time
+      @start_time = Time.now.strftime("%Y-%m-%d %H:%M:%S")
+    end
+    time = Time.now.strftime("%Y-%m-%d %H:%M:%S")
+    [status, script_basename, [@start_time, time].join("/"), user, project_number].join(',')
+  end
+
+  def perform(project_id, log_dir, script_basename, script_content)
+    #script_base_name = "test_job.sh"
+    job_script = generate_new_job_script(log_dir, script_basename, script_content)
+    log_file = job_script + "_o.log"
+    err_file = job_script + "_e.log"
+    command = "sbatch -o #{log_file} -e #{err_file} -N 1 #{job_script}"
+    puts command
+    ret = `#{command}`
+    job_id = ret.chomp.split.last
+    puts "JobID: #{job_id}"
+    db0 = Redis.new(port: 6380, db: 0) # state + alpha DB
+    db1 = Redis.new(port: 6380, db: 1) # log DB
+    db2 = Redis.new(port: 6380, db: 2) # project jobs DB
+    db1[job_id] = log_file
+    pre_state = nil
+    @start_time = nil
+    begin
+      command = "sacct --jobs=#{job_id} --format=state"
+      puts command
+      ret = `#{command}`
+      #print ret
+      state = ret.split(/\n/).last.strip
+      puts "state: #{state}"
+      #db.set(job_id, state)
+      db0[job_id] = update_time_status(state, script_basename, "sushi_lover", project_id)
+
+      unless state == pre_state
+        db0[job_id] = update_time_status(state, script_basename, "sushi_lover", project_id)
+        project_jobs = eval((db2[project_id]||[]).to_s)
+        project_jobs = Hash[*project_jobs]
+        project_jobs[job_id] = state
+        p project_jobs
+        db2[project_id] = project_jobs.to_a.flatten.last(200).to_s
+        #db2[project_id] = project_jobs.to_s
+      end
+      pre_state = state
+      sleep WORKER_INTERVAL
+    end while state =~ /RUNNING/  or state =~ /PENDING/ or state =~ /---/
+  end
+end
+

data/lib/workflow_manager/server.rb

@@ -4,6 +4,9 @@
 require 'drb/drb' 
 require 'fileutils'
 require 'csv'
+
+require 'job_checker'
+
 begin
   require 'redis'
   DB_MODE = "Redis"
@@ -159,6 +162,7 @@ module WorkflowManager
                   when "Redis"
                     RedisDB.new(1, @redis_conf)
                 end
+      @jobs = RedisDB.new(2, @redis_conf)
 
       @system_log = File.join(@log_dir, "system.log")
       @mutex = Mutex.new
@@ -172,15 +176,35 @@ module WorkflowManager
       log_puts("DB = #{DB_MODE}")
       log_puts("Cluster = #{@cluster.name}")
       log_puts("Server starts")
+      log_puts("Recovery check")
+      sleep 2
+      recovery_job_checker
+    end
+    def recovery_job_checker
+      @logs.transaction do |logs|
+      @statuses.transaction do |statuses|
+        statuses.each do |job_id, status|
+          # puts [job_id, status].join(",")
+          # 120249,RUNNING,QC_ventricles_100k.sh,2021-07-30 09:47:04/2021-07-30 09:47:04,masaomi,1535
+          stat, script_basename, time, user, project_number = status.split(",")
+          if stat == "RUNNING" or stat == "PENDING"
+            log_file = logs[job_id]
+            log_puts("JobID (in recovery check): #{job_id}")
+            puts "JobID (in recovery check): #{job_id}"
+            JobChecker.perform_async(job_id, script_basename, log_file, user, project_number)
+          end
+        end
+      end
+      end
     end
     def hello
-      'hello test hoge, '+ @cluster.name
+      'hello hoge hoge bar boo bundle, '+ @cluster.name
     end
     def copy_commands(org_dir, dest_parent_dir, now=nil)
       @cluster.copy_commands(org_dir, dest_parent_dir, now)
     end
     def kill_job(job_id)
-      status(job_id, 'fail')
+      status(job_id, 'FAIL')
       status = `#{@cluster.kill_command(job_id)}`
     end
     def delete_command(target)
@@ -272,6 +296,17 @@ module WorkflowManager
         Thread.current.kill
       end
     end
+    def start_monitoring3(script_path, script_content, user='sushi_lover', project_number=0, sge_options='', log_dir='')
+      script_basename = File.basename(script_path)
+      job_id, log_file, command = @cluster.submit_job(script_path, script_content, sge_options)
+      #p command
+      #p log_file
+      #p job_id
+      puts "JobID (in WorkflowManager): #{job_id}"
+      sleep 1
+      JobChecker.perform_async(job_id, script_basename, log_file, user, project_number)
+      job_id
+    end
     def start_monitoring2(script_path, script_content, user='sushi_lover', project_number=0, sge_options='', log_dir='')
       # script_path is only used to generate a log file name
       # It is not used to read the script contents
@@ -418,7 +453,7 @@ module WorkflowManager
       #@statuses.open(@db_stat)
       @statuses.transaction do |statuses|
         if new_status and stat = statuses[job_id.to_s]
-          status_list = ['success', 'running', 'pending', 'fail']
+          status_list = ['CONPLETED', 'RUNNING', 'PENDING', 'FAIL']
           if status_list.include?(new_status)
             items = stat.split(/,/)
             items.shift
@@ -438,17 +473,19 @@ module WorkflowManager
       job_idsh = if job_ids
                    Hash[*(job_ids.split(',')).map{|job_id| [job_id, true]}.flatten]
                  end
-      @statuses.transaction do |statuses|
-        statuses.each do |key, value|
-          if project_number 
-            if x = value.split(/,/)[4].to_i==project_number.to_i
-              s << [key, value]
-            end
-          else
-            s << [key, value]
+      s_ = {}
+      unless job_ids
+        @jobs.transaction do |jobs|
+          if project_jobs = jobs[project_number]
+            s_ = Hash[*eval(project_jobs)]
           end
         end
       end
+      @statuses.transaction do |statuses|
+        s_.each do |job_id, stat|
+          s << [job_id, statuses[job_id]]
+        end
+      end
       if job_ids
         s = s.select{|job_id, stat| job_idsh[job_id]}
       end

data/lib/workflow_manager/version.rb

@@ -1,3 +1,3 @@
 module WorkflowManager
-  VERSION = "0.6.0"
+  VERSION = "0.7.3"
 end

data/test/call_worker4.rb

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+# Version = '20210625-104318'
+
+require './lib/worker4'
+script_file = "./test/test_job1.sh"
+script_content = File.read(script_file)
+log_dir = "./logs"
+script_basename = File.basename(script_file)
+JobWorker.perform_async(1001, log_dir, script_basename, script_content)
+p "submitted test_job1.sh"

data/test/call_worker_method.rb

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+# Version = '20210625-162836'
+
+require 'workflow_manager'
+script_file = "./test/test_job1.sh"
+script_content = File.read(script_file)
+log_dir = "./logs"
+script_basename = File.basename(script_file)
+JobWorker.perform_async(1001, log_dir, script_basename, script_content)
+p "submitted test_job1.sh"

data/test/job_list.rb

@@ -0,0 +1,50 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+# Version = '20210723-134812'
+
+PORT = (ARGV[0]||6380).to_i
+require 'redis'
+db0 = Redis.new(port: PORT, db: 0)
+db1 = Redis.new(port: PORT, db: 1)
+db2 = Redis.new(port: PORT, db: 2)
+#db3 = Redis.new(port: 6380, db: 3)
+
+class Redis
+  def show_all
+    self.keys.sort.each do |key|
+      value = self.get(key)
+      puts [key, value].join("\t")
+    end
+  end
+end
+
+dbs = [db0, db1, db2]
+db_notes = ["state DB", "log DB", "project job DB"]
+
+dbs.each.with_index do |db, i|
+  note = db_notes[i]
+  puts ["db#{i}", note].join("\t")
+  db.show_all
+  puts
+end
+exit
+puts "db0, status DB"
+puts ["JobID", "Status"].join("\t")
+db0.keys.sort.each do |key|
+  value = db0.get(key)
+  puts [key, value].join("\t")
+end
+
+puts
+puts "db1, log DB"
+db1.keys.sort.each do |key|
+  value = db1.get(key)
+  puts [key, value].join("\t")
+end
+
+puts
+puts "db2, status DB2, project specific"
+db2.keys.sort.each do |key|
+  value = db2.get(key)
+  puts [key, value].join("\t")
+end

data/test/test_job1.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+echo "Job1"
+sleep 60
+echo "END"

data/workflow_manager.gemspec

@@ -19,6 +19,6 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "bundler", "~> 2.2.10"
   spec.add_development_dependency "rake"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: workflow_manager
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.3
 platform: ruby
 authors:
 - Functional Genomics Center Zurich
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-12-18 00:00:00.000000000 Z
+date: 2021-07-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: 2.2.10
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: 2.2.10
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -76,6 +76,9 @@ files:
 - config/environments/development.rb
 - config/environments/production.rb
 - config/environments/redis.conf
+- config/environments/sidekiq.yml
+- lib/job_checker.rb
+- lib/worker4.rb
 - lib/workflow_manager.rb
 - lib/workflow_manager/cluster.rb
 - lib/workflow_manager/optparse_ex.rb
@@ -84,6 +87,10 @@ files:
 - spec/cluster_spec.rb
 - spec/server_spec.rb
 - spec/spec_helper.rb
+- test/call_worker4.rb
+- test/call_worker_method.rb
+- test/job_list.rb
+- test/test_job1.sh
 - workflow_manager.gemspec
 homepage: ''
 licenses:
@@ -104,7 +111,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.3.1
 signing_key: 
 specification_version: 4
 summary: Workflow Manager manages job submissions using dRuby.
@@ -112,3 +119,7 @@ test_files:
 - spec/cluster_spec.rb
 - spec/server_spec.rb
 - spec/spec_helper.rb
+- test/call_worker4.rb
+- test/call_worker_method.rb
+- test/job_list.rb
+- test/test_job1.sh