workflow_manager 0.4.8 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5ec53a4ae0121d8432e1e350c11108ee64af3ba4
-  data.tar.gz: 8e6f1ac9f288811e62b9039c3a44aa117adebc71
+  metadata.gz: b8d2b5478dda4da2f42ca05754e19e446faa4cab
+  data.tar.gz: f039f6cfbb00dd6c3a8a911b5d417ed24b413809
 SHA512:
-  metadata.gz: a3a9279e745bed4d3e12645eed63940e10f4794395d515a1a3369920b55247778b6d68773dceb3dc5438c17fc84de9af239c0a2119dd42e034da68d3e622226d
-  data.tar.gz: 5e27327fbe2ae06d188b2a8037941f491758fc3f296ee1297579129d35c4d97f69c7bd07be745d7160d53fe697419390787011d7e67c3deb4150ba89586dee6d
+  metadata.gz: a1ab900a8327e7778e715ba909bf583bb2ad621058123523c41a280bbee55d5148591a70641b3aa8d4aa8df7aeccbca477d92673ea2f735a37cedd4696763cbd
+  data.tar.gz: eb034e572721c32a6e9b1858b8ca69899a1fd7705494430e4f9590844dc4b2f0e79dfd0dcf05478d75cdf4f2a324d50b74dec9d8b9b415669b02df121d316260

data/bin/wfm_job_list

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 # encoding: utf-8
 # 20121112 masa workflow manager client
-Version = '20170608-103901'
+Version = '20200324-134935'
 
 require 'drb/drb' 
 require 'workflow_manager/optparse_ex'
@@ -13,6 +13,21 @@ class PStore
     end
   end
 end
+require 'redis'
+class Redis
+  def [](key)
+    self.get(key)
+  end
+  def []=(key, value)
+    self.set(key, value)
+  end
+  def each
+    self.scan_each do |key|
+      value = self.get(key)
+      yield([key, value])
+    end
+  end
+end
 
 #require File.join((File.expand_path('../../lib',__FILE__)), 'optparse_ex.rb')
 

data/bin/workflow_manager

@@ -32,6 +32,8 @@ if opt.mode
     default_config_dir = File.join(app_dir, "../config/environments")
     default_config_file = File.join(default_config_dir, opt.mode+".rb")
     if File.exist?(default_config_file)
+      default_redis_config_file = File.join(default_config_dir, "redis.conf")
+      FileUtils.cp(default_redis_config_file, config_dir)
       FileUtils.cp(default_config_file, config_file)
     else
       raise "Configure file does not exist: #{config_file}"

data/config/environments/production.rb

@@ -6,6 +6,7 @@ WorkflowManager::Server.configure do |config|
   config.db_dir = 'dbs'
   config.interval = 30
   config.resubmit = 0
+  config.redis_conf = "config/environments/redis.conf"
   config.cluster = WorkflowManager::FGCZCluster.new('FGCZCluster')
 end
 

data/config/environments/redis.conf

@@ -0,0 +1,9 @@
+loglevel debug
+logfile ./logs/redis.log
+databases 2
+save 300 10
+rdbcompression yes
+dir ./dbs
+dbfilename redis.rdb
+maxmemory 10gb
+port 6379

data/lib/workflow_manager/server.rb

@@ -6,17 +6,36 @@ require 'fileutils'
 require 'csv'
 begin
   require 'kyotocabinet'
-  NO_KYOTO = false
+  DB_MODE = "KyotoCabinet"
 rescue LoadError
-  require 'pstore'
-  class PStore
-    def each
-      self.roots.each do |key|
-        yield(key, self[key])
+  begin
+    require 'redis'
+    DB_MODE = "Redis"
+    class Redis
+      def [](key)
+        self.get(key)
+      end
+      def []=(key, value)
+        self.set(key, value)
+      end
+      def each
+        self.scan_each do |key|
+          value = self.get(key)
+          yield([key, value])
+        end
+      end
+    end
+  rescue LoadError
+    require 'pstore'
+    DB_MODE = "PStore"
+    class PStore
+      def each
+        self.roots.each do |key|
+          yield(key, self[key])
+        end
       end
     end
   end
-  NO_KYOTO = true
 end
 
 module WorkflowManager
@@ -34,6 +53,7 @@ module WorkflowManager
       attr_accessor :interval
       attr_accessor :resubmit
       attr_accessor :cluster
+      attr_accessor :redis_conf
     end
     def self.config=(config)
       @@config = config
@@ -79,12 +99,40 @@ module WorkflowManager
         end
       end
     end
+    class RedisDB
+      def run_redis_server(redis_conf)
+        @pid = fork do
+          exec("redis-server #{redis_conf}")
+        end
+        @redis_thread = Thread.new do
+          Process.waitpid @pid
+        end
+      end
+      def initialize(db_no=0, redis_conf=nil)
+        if db_no==0
+          run_redis_server(redis_conf)
+        end
+        @db = Redis.new(db: db_no)
+      end
+      def transaction
+        #@db.multi do
+          yield(@db)
+        #end
+      end
+    end
 
     def initialize
       @interval = config.interval
       @resubmit = config.resubmit
-      extension = NO_KYOTO ? '.pstore' : '.kch'
-      db_mode = NO_KYOTO ? 'PStore' : 'KyotoCabinet'
+      extension = case DB_MODE
+                    when "PStore"
+                      '.pstore'
+                    when "KyotoCabinet"
+                      '.kch'
+                    when "Redis"
+                      @redis_conf = config.redis_conf
+                      '.rdb'
+                    end
       @db_stat = File.join(config.db_dir, 'statuses'+extension)
       @db_logs  = File.join(config.db_dir, 'logs'+extension)
 
@@ -92,16 +140,32 @@ module WorkflowManager
       @db_dir  = File.expand_path(config.db_dir)
       FileUtils.mkdir_p @log_dir unless File.exist?(@log_dir)
       FileUtils.mkdir_p @db_dir unless File.exist?(@db_dir)
-      #@statuses = KyotoCabinet::DB.new
-      @statuses = NO_KYOTO ? PStoreDB.new(@db_stat) : KyotoDB.new(@db_stat)
-      #@logs = KyotoCabinet::DB.new 
-      @logs = NO_KYOTO ? PStoreDB.new(@db_logs) : KyotoDB.new(@db_logs)
+      @statuses = case DB_MODE
+                    when "PStore"
+                      PStoreDB.new(@db_stat)
+                    when "KyotoCabinet"
+                      KyotoDB.new(@db_stat)
+                    when "Redis"
+                      RedisDB.new(0, @redis_conf)
+                  end
+      @logs = case DB_MODE
+                  when "PStore"
+                    PStoreDB.new(@db_logs)
+                  when "KyotoCabinet"
+                    KyotoDB.new(@db_logs)
+                  when "Redis"
+                    RedisDB.new(1)
+                end
+
       @system_log = File.join(@log_dir, "system.log")
       @mutex = Mutex.new
       @cluster = config.cluster
-      puts("DB = #{db_mode}")
+      puts("DB = #{DB_MODE}")
+      if DB_MODE == "Redis"
+        puts("Redis conf = #{config.redis_conf}")
+      end
       puts("Cluster = #{@cluster.name}")
-      log_puts("DB = #{db_mode}")
+      log_puts("DB = #{DB_MODE}")
       log_puts("Cluster = #{@cluster.name}")
       log_puts("Server starts")
     end

data/lib/workflow_manager/version.rb

@@ -1,3 +1,3 @@
 module WorkflowManager
-  VERSION = "0.4.8"
+  VERSION = "0.5.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: workflow_manager
 version: !ruby/object:Gem::Version
-  version: 0.4.8
+  version: 0.5.0
 platform: ruby
 authors:
 - Functional Genomics Center Zurich
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-12-05 00:00:00.000000000 Z
+date: 2020-03-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -75,6 +75,7 @@ files:
 - bin/workflow_manager
 - config/environments/development.rb
 - config/environments/production.rb
+- config/environments/redis.conf
 - lib/workflow_manager.rb
 - lib/workflow_manager/cluster.rb
 - lib/workflow_manager/optparse_ex.rb