tiny_ge 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 96bc1b77a659cad57646a1398044b96c7a4b71c3156383dad83851b01a02147e
+  data.tar.gz: 2004c17254e109aa2df31fcc984238b33bb994df6b603f33e5dd566d08b995b0
+SHA512:
+  metadata.gz: 70890650d6c60bfe67520de5ffca924f4402d53bb1a7453dcb56ca0511e74aa21b7fb948d5bd586fd6d1d26f14a9cd2b63692a4ae5cbfadad1df7bec45029414
+  data.tar.gz: dbd41fc053114b598965059296d2e60e345ccd06d3956ff1bf077f5475fa6502aadd25a54438fd0ebac6f08385019f9c498f572eff0f434fc44edc46482602b5

data/.gitignore

@@ -0,0 +1,8 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.travis.yml

@@ -0,0 +1,6 @@
+---
+language: ruby
+cache: bundler
+rvm:
+  - 2.7.2
+before_install: gem install bundler -v 2.1.4

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at shigeto_nishitani@me.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [https://contributor-covenant.org/version/1/4][version]
+
+[homepage]: https://contributor-covenant.org
+[version]: https://contributor-covenant.org/version/1/4/

data/Gemfile

@@ -0,0 +1,7 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in tiny_ge.gemspec
+gemspec
+
+gem "rake", "~> 12.0"
+gem "minitest", "~> 5.0"

data/Gemfile.lock

@@ -0,0 +1,38 @@
+PATH
+  remote: .
+  specs:
+    tiny_ge (0.1.0)
+      colorize
+      command_line
+      thor
+      yaml
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ansi (1.5.0)
+    builder (3.2.4)
+    colorize (0.8.1)
+    command_line (2.0.1)
+    minitest (5.14.3)
+    minitest-reporters (1.4.3)
+      ansi
+      builder
+      minitest (>= 5.0)
+      ruby-progressbar
+    rake (12.3.3)
+    ruby-progressbar (1.11.0)
+    thor (1.1.0)
+    yaml (0.1.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  minitest (~> 5.0)
+  minitest-reporters
+  rake (~> 12.0)
+  tiny_ge!
+
+BUNDLED WITH
+   2.1.4

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2021 Shigeto R. Nishitani
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.org

@@ -0,0 +1,189 @@
+#+qiita_private: 79491472592ae821dcdc
+#+OPTIONS: ^:{}
+#+STARTUP: indent nolineimages
+#+TITLE: linuxでprocessの排他処理
+#+AUTHOR: Shigeto R. Nishitani
+#+EMAIL:     (concat "shigeto_nishitani@mac.com")
+#+LANGUAGE:  jp
+# +OPTIONS:   H:4 toc:t num:2
+#+OPTIONS:   toc:nil
+#+TAG: Linux, 排他処理
+#+TWITTER: off
+# +SETUPFILE: ~/.emacs.d/org-mode/theme-readtheorg.setup
+
+* intro
+necのvector engineでprocessを排他処理する．
+
+necのVector Engineでmpirunを投げるとそのまま行っちゃう．
+そこでlock fileによる排他処理を提案されたが，そのままだと順序が保証できない．
+そこんとこちょっと改善
+
+* 方針
+統一ファイルを用意して，そこにveにsubmitしたjobのstatusを記録し，
+そこから排他処理と実行を行う．
+
+* 実装
+** 最初の提案
+例えば、ロックファイルを作るのはどうでしょうか？
+#+begin_src shell
+#!/bin/sh
+
+while [ -f "${HOME}/.running" ]; do
+  sleep 10
+done
+touch "${HOME}/.running"
+
+[...]
+mpirun -np 8 vasp_std 1> stdout 2> stderr
+
+rm "${HOME}/.running"
+#+end_src
+これをバックグラウンドで実行すればVEに複数ジョブが一度に入ることはなくなります。
+
+- 実行順は保証されません。
+
+** 最初の実装
+home directory配下にファイルを用意してそこにstatus([finished, running, waiting])を
+書き込んでそれを参照してjobを実行する．
+
+それぞれの投入ジョブのshellは次の通り．
+#+name: check_ve.sh
+#+include: "./test/check_ve.sh" src sh
+
+実際の稼働shellは以下の二つ．
+#+name: check_ve_lock
+#+include: "./lib/check_ve_lock" src ruby
+
+#+name: unlock_ve_lock
+#+include: "./lib/unlock_ve_lock" src ruby
+動いた．
+#+begin_example
+   12753:   finished: /home/bob/bin: 2021-01-26 20:45:56 +0900
+   13209:   finished: /home/bob/bin: 2021-01-26 20:46:05 +0900
+   13407:   finished: /home/bob/bin: 2021-01-26 20:46:14 +0900
+    8512:   finished: /home/bob/frenkel_aurora/al_110_lambda_05: 2021-01-26 23:37:06 +0900
+   20683:    running: /home/bob/frenkel_aurora/al_110_lambda_075: 2021-01-26 23:48:25 +0900
+#+end_example
+** rubyからbackgroundでの実行
+- [[https://stackoverflow.com/questions/11982057/how-can-i-trigger-a-shell-script-and-run-in-background-async-in-ruby][How can I trigger a shell script and run in background (async) in Ruby?]]
+
+に書かれている手法で，
+#+begin_src ruby
+    shell_file = "./test.sh"
+    File.write(shell_file, "sleep 10\necho \"hoge\"\n")
+    command_line("chmod u+x #{shell_file}")
+    p pid = spawn(shell_file, :out => "test.out", :err => "test.err")
+    Process.detach(pid)
+#+end_src
+にて実装．
+
+結果は，
+#+begin_example
+> ls -la --time-style=full-iso test*
+-rw-r--r--. 1 bob bob   0 2021-01-29 12:12:29.572812565 +0900 test.err
+-rw-r--r--. 1 bob bob   5 2021-01-29 12:12:39.575812241 +0900 test.out
+-rwxrw-r--. 1 bob bob  21 2021-01-29 12:12:29.571812565 +0900 test.sh*
+-rw-rw-r--. 1 bob bob 155 2021-01-28 10:01:57.380865004 +0900 test_helper.rb
+#+end_example
+となり，10秒後に書き込まれているのを確認．outファイルは実行直後に出来てたみたい．
+chmodが嫌ですね．
+でも，変なpermissionいらないからuser directoryで実行するshellを生成するのが良さそう．
+
+前回これを実装しようとして，child processとかで悩んだ．
+
+この検索過程で，gemでqueueシステムをいくつも発見．
+railsとかtest用にいくつも開発されている．
+- [[https://www.ruby-toolbox.com/categories/Background_Jobs]]
+- [[https://blog.appsignal.com/2019/04/02/background-processing-system-in-ruby.html][Ruby Magic Learning by building, a Background Processing System in Ruby]]
+ただ，難しそう．．．
+
+
+* Tiny GE
+次節の改善案にしたがってgemで実装．
+SGEのコマンドに似せて作成．
+
+
+#+begin_example
+> tge --help
+
+qsub [shell]        # submit shell job
+qstat               # show queue status
+qdel [pid]          # delete job
+qfinish [pid]       # finish forcely
+#+end_example
+
+shellを指定してqsubすると以下のtest.shが作成される．
+#+begin_src shell
+> cat test.sh
+#!/bin/sh
+while ! qsub 23520; do
+  sleep 10
+done
+
+sh /home/.../ve_lock_vasp.sh
+
+qfinish 23520
+#+end_src
+
+defaultでは~/.tge_test_jobs.txtにqueueがYAML形式で保存されている．
+
+qstatで'running'などの状況を確認．
+#+begin_example shell
+> qstat
+11670: 11702:   finished: /home/bob/tiny_ge/test/hello_world.sh
+14735: 14764:   finished: /home/bob/tiny_ge/test/hello_world.sh
+18515: 18545:   finished: /home/bob/frenkel_aurora/00_1000_10/ve_lock_vasp.sh
+29533: 29562:   finished: /home/bob/frenkel_aurora/00_1000_10/ve_lock_vasp.sh
+18648: 18678:    running: /home/bob/frenkel_aurora/00_1000_10/ve_lock_vasp.sh
+#+end_example
+
+* 改善案
+- [X] テキストにして置いておくより，yamlかjsonが良さそう．
+    見にくいけれど，間違いがないだろうから．
+- [X] qsubを常駐させて，そこにsubmitするという手はないか？
+   1. 常駐させる必要はなくて，qsubに対してve_lock用のshellを常駐させればいい．
+   1. /tmpにおく？
+- [ ] 走ってないゾンビをチェックする必要あり
+- [X] qstat, qdelが必要
+
+- [X] gem化するのが良さそう．そこでは
+     : exeに個別のコマンドを用意するが，実体は同じclassの違う振る舞い
+     とすればいい．
+
+- [X] qsub -> TGE.qsub(pid, shell_path)
+- [X] qfinish -> TGE.qfinish(pid)
+- [X] qstat -> TGE.qstat(line = 10)
+- [X] qdel -> TGE.qdel(pid)
+なんかのmodule methodを用意して，それぞれのコマンドを実行させればいい．
+それにはThorなんかのCLIはいらない．
+
+* テスト
+- [[https://github.com/minitest-reporters/minitest-reporters][minitest-reporters]]
+
+** test_helperの呼び方
+rake testでやるときと，
+: ruby tiny_ge_test.rb -n test_qsub
+とかでやるときでtest_helperが呼ばれない時がある．
+
+: require_relative "./test_helper"
+
+とすると両方で呼ばれる．
+
+#+name: test_helper.rb
+#+begin_ruby
+$LOAD_PATH.unshift(File.expand_path('../../lib', __FILE__))
+#require 'test/unit'
+require "tiny_ge"
+
+require "minitest/autorun"
+require "minitest/reporters"
+Minitest::Reporters.use!
+#+end_ruby
+- qconf
+- かきこ
+** kill zombie
+開発の段階でzombie processの大量発生が起こった．
+
+:  ps -xal |grep test.sh
+とかで親プロセスまで見ることができて．殺せる．
+第４出力が親プロセス．

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
+end
+
+task :default => :test

data/exe/qdel

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require "tiny_ge"
+
+TGE.new.qdel(ARGV[0].to_i)
+

data/exe/qdel~

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require "tiny_ge"
+
+TGE::qstat
+

data/exe/qfinish

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require "tiny_ge"
+
+TGE.new.qfinish(ARGV[0].to_i)
+

data/exe/qfinish~

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require "tiny_ge"
+
+TGE.new.qdel(ARGV[0].to_i)
+

data/exe/qstat

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require "tiny_ge"
+
+TGE.new.qstat(ARGV[0].to_i)
+
+

data/exe/qstat~

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require "tiny_ge"
+
+TGE::qstat(-10)
+

data/exe/qsub

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+
+require "tiny_ge"
+
+pid = $$
+file = ARGV[0]
+accepted_formats = ['.sh']
+unless accepted_formats.include? File.extname(file)
+  pid = file.to_i
+end
+shell_path = File.join(Dir.pwd, ARGV[0])
+
+res = TGE.new.qsub(pid, shell_path)
+#p ['qsub_status', res, pid]
+if res
+  exit 0
+else
+  exit 1
+end
+

data/exe/qsub~

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+
+require "tiny_ge"
+require 'thor'
+
+module TinyGE
+  class MyCLI < Thor
+    desc "hello NAME", "say hello to NAME"
+    def hello(name)
+      puts "Hello " + name
+    end
+  end
+end
+
+TinyGE::MyCLI.start(ARGV)
+

data/exe/tge

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+str = <<-EOF
+tge : tiny GE is a tiny queue system like SGE
+
+qsub [shell]        # submit job
+qstat               # show queue stats
+qdel [pid]          # delete job
+qfinish             # use internal
+EOF
+
+puts str
+

data/exe/tge~

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+str = <<- EOF
+hello
+EOF
+
+puts str
+

data/lib/#tiny_ge.rb#

@@ -0,0 +1,121 @@
+require "tiny_ge/version"
+require 'yaml'
+require 'thor'
+require 'command_line/global'
+
+#VE_SUBMIT_JOBS_FILE = File.join(ENV['HOME'],".ve_submit_jobs.txt")
+VE_TEST_FILE = File.join(ENV['HOME'],".tge_test_jobs.txt")
+
+class TGE
+  def initialize(q_file=VE_TEST_FILE)
+    @q_file = VE_TEST_FILE
+    @data = YAML.load(File.read(@q_file))
+  end
+
+  def add_job(pid, shell_path)
+    shell_file = "./test.s#{pid}"
+    shell_script = mk_shell_script(pid, shell_path)
+    File.write(shell_file, shell_script)
+
+    p pid0 = spawn("sh #{shell_file}", :out => "test.o#{pid}", :err => "test.e#{pid}")
+    Process.detach(pid0)
+    puts "#{pid} is added on the queue."
+
+    @data << {pid: pid, status: 'waiting', shell_path: shell_path,
+      real_pid: pid0,
+      submit: Time.now,
+      start: nil,
+      finish: nil
+    }
+    File.write(VE_TEST_FILE, YAML.dump(@data))
+  end
+
+  def change_job_status(pid, status)
+    @data.each do |job, i|
+      if job[:pid] == pid
+        job[:status] = status
+        case status
+        when 'running' ;  job[:start] = Time.now
+        when 'finished';  job[:finish] = Time.now
+        end
+        File.write(VE_TEST_FILE, YAML.dump(@data))
+        break
+      end
+    end
+  end
+
+  def qfinish(pid)
+    change_job_status(pid, 'finished')
+    return true
+  end
+
+  def qsub(pid, shell_path=Dir.pwd)
+    unless pid_on_file(pid)
+      add_job(pid, shell_path)
+      return false
+    end
+    last_finished = -1
+    @data.each_with_index do |job, i|
+      if job[:pid] == pid
+        if job[:status] == 'waiting' and i == last_finished + 1
+          change_job_status(pid, 'running')
+          return true
+        end
+        if job[:status] == 'running'
+          return true
+        end
+        return false
+      end
+      last_finished = i if job[:status] == 'finished'
+    end
+  end
+
+  def pid_on_file(pid)
+    @data.each do |job, i|
+      return job[:status] if job[:pid] == pid
+    end
+    return false
+  end
+
+  def qdel(pid)
+    unless pid_on_file(pid)
+      puts "#{pid} is not on the qeueu."
+      return false
+    end
+    @data.each_with_index do |job, i|
+      if job[:pid] == pid
+        res = command_line("kill -9 #{job[:real_pid]}")
+        p res
+        @data.delete_at(i)
+        File.write(VE_TEST_FILE, YAML.dump(@data))
+        puts "#{pid} is deleted from the qeueu."
+        return true
+      end
+    end
+  end
+
+  def qstat(item_num=0)
+    if item_num == '-f'
+      item_num 
+    @data = YAML.load(File.read(VE_TEST_FILE))
+    @data[item_num..-1].each do |job, i|
+      real_pid = job[:real_pid] || 0
+      puts "%5d: %5d: %10s: %s" % [job[:pid], real_pid, job[:status], job[:shell_path]]
+    end
+  end
+
+  def mk_shell_script(pid, shell_path)
+    return <<~EOS
+    #!/bin/sh
+    while ! qsub #{pid}; do
+      sleep 10
+    done
+
+    sh #{shell_path}
+
+    qfinish #{pid}
+EOS
+  end
+
+end
+

data/lib/.#tiny_ge.rb

@@ -0,0 +1 @@
+lib/bob@aurora0.13411:1611034391

data/lib/check_ve_lock

@@ -0,0 +1,30 @@
+#!/bin/env ruby
+#VE_SUBMIT_JOBS_FILE = File.join(ENV['HOME'],".ve_submit_jobs.txt")
+VE_SUBMIT_JOBS_FILE = File.join(ENV['HOME'],".tge_test_jobs.txt")
+pid = ARGV[0] || 1234
+pid = pid.to_i
+
+status = ['not_on_file', -1]
+lines = File.readlines(VE_SUBMIT_JOBS_FILE)
+finished_job = -1
+lines.each_with_index do |line, i|
+  data = line.match(/\s+(\d+):\s+(\w+):\s+(.+):/)
+  if data[1].to_i == pid
+    status = [data[2], i]
+    if status[1] == finished_job + 1
+      line.gsub!('waiting','running')
+      File.write(VE_SUBMIT_JOBS_FILE, lines.join)
+      exit 0
+    end
+    break 
+  end
+  finished_job = i if data[2] == 'finished'
+end
+
+if status[0] == 'not_on_file'
+  line = "%8d: %10s: %s: %s\n" % [pid, 'waiting', Dir.pwd(), Time.now]
+  File.open(VE_SUBMIT_JOBS_FILE, 'a'){ |f| f.write line }
+end
+
+exit 1 # return false
+#exit 0 # return true

data/lib/check_ve_lock~

@@ -0,0 +1,30 @@
+#!/bin/env ruby
+#VE_SUBMIT_JOBS_FILE = File.join(ENV['HOME'],".ve_submit_jobs.txt")
+VE_SUBMIT_JOBS_FILE = File.join(ENV['HOME'],".tge_test_jobs.txt")
+pid = ARGV[0] || 1234
+pid = pid.to_i
+
+status = ['not_on_file', -1]
+lines = File.readlines(VE_SUBMIT_JOBS_FILE)
+finished_job = -1
+lines.each_with_index do |line, i|
+  data = line.match(/\s+(\d+):\s+(\w+):\s+(.+):/)
+  if data[1].to_i == pid
+    status = [data[2], i]
+    if status[1] == finished_job + 1
+      line.gsub!('waiting','running')
+      File.write(VE_SUBMIT_JOBS_FILE, lines.join)
+      exit 0
+    end
+    break 
+  end
+  finished_job = i if data[2] == 'finished'
+end
+
+if status[0] == 'not_on_file'
+  line = "%8d: %10s: %s: %s\n" % [pid, 'waiting', Dir.pwd(), Time.now]
+  File.open(VE_SUBMIT_JOBS_FILE, 'a'){ |f| f.write line }
+end
+
+exit 1 # return false
+#exit 0 # return true

data/lib/tiny_ge.rb

@@ -0,0 +1,119 @@
+require "tiny_ge/version"
+require 'yaml'
+require 'thor'
+require 'command_line/global'
+
+#VE_SUBMIT_JOBS_FILE = File.join(ENV['HOME'],".ve_submit_jobs.txt")
+VE_TEST_FILE = File.join(ENV['HOME'],".tge_test_jobs.txt")
+
+class TGE
+  def initialize(q_file=VE_TEST_FILE)
+    @q_file = VE_TEST_FILE
+    @data = YAML.load(File.read(@q_file))
+  end
+
+  def add_job(pid, shell_path)
+    shell_file = "./test.s#{pid}"
+    shell_script = mk_shell_script(pid, shell_path)
+    File.write(shell_file, shell_script)
+
+    p pid0 = spawn("sh #{shell_file}", :out => "test.o#{pid}", :err => "test.e#{pid}")
+    Process.detach(pid0)
+    puts "#{pid} is added on the queue."
+
+    @data << {pid: pid, status: 'waiting', shell_path: shell_path,
+      real_pid: pid0,
+      submit: Time.now,
+      start: nil,
+      finish: nil
+    }
+    File.write(VE_TEST_FILE, YAML.dump(@data))
+  end
+
+  def change_job_status(pid, status)
+    @data.each do |job, i|
+      if job[:pid] == pid
+        job[:status] = status
+        case status
+        when 'running' ;  job[:start] = Time.now
+        when 'finished';  job[:finish] = Time.now
+        end
+        File.write(VE_TEST_FILE, YAML.dump(@data))
+        break
+      end
+    end
+  end
+
+  def qfinish(pid)
+    change_job_status(pid, 'finished')
+    return true
+  end
+
+  def qsub(pid, shell_path=Dir.pwd)
+    unless pid_on_file(pid)
+      add_job(pid, shell_path)
+      return false
+    end
+    last_finished = -1
+    @data.each_with_index do |job, i|
+      if job[:pid] == pid
+        if job[:status] == 'waiting' and i == last_finished + 1
+          change_job_status(pid, 'running')
+          return true
+        end
+        if job[:status] == 'running'
+          return true
+        end
+        return false
+      end
+      last_finished = i if job[:status] == 'finished'
+    end
+  end
+
+  def pid_on_file(pid)
+    @data.each do |job, i|
+      return job[:status] if job[:pid] == pid
+    end
+    return false
+  end
+
+  def qdel(pid)
+    unless pid_on_file(pid)
+      puts "#{pid} is not on the qeueu."
+      return false
+    end
+    @data.each_with_index do |job, i|
+      if job[:pid] == pid
+        res = command_line("kill -9 #{job[:real_pid]}")
+        p res
+        @data.delete_at(i)
+        File.write(VE_TEST_FILE, YAML.dump(@data))
+        puts "#{pid} is deleted from the qeueu."
+        return true
+      end
+    end
+  end
+
+  def qstat(item_num=0)
+    @data = YAML.load(File.read(VE_TEST_FILE))
+    @data[item_num..-1].each do |job, i|
+      real_pid = job[:real_pid] || 0
+      puts "%5d: %5d: %10s: %s" % [job[:pid], real_pid, job[:status], job[:shell_path]]
+    end
+  end
+
+  def mk_shell_script(pid, shell_path)
+    return <<~EOS
+    #!/bin/sh
+    while ! qsub #{pid}; do
+      sleep 10
+    done
+
+    sh #{shell_path}
+
+    qfinish #{pid}
+EOS
+  end
+
+end
+

data/lib/tiny_ge/version.rb

@@ -0,0 +1,3 @@
+class TGE
+  VERSION = "0.1.0"
+end

data/lib/unlock_ve_lock

@@ -0,0 +1,16 @@
+#!/bin/env ruby
+#VE_SUBMIT_JOBS_FILE = File.join(ENV['HOME'],".ve_submit_jobs.txt")
+VE_SUBMIT_JOBS_FILE = File.join(ENV['HOME'],".tge_test_jobs.txt")
+pid = ARGV[0] || 1234
+pid = pid.to_i
+lines = File.readlines(VE_SUBMIT_JOBS_FILE)
+lines.each_with_index do |line, i|
+  data = line.match(/\s+(\d+):\s+(\w+):\s+(.+)/)
+  if data[1].to_i == pid
+    line.gsub!(' running', 'finished')
+    File.write(VE_SUBMIT_JOBS_FILE, lines.join)
+    break 
+  end
+end
+
+exit 0 # return true

data/lib/unlock_ve_lock~

@@ -0,0 +1,15 @@
+#!/bin/env ruby
+VE_SUBMIT_JOBS_FILE = File.join(ENV['HOME'],".ve_submit_jobs.txt")
+pid = ARGV[0] || 1234
+pid = pid.to_i
+lines = File.readlines(VE_SUBMIT_JOBS_FILE)
+lines.each_with_index do |line, i|
+  data = line.match(/\s+(\d+):\s+(\w+):\s+(.+)/)
+  if data[1].to_i == pid
+    line.gsub!(' running', 'finished')
+    File.write(VE_SUBMIT_JOBS_FILE, lines.join)
+    break 
+  end
+end
+
+exit 0 # return true

data/tiny_ge.gemspec

@@ -0,0 +1,34 @@
+require_relative 'lib/tiny_ge/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "tiny_ge"
+  spec.version       = TGE::VERSION
+  spec.authors       = ["Shigeto R. Nishitani"]
+  spec.email         = ["shigeto_nishitani@me.com"]
+
+  spec.summary       = %q{tiny_ge is a tiny queue system like SGE}
+  spec.description   = %q{tiny_ge is a tiny queue system like SGE}
+  spec.homepage      = "https://github.com/daddygongon/tiny_ge"
+  spec.license       = "MIT"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+
+#  spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/daddygongon/tiny_ge"
+  spec.metadata["changelog_uri"] = "https://github.com/daddygongon/tiny_ge"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+  spec.add_runtime_dependency('thor')
+  spec.add_runtime_dependency "colorize"
+  spec.add_runtime_dependency "command_line"
+  spec.add_runtime_dependency "yaml"
+  spec.add_development_dependency "minitest-reporters"
+end

metadata

@@ -0,0 +1,153 @@
+--- !ruby/object:Gem::Specification
+name: tiny_ge
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Shigeto R. Nishitani
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2021-02-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: command_line
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yaml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest-reporters
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: tiny_ge is a tiny queue system like SGE
+email:
+- shigeto_nishitani@me.com
+executables:
+- qdel
+- qdel~
+- qfinish
+- qfinish~
+- qstat
+- qstat~
+- qsub
+- qsub~
+- tge
+- tge~
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.org
+- Rakefile
+- exe/qdel
+- exe/qdel~
+- exe/qfinish
+- exe/qfinish~
+- exe/qstat
+- exe/qstat~
+- exe/qsub
+- exe/qsub~
+- exe/tge
+- exe/tge~
+- lib/#tiny_ge.rb#
+- lib/.#tiny_ge.rb
+- lib/check_ve_lock
+- lib/check_ve_lock~
+- lib/tiny_ge.rb
+- lib/tiny_ge/version.rb
+- lib/unlock_ve_lock
+- lib/unlock_ve_lock~
+- tiny_ge.gemspec
+homepage: https://github.com/daddygongon/tiny_ge
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/daddygongon/tiny_ge
+  source_code_uri: https://github.com/daddygongon/tiny_ge
+  changelog_uri: https://github.com/daddygongon/tiny_ge
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.4
+signing_key: 
+specification_version: 4
+summary: tiny_ge is a tiny queue system like SGE
+test_files: []