jkr 0.1.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5cc3028f9f18f51c7dae2c3c382bfa44763c0df2
-  data.tar.gz: 538c2f84ff9a128ef584be7d5f37d9e9c9263731
+  metadata.gz: c2aa3b45798602f7adfa2ef39671bbd0c67248ad
+  data.tar.gz: 9033b8152c9b54663ba76df543cb11546bdb5f39
 SHA512:
-  metadata.gz: 8ccfb21a505843e30005c5a402fcbf6da1cab077a25f8781d7e3b3ee5df3761567bbfccbcf4abd07585e9f06466b0587db63c3d55994160de3f6ed34cbcc49ff
-  data.tar.gz: 55ec247337556c9fd37c6d446934f3962b34e932615acb478ad9018ae36c0fd365e7c04a1dc34bc4e9e7ee3b7326cc002751e2fe4ddd37fad8390e72849c0ede
+  metadata.gz: 9dc067dcb1007050e2317a34a2c6afa23a1dd82caa66b06fdef2e58b8d2779fab496780f39efedfbbd338cd41d7408faf3569273cb608eefa4cf3772c2f45128
+  data.tar.gz: 37d50d20c882cde08ed6527131f5d3ed4c2e690c60c7f05052727bb5244877449aa36ad45310e0e12a0e19f9c61696309f70891a06b7c892b3839cdf2f68962e

data/.gitignore

@@ -11,3 +11,4 @@
 /bin/rake2thor
 /bin/thor
 /pkg/*.gem
+/Gemfile.lock

data/etc/zsh-comp.sh

@@ -76,7 +76,10 @@ _jkr() {
 
 __jkr_subcmds() {
     _values 'Jkr command' \
+            'init[Init a Jkr environment]' \
             'execute[Execute a plan]' \
+            'resume[Resume in-progress execution]' \
             'list[List jkr plans]' \
-            'analyze[Analyze a result]'
+            'analyze[Analyze a result]' \
+            'query[Query interesting result]'
 }

data/lib/jkr/cli.rb

@@ -43,13 +43,7 @@ module Jkr
 
     desc "list", "List executable plans"
     def list()
-      begin
-        @jkr_env = Jkr::Env.new(options[:directory])
-      rescue Errno::ENOENT
-        puts(red("[ERROR] jkr dir not found at #{@options[:directory]}"))
-        puts(red("        Maybe you are in a wrong directory."))
-        exit(false)
-      end
+      @jkr_env = create_env()
 
       plans = Dir.glob("#{@jkr_env.jkr_plan_dir}/*.plan").map do |plan_file_path|
         plan = Jkr::Plan.create_by_name(@jkr_env, File.basename(plan_file_path, ".plan"))
@@ -74,7 +68,7 @@ module Jkr
 
     desc "execute <plan> [<plan> ...]", "Execute plans"
     def execute(*plan_names)
-      @jkr_env = Jkr::Env.new(options[:directory])
+      @jkr_env = create_env()
 
       if options[:debug]
         delete_files_on_error = false
@@ -89,21 +83,84 @@ module Jkr
       end
     end
 
-    desc "analyze <result> [<result> ...]", "Run analysis script for executed results"
+    desc "resume [<result> ...]", "Resume in-progress execution"
+    def resume(*result_ids)
+      @jkr_env = create_env()
+      cur_result_id = Jkr::Env.find_result(Dir.pwd)
+      if cur_result_id
+        result_ids.push(cur_result_id)
+      end
+
+      result_ids.each do |rid|
+        plan = Plan.create_by_result_id(@jkr_env, rid)
+        Jkr::Trial.resume(@jkr_env, plan)
+      end
+    end
+
+    desc "analyze [<result> ...]", "Run analysis script for executed results"
     def analyze(*result_ids)
-      @jkr_env = Jkr::Env.new(options[:directory])
+      @jkr_env = create_env()
+
+      # check if current dir is a result dir
+      cur_result_id = Jkr::Env.find_result(Dir.pwd)
+      if cur_result_id
+        result_ids.push(cur_result_id)
+      end
 
       result_ids.each do |arg|
         Jkr::Analysis.analyze(@jkr_env, arg)
       end
     end
 
+    desc "query <result>", "Query interesting result"
+    def query(result_id = nil)
+      @jkr_env = create_env()
+
+      if result_id == nil
+        # check if current dir is a result dir
+        cur_result_id = Jkr::Env.find_result(Dir.pwd)
+        if cur_result_id
+          result_id = cur_result_id
+        else
+          raise ArgumentError.new("Result ID must be specified.")
+        end
+      end
+
+      result_dir = Dir.glob(sprintf("#{@jkr_env.jkr_result_dir}/%05d*", result_id.to_i)).first
+
+      Dir.glob("#{result_dir}/[0-9][0-9][0-9][0-9][0-9]/metastore.msh").sort.each do |m|
+        metastore = Marshal.load(File.open(m))
+        params = Marshal.load(File.open(File.expand_path("../params.msh", m)))
+
+        disp = metastore[:vars].map do |var|
+          "#{var}: #{params[var].inspect}"
+        end.join(", ")
+
+        puts "#{File.basename(File.dirname(m))}	| #{disp}"
+      end
+    end
+
     no_commands do
       def find_plan_file(plan_name)
         @jkr_env.plans.find do |plan_file_path|
           File.basename(plan_file_path) == plan_name + ".plan"
         end
       end
+
+      def create_env()
+        begin
+          if options[:directory]
+            return Jkr::Env.new(options[:directory])
+          else
+            return Jkr::Env.new(Jkr::Env.find(Dir.pwd))
+          end
+        rescue Errno::ENOENT
+          $stderr.puts(red("[ERROR] jkr dir not found at #{@options[:directory]}"))
+          $stderr.puts(red("        Maybe you are in a wrong directory."))
+          exit(false)
+        end
+
+      end
     end
   end
 

data/lib/jkr/env.rb

@@ -21,7 +21,7 @@ module Jkr
       @jkr_result_dir = File.join(@jkr_dir, RESULT_DIR)
       @jkr_script_dir = File.join(@jkr_dir, SCRIPT_DIR)
 
-      unless Dir.exists?(@jkr_dir)
+      unless Dir.exists?(@env_dir)
         raise Errno::ENOENT.new(@jkr_dir)
       end
 
@@ -34,5 +34,64 @@ module Jkr
         end
       end
     end
+
+    def self.valid_env_dir?(dir)
+      jkr_dir = File.expand_path("jkr", dir)
+      plan_dir = File.expand_path("plan", jkr_dir)
+      script_dir = File.expand_path("script", jkr_dir)
+      result_dir = File.expand_path("result", jkr_dir)
+
+      [jkr_dir, result_dir, plan_dir, script_dir].each do |dir_|
+        unless Dir.exists?(dir_)
+          return false
+        end
+      end
+
+      true
+    end
+
+    # Find Jkr env dir if 'dir' is under an valid Jkr environemnt directory.
+    def self.find(dir)
+      dir = File.expand_path("./", dir)
+      while true
+        if valid_env_dir?(dir)
+          return dir
+        end
+
+        parent_dir = File.expand_path("../", dir)
+        if parent_dir == dir
+          break
+        else
+          dir = parent_dir
+        end
+      end
+
+      nil
+    end
+
+    # Find an executed Jkr result if 'dir' is under the result dir, and return result id.
+    # return nil otherwise.
+    def self.find_result(dir)
+      dir = File.expand_path("./", dir)
+      while true
+        parent_dir = File.expand_path("../", dir)
+        gp_dir = File.expand_path("../../", dir) # grand parent
+        ggp_dir = File.expand_path("../../../", dir) # grand grand parent
+
+        if ggp_dir == gp_dir
+          return nil
+        end
+
+        if valid_env_dir?(ggp_dir)
+          if File.basename(gp_dir) == "jkr" &&
+              File.basename(parent_dir) == "result" &&
+              File.basename(dir) =~ /^[0-9]+/
+            return File.basename(dir).to_i
+          end
+        end
+
+        dir = File.expand_path("../", dir)
+      end
+    end
   end
 end

data/lib/jkr/plan.rb

@@ -93,12 +93,22 @@ module Jkr
       plan.file_path = finder.find_by_result_id(ret_id)
 
       plan.plan_search_path = [File.expand_path("../plan", plan.file_path)]
-      plan.script_search_path = [File.expand_path("../script", plan.file_path)]
+      plan.script_search_path = [File.expand_path("../script", plan.file_path),
+                                 plan.jkr_env.jkr_script_dir]
 
       unless plan.file_path
         raise ArgumentError.new("Not valid result ID: #{ret_id}")
       end
 
+      resultset_num = sprintf "%05d", ret_id
+      dirs = Dir.glob(File.join(plan.jkr_env.jkr_result_dir, resultset_num)+"*")
+      if dirs.size == 0
+        raise RuntimeError.new("Resultset not found: #{ret_id}")
+      elsif dirs.size > 1
+        raise RuntimeError.new("Cannot identify result set directory")
+      end
+      plan.resultset_dir = dirs.first
+
       PlanLoader.load_plan(plan)
 
       plan
@@ -185,7 +195,7 @@ module Jkr
         @plan
       end
 
-      def use_script(name)
+      def use_script(name, use = true)
         # find script file
         if name.is_a? Symbol
           name = name.to_s + ".rb"
@@ -206,12 +216,18 @@ module Jkr
 
         if path
           load path
-          @plan.used_scripts.push(path)
+          if use
+            @plan.used_scripts.push(path)
+          end
         else
           raise RuntimeError.new("Cannot use script: #{name}")
         end
       end
 
+      def load_script(name)
+        use_script(name, false)
+      end
+
       def extend(base_plan_name)
         base_plan = Plan.create_by_name(self.plan.jkr_env, base_plan_name.to_s,
                                         :plan_search_path => @plan.plan_search_path,
@@ -374,7 +390,7 @@ module Jkr
                             {'message'=>message})
       end
 
-      def sh(*args)
+      def sh_(*args)
         puts "sh: #{args.join(' ')}"
         return system(*args)
       end
@@ -387,8 +403,12 @@ module Jkr
         true
       end
 
+      # for backward compatibility
       alias :system_ :sh!
 
+      # raise Error on failure by default
+      alias :sh :sh!
+
       def su_sh(*args)
         puts "su_sh: #{args.join(' ')}"
         su_cmd = File.expand_path("../su_cmd", __FILE__)

data/lib/jkr/trial.rb

@@ -6,6 +6,8 @@ require 'jkr/utils'
 module Jkr
   class Trial
     attr_reader :params
+    attr_reader :result_dir
+
     def self.make_trials(resultset_dir, plan)
       var_combs = [{}]
       plan.vars.each do |key, vals|
@@ -25,8 +27,7 @@ module Jkr
       params_list = params_list * plan.routine_nr_run
 
       params_list.map do |params|
-        result_dir = Utils.reserve_next_dir(resultset_dir)
-        Trial.new(result_dir, plan, params)
+        Trial.new(resultset_dir, plan, params)
       end
     end
 
@@ -40,7 +41,7 @@ module Jkr
 
     def self.run(env, plan, delete_files_on_error = true)
       plan_suffix = File.basename(plan.file_path, ".plan")
-      plan_suffix += "_#{plan.short_desc}" if plan.short_desc
+      plan_suffix += "_#{plan.short_desc}" if plan.short_desc && plan.short_desc.size > 0
       resultset_dir = Utils.reserve_next_dir(env.jkr_result_dir, plan_suffix)
       plan.resultset_dir = resultset_dir
       FileUtils.mkdir_p(File.join(resultset_dir, "plan"))
@@ -73,28 +74,71 @@ module Jkr
         params = plan.params.merge(plan.vars)
         plan.freeze
 
+        save_info(plan, :start_time => Time.now)
+
         # show estimated execution time if available
         if plan.exec_time_estimate
           puts("")
           puts("== estimated execution time: #{pretty_time(plan.exec_time_estimate.call(plan))} ==")
           puts("")
         end
-
-        plan.do_prep()
-        trials.each do |trial|
-          trial.run
-        end
-        plan.do_cleanup()
       rescue Exception => err
         if delete_files_on_error
           FileUtils.rm_rf(resultset_dir)
         end
         raise err
       end
+
+      resume(env, plan)
+    end
+
+    def self.resume(env, plan)
+      trials = self.make_trials(plan.resultset_dir, plan)
+
+      save_info(plan, :last_resume_time => Time.now)
+
+      plan.do_prep()
+      trials.each do |trial|
+        begin
+          trial.run
+        rescue Exception => err
+          failed_dir = File.expand_path("../_failed_" + File.basename(trial.result_dir),
+                                        trial.result_dir)
+          i = 1
+          while Dir.exists?(failed_dir)
+            failed_dir = File.expand_path("../_failed_" + File.basename(trial.result_dir) + "_#{i}",
+                                          trial.result_dir)
+            i += 1
+          end
+          FileUtils.mv(trial.result_dir, failed_dir)
+          save_info(plan, :last_failure_time => Time.now)
+
+          raise err
+        end
+      end
+      plan.do_cleanup()
+
+      save_info(plan, :finish_time => Time.now)
+    end
+
+    def self.save_info(plan, data = {})
+      info_path = File.expand_path("INFO", plan.resultset_dir)
+      unless File.exists?(info_path)
+        File.open(info_path, "w") do |f|
+          f.puts(JSON.pretty_generate({}))
+        end
+      end
+
+      info_data = JSON.load(File.open(info_path))
+      info_data = info_data.merge(data)
+
+      File.open(info_path, "w") do |f|
+        f.puts(JSON.pretty_generate(info_data))
+      end
     end
 
-    def initialize(result_dir, plan, params)
-      @result_dir = result_dir
+    def initialize(resultset_dir, plan, params)
+      @resultset_dir = resultset_dir
       @plan = plan
       @params = params
     end
@@ -102,27 +146,50 @@ module Jkr
 
     def run()
       plan = @plan
-      File.open("#{@result_dir}/params.json", "w") do |f|
-        f.puts(@params.to_json)
+
+      # check duplicate execution
+      Dir.glob("#{@resultset_dir}/*").select do |path|
+        File.basename(path) =~ /^[0-9]{5}$/
+      end.each do |result_dir|
+        params = Marshal.load(File.open("#{result_dir}/params.msh"))
+
+        if params == @params
+          # already executed trial. skip.
+          return false
+        end
       end
 
-      # reset plan.metastore
-      plan.metastore.clear
+      # make result dir for this trial
+      @result_dir = Utils.reserve_next_dir(@resultset_dir)
+
+      Dir.chdir(@result_dir) do
+        # save params
+        File.open("#{@result_dir}/params.msh", "w") do |f|
+          Marshal.dump(@params, f)
+        end
+        File.open("#{@result_dir}/params.json", "w") do |f|
+          f.puts(JSON.pretty_generate(@params))
+        end
 
-      # define utility functions for plan.routine object
-      Jkr::TrialUtils.define_routine_utils(@result_dir, @plan, @params)
+        # reset plan.metastore
+        plan.metastore.clear
 
-      @plan.metastore[:trial_start_time] = Time.now
-      @plan.do_routine(@plan, @params)
-      @plan.metastore[:trial_end_time] = Time.now
+        # define utility functions for plan.routine object
+        Jkr::TrialUtils.define_routine_utils(@result_dir, @plan, @params)
 
-      Jkr::TrialUtils.undef_routine_utils(@plan)
+        @plan.metastore[:vars] = @plan.vars.keys
+        @plan.metastore[:trial_start_time] = Time.now
+        @plan.do_routine(@plan, @params)
+        @plan.metastore[:trial_end_time] = Time.now
 
-      # save plan.metastore
-      Marshal.dump(@plan.metastore,
-                   File.open("#{@result_dir}/metastore.msh", "w"))
-      File.open("#{@result_dir}/metastore.json", "w") do |f|
-        f.puts(@plan.metastore.to_json)
+        Jkr::TrialUtils.undef_routine_utils(@plan)
+
+        # save plan.metastore
+        Marshal.dump(@plan.metastore,
+                     File.open("#{@result_dir}/metastore.msh", "w"))
+        File.open("#{@result_dir}/metastore.json", "w") do |f|
+          f.puts(JSON.pretty_generate(@plan.metastore))
+        end
       end
     end
   end

data/lib/jkr/version.rb

@@ -1,3 +1,3 @@
 module Jkr
-  VERSION = '0.1.0'
+  VERSION = '0.2.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: jkr
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Yuto Hayamizu
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-10-08 00:00:00.000000000 Z
+date: 2016-11-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -133,7 +133,6 @@ files:
 - ".gitignore"
 - ".rspec"
 - Gemfile
-- Gemfile.lock
 - Guardfile
 - History.txt
 - LICENSE.txt

data/Gemfile.lock

@@ -1,81 +0,0 @@
-PATH
-  remote: .
-  specs:
-    jkr (0.1.0)
-      term-ansicolor
-      thor
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    coderay (1.1.1)
-    diff-lcs (1.2.5)
-    ffi (1.9.14)
-    formatador (0.2.5)
-    guard (2.14.0)
-      formatador (>= 0.2.4)
-      listen (>= 2.7, < 4.0)
-      lumberjack (~> 1.0)
-      nenv (~> 0.1)
-      notiffany (~> 0.0)
-      pry (>= 0.9.12)
-      shellany (~> 0.0)
-      thor (>= 0.18.1)
-    guard-compat (1.2.1)
-    guard-rspec (4.7.3)
-      guard (~> 2.1)
-      guard-compat (~> 1.1)
-      rspec (>= 2.99.0, < 4.0)
-    listen (3.1.5)
-      rb-fsevent (~> 0.9, >= 0.9.4)
-      rb-inotify (~> 0.9, >= 0.9.7)
-      ruby_dep (~> 1.2)
-    lumberjack (1.0.10)
-    method_source (0.8.2)
-    nenv (0.3.0)
-    notiffany (0.1.1)
-      nenv (~> 0.1)
-      shellany (~> 0.0)
-    pry (0.10.4)
-      coderay (~> 1.1.0)
-      method_source (~> 0.8.1)
-      slop (~> 3.4)
-    rake (11.2.2)
-    rb-fsevent (0.9.7)
-    rb-inotify (0.9.7)
-      ffi (>= 0.5.0)
-    rspec (3.5.0)
-      rspec-core (~> 3.5.0)
-      rspec-expectations (~> 3.5.0)
-      rspec-mocks (~> 3.5.0)
-    rspec-core (3.5.2)
-      rspec-support (~> 3.5.0)
-    rspec-expectations (3.5.0)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.5.0)
-    rspec-mocks (3.5.0)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.5.0)
-    rspec-support (3.5.0)
-    ruby_dep (1.4.0)
-    shellany (0.0.1)
-    slop (3.6.0)
-    term-ansicolor (1.3.2)
-      tins (~> 1.0)
-    thor (0.19.1)
-    tins (1.12.0)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  bundler
-  guard
-  guard-rspec
-  jkr!
-  pry
-  rake
-  rspec
-
-BUNDLED WITH
-   1.11.2