rbbt-util 5.23.35 → 5.23.36

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8778e323b3d7445ea05f92a9f4a5db9e4c2ddbc1
-  data.tar.gz: 863e8f1e3e03559bf9bac87b60f6cb725ec7bf6e
+  metadata.gz: a48bd703a6bcf70e3b03ea0a80df3d9e419d2a5d
+  data.tar.gz: e5ddc970668f140bd1923ce200b2754d352c8e61
 SHA512:
-  metadata.gz: 6866e00f193fca319f313c168d08e7cadbb50dae04dd0299d602a6baee292baf4c1dc9f18dedf186bac95caf21ee0d984ecae42edd12ccbf1fc012b46309cb63
-  data.tar.gz: 07e1418d61e8188df8bdd77d496243c4c9c09526c6250722449369607d755b867f6620007cafd12828f3b5e6f6b8083a387bee9764e52c1a28dbfec3c733194b
+  metadata.gz: 2023508ca619102a57ecc4927e08b97c14b2429cea38c57e14c75ce743006772080ab8daad7e7d17a721a11646da7806a9376dbb17e6f25686129418b2fa04db
+  data.tar.gz: 5439545d3471476c701f42c133398d8211bb1cfd5adc4a6d802954ece07fb0a509d0f0c59fa07085f9b7f4ec6097390d9c0cd527e2c90ff735eca8d7e84c0ad4

data/lib/rbbt/monitor.rb

@@ -138,18 +138,45 @@ module Rbbt
     dirs.collect do |dir|
       next unless Open.exists? dir
 
-      dir.glob("*").collect do |workflowdir|
+      workflowdirs = if (dir_sub_path = Open.find_repo_dir(workflowdir))
+                       repo_dir, sub_path = dir_sub_path
+                       Open.list_repo_files(*dir_sub_path).collect{|f| f.split("/").first}.uniq.collect{|f| File.join(repo_dir, f)}.uniq
+                     else
+                       dir.glob("*")
+                     end
+
+      workflowdirs.collect do |workflowdir|
         workflow = File.basename(workflowdir)
         next if workflows and not workflows.include? workflow
 
-        workflowdir.glob("*").collect do |taskdir|
+        tasks_dirs = if (dir_sub_path = Open.find_repo_dir(workflowdir))
+                       repo_dir, sub_path = dir_sub_path
+                       Open.list_repo_files(*dir_sub_path).collect{|f| f.split("/").first}.uniq.collect{|f| File.join(repo_dir, f)}.uniq
+                     else
+                       workflowdir.glob("*")
+                     end
+
+        tasks_dirs.collect do |taskdir|
           task = File.basename(taskdir)
           next if tasks and not tasks.include? task
 
-          #cmd = "find -L '#{ taskdir }/'  -not \\( -path \"#{taskdir}/*.files/*\" -prune \\) -not -name '*.pid' -not -name '*.notify' -not -name '\\.*' 2>/dev/null"
-          cmd = "find -L '#{ taskdir }/' -not \\( -path \"#{taskdir}/.info/*\" -prune \\) -not \\( -path \"#{taskdir}/*.files/*\" -prune \\) -not -name '*.pid' -not -name '*.notify' -not -name '\\.*' \\( -not -type d -o -name '*.files' \\)  2>/dev/null"
 
-          files = CMD.cmd(cmd, :pipe => true)
+          files = if (dir_sub_path = Open.find_repo_dir(taskdir))
+                    repo_dir, sub_path = dir_sub_path
+                    Open.list_repo_files(*dir_sub_path).reject do |f|
+                      f.include?("/.info/") ||
+                        f.include?(".files/") ||
+                        f.include?(".pid/") ||
+                        File.directory?(f)
+                    end.collect do |f|
+                      File.join(repo_dir, f)
+                    end
+                  else
+                    #cmd = "find -L '#{ taskdir }/'  -not \\( -path \"#{taskdir}/*.files/*\" -prune \\) -not -name '*.pid' -not -name '*.notify' -not -name '\\.*' 2>/dev/null"
+                    cmd = "find -L '#{ taskdir }/' -not \\( -path \"#{taskdir}/.info/*\" -prune \\) -not \\( -path \"#{taskdir}/*.files/*\" -prune \\) -not -name '*.pid' -not -name '*.notify' -not -name '\\.*' \\( -not -type d -o -name '*.files' \\)  2>/dev/null"
+
+                    CMD.cmd(cmd, :pipe => true)
+                  end
           TSV.traverse files, :type => :array, :into => jobs, :_bar => "Finding jobs in #{ taskdir }" do |file|
             _files << file
             if m = file.match(/(.*)\.(info|pid|files)$/)
@@ -167,10 +194,10 @@ module Rbbt
             info[:task] = task
             info[:name] = name
 
-            if File.exists? file
+            if Open.exists? file
               info = info.merge(file_time(file))
               info[:done] = true
-              info[:info_file] = File.exist?(info_file) ? info_file : nil
+              info[:info_file] = Open.exist?(info_file) ? info_file : nil
             else
               info = info.merge({:info_file => info_file, :done => false})
             end

data/lib/rbbt/persist/tsv/adapter.rb

@@ -86,7 +86,6 @@ module Persist
     end
 
     def read_and_close
-      #mutex.synchronize do
       read if closed? or not read?
       res = begin
               yield
@@ -94,7 +93,6 @@ module Persist
               close
             end
       res
-      #end
     end
 
 

data/lib/rbbt/util/open.rb

@@ -207,6 +207,15 @@ module Open
     end
   end
 
+  def self.list_repo_files(dir, sub_path = nil)
+    repo = get_repo_from_dir(dir)
+    files = repo.keys
+    files.reject!{|f| f[0] == "."}
+    return files unless sub_path
+
+    files.select{|file| file.start_with?(sub_path) }
+  end
+
   def self.exists_in_repo(dir, sub_path, content)
     repo = get_repo_from_dir(dir)
     repo.read_and_close do
@@ -219,7 +228,7 @@ module Open
       dir = dir + '/' unless dir.chars[-1] == "/"
 
       begin
-        if file.start_with? dir
+        if file.start_with?(dir) || file == dir[0..-2]
           sub_path = file.to_s[dir.length..-1]
           return [dir, sub_path]
         else 
@@ -376,10 +385,12 @@ module Open
     repo_source = get_repo_from_dir(dir_sub_path_source[0])
     repo_target = get_repo_from_dir(dir_sub_path_target[0])
 
-    repo_source.read_and_close do
-      repo_target.write_and_close do
-        repo_source[dir_sub_path_source[1]] = repo_target[dir_sub_path_target[1]]
-      end
+    content = repo_source.read_and_close do
+      repo_target[dir_sub_path_target[1]]
+    end
+
+    repo_target.write_and_close do
+      repo_source[dir_sub_path_source[1]] = content
     end
 
     return nil
@@ -410,10 +421,15 @@ module Open
     repo_source = get_repo_from_dir(dir_sub_path_source[0])
     repo_target = get_repo_from_dir(dir_sub_path_target[0])
 
+    content = repo_source.read_and_close do
+      repo_target[dir_sub_path_target[1]]
+    end
+
+    repo_target.write_and_close do
+      repo_source[dir_sub_path_source[1]] = content
+    end
+
     repo_source.write_and_close do
-      repo_target.write_and_close do
-        repo_source[dir_sub_path_source[1]] = repo_target[dir_sub_path_target[1]]
-      end
       repo_source.delete dir_sub_path_source[1]
     end
 

data/lib/rbbt/workflow/archive.rb

@@ -9,7 +9,7 @@ class Step
     return if target_dir[workflow][task][name].exists? || File.symlink?(target_dir[workflow][task][name].find)
     Log.debug "Linking #{ path }"
     FileUtils.mkdir_p target_dir[workflow][task] unless target_dir[workflow][task].exists?
-    FileUtils.ln_s path, target_dir[workflow][task][name].find
+    FileUtils.ln_s path, target_dir[workflow][task][name].find if File.exists?(path)
     FileUtils.ln_s path + '.files', target_dir[workflow][task][name].find + '.files' if File.exists?(path + '.files')
     FileUtils.ln_s path + '.info', target_dir[workflow][task][name].find + '.info' if File.exists?(path + '.info')
   end

data/share/rbbt_commands/system/clean

@@ -118,8 +118,8 @@ TSV.traverse jobs, :_bar => "Checking job status" do |file,i|
     status = status.to_s
   end
 
-  if time and File.exists?(file)
-    old = Time.now - File.atime(file)
+  if time and Open.exists?(file)
+    old = Time.now - Open.atime(file)
     if old > time
       status = 'old'
     end

data/share/rbbt_commands/system/status

@@ -67,6 +67,35 @@ def status_msg(status)
           end
   Log.color(color, status.to_s)
 end
+
+def input_msg(file, inputs)
+
+  str = ""
+  job_inputs = Workflow.load_step(file).recursive_inputs.to_hash
+  IndiferentHash.setup(job_inputs)
+
+  inputs.each do |input|
+    value = job_inputs[input]
+    next if  value.nil?
+    value_str = Misc.fingerprint(value)
+    str << "\t#{Log.color :magenta, input}=#{value_str}"
+  end
+  str
+end
+
+def info_msg(info, info_fields)
+
+  str = ""
+  info_fields.each do |field|
+    value = info[field]
+    next if  value.nil?
+    value_str = Misc.fingerprint(value)
+    str << "\t#{Log.color :magenta, field}=#{value_str}"
+  end
+  str
+end
+
+
 puts Log.color(:magenta, "# System report")
 puts
 sort_files = Proc.new do |a,b|
@@ -143,6 +172,21 @@ workflows.sort.each do |workflow,tasks|
       if options[:quick] and i[:done]
         status = 'done'
         str << " #{ status_msg status }"
+        if inputs and inputs.any? 
+          str << input_msg(file, inputs)
+        end
+
+        if info_fields and info_fields.any?
+          info = begin
+                   Open.open(i[:info_file]) do |f|
+                     Step::INFO_SERIALIAZER.load(f)
+                   end
+                 rescue
+                   {:status => :noinfo}
+                 end
+          IndiferentHash.setup(info)
+          str << info_msg(info, info_fields)
+        end
       else
         info = begin
                  Open.open(i[:info_file]) do |f|
@@ -155,7 +199,7 @@ workflows.sort.each do |workflow,tasks|
 
         pid = info[:pid]
         status = info[:status]
-        status = :missing if status == :done and not File.exist? file
+        status = :missing if status == :done and not Open.exist? file
         status = status.to_s
         if status != "done" and pid and not Misc.pid_exists?(pid)
           if File.exist? file
@@ -167,25 +211,12 @@ workflows.sort.each do |workflow,tasks|
         str << " #{ status_msg status }"
         str << " (dirty)" if status == 'done' && Workflow.load_step(file).dirty?
 
-        if inputs and inputs.any? and info[:inputs]
-          job_inputs = Workflow.load_step(file).recursive_inputs.to_hash
-          IndiferentHash.setup(job_inputs)
-
-          inputs.each do |input|
-            value = job_inputs[input]
-            next if  value.nil?
-            value_str = Misc.fingerprint(value)
-            str << "\t#{Log.color :magenta, input}=#{value_str}"
-          end
+        if inputs and inputs.any? 
+          str << input_msg(file, inputs)
         end
 
         if info_fields and info_fields.any?
-          info_fields.each do |field|
-            value = info[field]
-            next if  value.nil?
-            value_str = Misc.fingerprint(value)
-            str << "\t#{Log.color :magenta, field}=#{value_str}"
-          end
+          str << info_msg(info, info_fields)
         end
       end
       str << "; #{pid_msg pid}" unless status == "done"

data/share/rbbt_commands/workflow/prov

@@ -22,6 +22,7 @@ $ rbbt workflow prov <job-result>
 -p--plot* draw the dependency plot into <file.png>
 -i--inputs* List of inputs to print
 -if--info_fields* List of info fields to print
+-t--touch Update modification times to be consistent
 EOF
 
 SOPT.usage if options[:help]
@@ -106,6 +107,18 @@ def report_msg(status, name, path, info = nil)
   str << "\n"
 end
 
+def touch(step)
+  return unless File.exists?(step.path)
+  step.dependencies.each do |dep|
+    next unless Open.exists?(dep.path)
+    touch(dep)
+    if Open.mtime(step.path) < Open.mtime(dep.path) 
+      Log.debug("Updating #{step.path} to #{dep.path}")
+      CMD.cmd("touch -r '#{dep.path}' '#{step.path}'")
+    end
+  end if step.dependencies
+end
+
 def report(step, offset = 0, task = nil)
   info = step.info  || {}
   info[:task_name] = task
@@ -164,6 +177,10 @@ def adjacency(step)
   [id, edges, node_info]
 end
 
+if options[:touch]
+  touch(step)
+end
+
 if options[:plot]
   id, edges, node_info = adjacency(step)
   node_info[id][:color] = 'red'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.23.35
+  version: 5.23.36
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-26 00:00:00.000000000 Z
+date: 2018-11-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake