rbbt-util 5.21.137 → 5.22.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ba7c142553e810f2dbced269f0aed2a3b52051cb
-  data.tar.gz: b58daa013d65a07b80ecf4e7de2cea5265ab387b
+  metadata.gz: 8d7a4ccc9fa31a719cf11e4b0c5341f610a05fbe
+  data.tar.gz: 6f001470b29bab8072d156f18039a13823176f89
 SHA512:
-  metadata.gz: a94c595521d98f0f8bfaf15eac02e38f3a9603a65d77d29d34ed6e090b014ec6b269ff9606e6ed62e8a1de47d3b52d5cea210d79e9dba416802686a225be6cbb
-  data.tar.gz: 3b647173c446127c0f29584f7097b39a4f144aef923b88b2d6ccef9f262ed24b070f28e722326d45e0e382e9e887034b07d3b33fe243a2d314d0af5997ababce
+  metadata.gz: 553306c45688f65865f87be18554a92d90a954d1f3402b6d657300c253d5ce6c21183fddd4fe60e99ef867da11338256165b3b30f1598ab4a15fe4f01d2e2be4
+  data.tar.gz: 8952ede274862e68aef03b6117560335fb7a552a146a1b36678ebe8bf606dcfc7483d1c060a3501361a6680d930b9c6628601e476e6b000fbd4f93024aad0157

data/bin/rbbt

@@ -27,6 +27,7 @@ end
 
 require 'rbbt'
 require 'rbbt/util/simpleopt'
+require 'rbbt/util/config'
 
 Log.nocolor = true if ARGV.include? "--nocolor"
 
@@ -49,6 +50,7 @@ $ rbbt <command> <subcommand> ... -a --arg1 --arg2='value' --arg3 'another-value
 --dump_mem*             #{Log.color :yellow, "Dump strings in memory each second into file"}
 -nolock--no_lock_id            #{Log.color :yellow, "Do not track lockfiles with ids (prevent stale file handlers for high-througput and high-concurrency)"}
 -ji--jobname_as_inputs  #{Log.color :yellow, "Use inputs as part of the jobname in workflows instead of digesting them"}
+-ck--config_keys*       #{Log.color :yellow, "Override some config keys"}
 EOF
 
 if options[:jobname_as_inputs]
@@ -89,6 +91,18 @@ if mem_dump = options.delete(:dump_mem)
   Rbbt.dump_memory(mem_dump, String)
 end
 
+if options[:config_keys]
+  options[:config_keys].split(",").each do |config|
+    key, value, *tokens = config.split(/\s/)
+    tokens = ['key:' << key << '::0'] if tokens.empty?
+    tokens = tokens.collect do |tok|
+      tok, _sep, prio = tok.partition("::")
+      prio = "0" if prio.nil? or prio.empty?
+      [tok, prio] * "::"
+    end
+    Rbbt::Config.set({key => value}, *tokens)
+  end
+end
 
 if options.delete(:update_persist)
   ENV["RBBT_UPDATE_TSV_PERSIST"] = "true"

data/lib/rbbt-util.rb

@@ -15,3 +15,5 @@ TmpFile.tmpdir       = Rbbt.tmp.find :user
 require 'rbbt/util/cmd'
 require 'rbbt/tsv'
 
+require 'rbbt/util/config'
+

data/lib/rbbt/persist.rb

@@ -316,6 +316,8 @@ module Persist
 
         save_file(path, type, res, lockfile)
 
+        Open.notify_write(path)
+
         return path if persist_options[:no_load]
 
         res

data/lib/rbbt/util/config.rb

@@ -0,0 +1,109 @@
+require 'rbbt-util'
+require 'rbbt/resource/path'
+
+module Rbbt::Config
+
+  CACHE = IndiferentHash.setup({})
+
+  def self.add_entry(key, value, tokens)
+    CACHE[key.to_s] ||= [] 
+    CACHE[key.to_s] << [tokens, value]
+  end
+
+  def self.load_file(file)
+    Log.debug "Loading file: #{ file }"
+    TSV.traverse file, :type => :array do |line|
+      next if line =~ /^#/
+      key, value, *tokens = line.split(/\s/)
+
+      self.add_entry(key, value, tokens) if key
+    end
+  end
+
+  def self.load_config
+    Rbbt.etc.config.find_all.each do |file|
+      self.load_file(file)
+    end
+  end
+
+
+  def self.set(values, *tokens)
+    values.each do |key,value|
+      add_entry key, value, tokens
+    end
+  end
+
+  def self.token_priority(token)
+    token, _sep, priority = token.to_s.partition("::")
+
+    if priority.nil? || priority.empty?
+      type, _sep, rest = token.partition(":")
+      priority = case type
+                 when "workflow"
+                   4
+                 when "task"
+                   3
+                 when "file"
+                   2
+                 when "line"
+                   1
+                 when "key"
+                   20
+                 else
+                   10
+                 end
+    else
+      priority = priority.to_i
+    end
+
+    [token, priority]
+  end
+
+  def self.match(entries, token)
+    priorities = {}
+    entries.each do |tokens, value|
+      best_prio = nil
+      tokens.each do |tok|
+        tok, prio = token_priority tok
+        best_prio = prio if best_prio.nil? or best_prio > prio
+        next if prio > best_prio
+        next unless tok == token
+        priorities[prio] ||= []
+        priorities[prio] << value
+      end
+    end
+    priorities
+  end
+
+  def self.get(key, *tokens)
+    tokens = tokens.flatten
+    file, _sep, line = caller.reject{|l| 
+      l =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
+        l =~ /rbbt\/resource\/path\.rb/ or
+        l =~ /rbbt\/util\/misc\.rb/ or
+        l =~ /progress-monitor\.rb/ 
+    }.first.partition(":")
+
+    File.expand_path(file)
+
+    tokens << ("file:" << file)
+    tokens << ("line:" << file << ":" << line)
+
+    entries = CACHE[key.to_s]
+    priorities = {}
+    tokens = tokens + ["key:" << key.to_s]
+    tokens.each do |token|
+      token_prio = match entries, token.to_s
+      token_prio.each do |prio, values|
+        priorities[prio] ||= []
+        priorities[prio].concat(values)
+      end
+    end
+
+    return nil if priorities.empty?
+
+    priorities.sort_by{|p,v| p}.first.last.first
+  end
+
+  self.load_config
+end

data/lib/rbbt/util/misc/system.rb

@@ -97,5 +97,4 @@ end
     return true if String === string and string.length < 265 and File.exist? string
     return false
   end
-
 end

data/lib/rbbt/workflow/accessor.rb

@@ -532,6 +532,23 @@ class Step
     end
     provenance
   end
+
+  def config(key, *tokens)
+    options = tokens.pop if Hash === tokens.last
+    default = options[:default] if options
+
+    new_tokens = []
+    if workflow
+      workflow_name = workflow.to_s
+      new_tokens << ("workflow:" << workflow_name)
+      new_tokens << ("task:" << workflow_name << "#" << task_name.to_s)
+    end
+    new_tokens << ("task:" << task_name.to_s)
+
+    value = Rbbt::Config.get(key, tokens + new_tokens)
+
+    value || default
+  end
 end
 
 module Workflow

data/lib/rbbt/workflow/step/run.rb

@@ -21,6 +21,7 @@ class Step
   def resolve_input_steps
     step = false
     pos = 0
+    input_options = workflow.task_info(task_name)[:input_options]
     new_inputs = inputs.collect do |i| 
       begin
         if Step === i
@@ -38,7 +39,11 @@ class Step
             if (task.input_options[task.inputs[pos]] || {})[:stream]
               TSV.get_stream i
             else
-              i.load
+              if (task.input_options[task.inputs[pos]] || {})[:nofile]
+                i.path
+              else
+                i.load
+              end
             end
           elsif i.streaming? and (task.input_options[task.inputs[pos]] || {})[:stream]
             TSV.get_stream i
@@ -47,7 +52,11 @@ class Step
             if (task.input_options[task.inputs[pos]] || {})[:stream]
               TSV.get_stream i
             else
-              i.load
+              if (task.input_options[task.inputs[pos]] || {})[:nofile]
+                i.path
+              else
+                i.load
+              end
             end
           end
         else

data/test/rbbt/util/test_config.rb

@@ -0,0 +1,18 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
+require 'rbbt/util/config'
+
+class TestConfig < Test::Unit::TestCase
+  def setup
+    Rbbt::Config.set({:cpus => 30}, :test_config, :test)
+    Rbbt::Config.set({:cpus => 5}, "slow::2", :test)
+  end
+
+  def test_simple
+    assert_equal 30, Rbbt::Config.get(:cpus, :test_config)
+  end
+
+  def test_prio
+    assert_equal 5, Rbbt::Config.get(:cpus, :slow, :test)
+  end
+end
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.21.137
+  version: 5.22.0
 platform: ruby
 authors:
 - Miguel Vazquez
@@ -274,6 +274,7 @@ files:
 - lib/rbbt/util/concurrency/processes/socket_old.rb
 - lib/rbbt/util/concurrency/processes/worker.rb
 - lib/rbbt/util/concurrency/threads.rb
+- lib/rbbt/util/config.rb
 - lib/rbbt/util/docker.rb
 - lib/rbbt/util/excel2tsv.rb
 - lib/rbbt/util/filecache.rb
@@ -474,6 +475,7 @@ files:
 - test/rbbt/util/test_cmd.rb
 - test/rbbt/util/test_colorize.rb
 - test/rbbt/util/test_concurrency.rb
+- test/rbbt/util/test_config.rb
 - test/rbbt/util/test_excel2tsv.rb
 - test/rbbt/util/test_filecache.rb
 - test/rbbt/util/test_log.rb
@@ -530,6 +532,7 @@ test_files:
 - test/rbbt/util/R/test_plot.rb
 - test/rbbt/util/R/test_eval.rb
 - test/rbbt/util/R/test_model.rb
+- test/rbbt/util/test_config.rb
 - test/rbbt/util/test_log.rb
 - test/rbbt/util/test_simpleDSL.rb
 - test/rbbt/util/log/test_progress.rb