rbbt-util 5.44.1 → 6.0.4

data/lib/rbbt/util/procpath.rb

@@ -1,4 +1,4 @@
-require 'rbbt/util/cmd'
+require 'scout/cmd'
 module ProcPath
   CMD.tool :procpath do
     'pip install procpath'
@@ -6,11 +6,11 @@ module ProcPath
 
   def self.record(pid, path, options = {})
     IndiferentHash.setup(options)
-    options = Misc.add_defaults options, "interval" => 30
+    options = IndiferentHash.add_defaults options, "interval" => 30
 
     cmd_options = %w(interval recnum reevalnum).inject({}){|acc,k| acc[k] = options[k]; acc}
 
-    Log.debug "ProcPath recording #{pid} in #{path} (#{Misc.fingerprint options})"
+    Log.debug "ProcPath recording #{pid} in #{path} (#{Log.fingerprint options})"
     procpath_thread = Thread.new do 
       begin
         procpath_pid = CMD.cmd_pid(:procpath, "record --database-file '#{path}' '$..children[?(@.stat.pid == #{pid})]'", cmd_options.merge(:nofail => true, :add_option_dashes => true))
@@ -28,7 +28,7 @@ module ProcPath
 
   def self.plot(path, output, options = {})
     IndiferentHash.setup(options)
-    options = Misc.add_defaults options, "query-name" => 'rss', 'epsilon' => 0.5, "moving-average-window" => 10
+    options = IndiferentHash.add_defaults options, "query-name" => 'rss', 'epsilon' => 0.5, "moving-average-window" => 10
 
     cmd_options = %w(query-name epsilon monitor-average-window title logarithmic after before custom-query-file custom-value-expr).inject({}){|acc,k| acc[k] = options[k]; acc}
     CMD.cmd_log(:procpath, "plot --database-file '#{path}' --plot-file '#{output}' ", cmd_options.merge(:nofail => true, :add_option_dashes => true))
@@ -36,10 +36,10 @@ module ProcPath
 
   def self.monitor(pid, path)
     database, options_str = path.split("#")
-    options = options_str.nil? ? {} : Misc.string2hash(options_str)
+    options = options_str.nil? ? {} : IndiferentHash.string2hash(options_str)
 
     database = File.expand_path database
-    Log.low "ProcPath monitor #{pid} in #{database} (#{Misc.fingerprint options})"
+    Log.low "ProcPath monitor #{pid} in #{database} (#{Log.fingerprint options})"
 
     ProcPath.record(pid, database + '.sqlite3', options)
     ProcPath.plot(database + '.sqlite3', database + '.cpu.svg', options.merge("query-name" => 'cpu'))

data/lib/rbbt/util/python/paths.rb

@@ -0,0 +1,27 @@
+module RbbtPython
+  class << self
+    attr_accessor :paths
+    def paths
+      @paths ||= []
+    end
+  end
+
+  def self.add_path(path)
+    self.paths << path
+  end
+
+  def self.add_paths(paths)
+    self.paths.concat paths
+  end
+
+  def self.process_paths
+    RbbtPython.run_direct 'sys' do
+      RbbtPython.paths.each do |path|
+        sys.path.append path
+      end
+      nil
+    end
+  end
+
+  add_paths(Rbbt.python.find_all)
+end

data/lib/rbbt/util/python/run.rb

@@ -0,0 +1,115 @@
+module RbbtPython
+  class << self
+    attr_accessor :thread
+  end
+
+  def self.thread
+    @thread ||= defined?(@thread) ? @thread : nil
+  end
+
+  MUTEX= Mutex.new
+  QUEUE_IN ||= Queue.new
+  QUEUE_OUT ||= Queue.new
+  def self.synchronize(&block)
+    MUTEX.synchronize &block
+  end
+
+  def self.init_thread
+    if defined?(self.thread) && (self.thread && ! self.thread.alive?)
+      Log.warn "Reloading RbbtPython thread"
+      self.thread.join
+      self.thread = nil
+    end
+
+    self.thread ||= Thread.new do
+      require 'pycall'
+      RbbtPython.process_paths
+      begin
+        while block = QUEUE_IN.pop
+          break if block == :stop
+          res = 
+            begin
+              module_eval(&block)
+            rescue Exception
+              Log.exception $!
+              raise $!
+            end
+
+          QUEUE_OUT.push res
+        end
+      rescue Exception
+        Log.exception $!
+        raise $!
+      ensure
+        PyCall.finalize if PyCall.respond_to?(:finalize)
+      end
+    end
+  end
+
+  def self.run_in_thread(&block)
+    self.synchronize do
+      init_thread
+      QUEUE_IN.push block
+      QUEUE_OUT.pop
+    end
+  end
+
+  def self.stop_thread
+    self.synchronize do
+      QUEUE_IN.push :stop
+    end if self.thread && self.thread.alive?
+    self.thread.join if self.thread
+  end
+
+  def self.run_direct(mod = nil, imports = nil, &block)
+    if mod
+      if Hash === imports
+        pyimport mod, **imports
+      elsif imports.nil?
+        pyimport mod 
+      else
+        pyfrom mod, :import => imports
+      end
+    end 
+
+    module_eval(&block)
+  end
+
+  def self.run_threaded(mod = nil, imports = nil, &block)
+    run_in_thread do
+        if Hash === imports
+          pyimport mod, **imports
+        elsif imports.nil?
+          pyimport mod 
+        else
+          pyfrom mod, :import => imports
+        end
+    end if mod
+
+    run_in_thread(&block)
+  end
+
+  def self.run_simple(mod = nil, imports = nil, &block)
+    self.synchronize do
+      RbbtPython.process_paths
+      run_direct(mod, imports, &block)
+    end
+  end
+
+  class << self
+    alias run run_simple
+  end
+
+  def self.run_log(mod = nil, imports = nil, severity = 0, severity_err = nil, &block)
+    Log.trap_std("Python STDOUT", "Python STDERR", severity, severity_err) do
+      run(mod, imports, &block)
+    end
+  end
+
+  def self.run_log_stderr(mod = nil, imports = nil, severity = 0, &block)
+    Log.trap_stderr("Python STDERR", severity) do
+      run(mod, imports, &block)
+    end
+  end
+
+end

data/lib/rbbt/util/python/script.rb

@@ -0,0 +1,110 @@
+module RbbtPython
+  def self.ruby2python(object)
+    case object
+    when Float::INFINITY
+      "inf"
+    when nil
+      "None"
+    when ":NA"
+      "None"
+    when Symbol
+      "#{ object }"
+    when String
+      object = object.dup if Path === object
+      object[0] == ":" ? object[1..-1] : "'#{ object }'"
+    when Numeric
+      object
+    when TrueClass
+      "True"
+    when FalseClass
+      "False"
+    when Array
+      "[#{object.collect{|e| ruby2python(e) } * ", "}]"
+    when Hash
+      "{" << object.collect{|k,v| [ruby2python(k.to_s), ruby2python(v)] * ":"} * ", " << "}"
+    else
+      raise "Type of object not known: #{ object.inspect }"
+    end
+  end
+
+  def self.load_script_variables(variables = {})
+    code = "# Variables\nimport rbbt\n"
+    tmp_files = []
+    variables.each do |name,value|
+      case value
+      when TSV
+        tmp_file = TmpFile.tmp_file
+        tmp_files << tmp_file
+        Open.write(tmp_file, value.to_s)
+        code << "#{name} = rbbt.tsv('#{tmp_file}')" << "\n"
+      else
+        code << "#{name} = #{RbbtPython.ruby2python(value)}" << "\n"
+      end
+    end
+
+    [code, tmp_files]
+  end
+
+  def self.save_script_result_pickle(file)
+    <<-EOF
+
+# Save
+try: result
+except NameError: result = None
+if result is not None:
+  import pickle
+  file = open('#{file}', 'wb')
+  # dump information to that file
+  pickle.dump(result, file)
+    EOF
+  end
+
+  def self.load_pickle(file)
+    require 'python/pickle'
+    Log.debug ("Loading pickle #{file}")
+    Python::Pickle.load_file(file)
+  end
+
+  def self.save_script_result_json(file)
+    <<-EOF
+
+# Save
+try: result
+except NameError: result = None
+if result is not None:
+  import json
+  file = open('#{file}', 'w', encoding='utf-8')
+  # dump information to that file
+  file.write(json.dumps(result))
+  file.flush
+  file.close
+    EOF
+  end
+
+  def self.load_json(file)
+    JSON.load_file(file)
+  end
+
+  class << self
+    alias save_script_result save_script_result_pickle
+    alias load_result load_pickle
+  end
+
+  def self.script(text, variables = {})
+    if variables.any?
+      variable_definitions, tmp_files = load_script_variables(variables)
+      text = variable_definitions + "\n# Script\n" + text
+    end
+
+    TmpFile.with_file do |tmp_file|
+      text += save_script_result(tmp_file)
+      Log.debug "Running python script:\n#{text.dup}"
+      path_env = RbbtPython.paths * ":"
+      CMD.cmd_log("env PYTHONPATH=#{path_env} python", {in: text})
+      tmp_files.each{|file| Open.rm_rf file } if tmp_files
+      if Open.exists?(tmp_file)
+        load_result(tmp_file)
+      end
+    end
+  end
+end

data/lib/rbbt/util/python/util.rb

@@ -10,7 +10,7 @@ module RbbtPython
 
   def self.tsv2df(tsv)
     df = nil
-    RbbtPython.run 'pandas' do
+    RbbtPython.run_direct 'pandas' do
       df = pandas.DataFrame.new(tsv.values, columns: tsv.fields, index: tsv.keys)
       df.columns.name = tsv.key_field
     end
@@ -21,9 +21,9 @@ module RbbtPython
     options = Misc.add_defaults options, :type => :list
     IndiferentHash.setup options
     tsv = TSV.setup({}, options)
-    tsv.key_field = options[:key_field] || tuple.columns.name
+    tsv.key_field = options[:key_field] || tuple.columns.name.to_s
     tsv.fields = py2ruby_a(tuple.columns.values)
-    keys = tuple.index.values
+    keys = py2ruby_a(tuple.index.values)
     PyCall.len(tuple.index).times do |i|
       k = keys[i]
       tsv[k] = py2ruby_a(tuple.values[i])

data/lib/rbbt/util/python.rb

@@ -1,6 +1,10 @@
 require 'rbbt-util'
-require 'pycall/import'
+require 'rbbt/util/python/paths'
+require 'rbbt/util/python/run'
 require 'rbbt/util/python/util'
+require 'rbbt/util/python/script'
+
+require 'pycall/import'
 
 module RbbtPython
   extend PyCall::Import
@@ -15,41 +19,19 @@ module RbbtPython
     end
   end
 
-  def self.script(text, options = {})
-    Log.debug "Running python script:\n#{text.dup}"
-    text = StringIO.new text unless IO === text
-    CMD.cmd_log(:python, options.merge(:in => text))
-  end
-
-  def self.add_path(path)
-    begin
-      self.run 'sys' do
-        sys.path.append path
-      end
-    rescue
-      raise RbbtPythonException, 
-        "Could not add path #{Misc.fingerprint path} to python sys: " + $!.message
-    end
-  end
-
-  def self.add_paths(paths)
-    self.run 'sys' do
-      paths.each do |path|
-        sys.path.append path
-      end
-    end
-  end
-
   def self.init_rbbt
     if ! defined?(@@__init_rbbt_python) || ! @@__init_rbbt_python
-      Log.debug "Loading python 'rbbt' module into pycall RbbtPython module"
-      RbbtPython.add_paths(Rbbt.python.find_all)
-      RbbtPython.pyimport("rbbt")
+      RbbtPython.process_paths
+      res = RbbtPython.run do
+        Log.debug "Loading python 'rbbt' module into pycall RbbtPython module"
+        pyimport("rbbt")
+      end
       @@__init_rbbt_python = true
     end
   end
 
   def self.import_method(module_name, method_name, as = nil)
+    init_rbbt
     RbbtPython.pyfrom module_name, import: method_name
     RbbtPython.method(method_name)
   end
@@ -57,11 +39,17 @@ module RbbtPython
   def self.call_method(module_name, method_name, *args)
     RbbtPython.import_method(module_name, method_name).call(*args)
   end
-
-  def self.get_class(module_name, class_name)
+  
+  def self.get_module(module_name)
+    init_rbbt
     save_module_name = module_name.to_s.gsub(".", "_")
     RbbtPython.pyimport(module_name, as: save_module_name)
-    RbbtPython.send(save_module_name).send(class_name)
+    RbbtPython.send(save_module_name)
+  end
+
+  def self.get_class(module_name, class_name)
+    mod = get_module(module_name)
+    mod.send(class_name)
   end
 
   def self.class_new_obj(module_name, class_name, args={})
@@ -73,7 +61,6 @@ module RbbtPython
   end
 
   def self.iterate_index(elem, options = {})
-    iii :interate_index
     bar = options[:bar]
 
     len = PyCall.len(elem)
@@ -152,54 +139,6 @@ module RbbtPython
     acc
   end
 
-  def self.run(mod = nil, imports = nil, &block)
-    if mod
-      if Hash === imports
-        pyimport mod, **imports
-      elsif imports.nil?
-        pyimport mod 
-      else
-        pyfrom mod, :import => imports
-      end
-    end
-
-    module_eval(&block)
-  end
-
-  def self.run_log(mod = nil, imports = nil, severity = 0, severity_err = nil, &block)
-    if mod
-      if imports == "*" || imports == ["*"]
-        pyfrom mod
-      elsif Array === imports
-        pyfrom mod, :import => imports
-      elsif Hash === imports
-        pyimport mod, imports
-      else
-        pyimport mod 
-      end
-    end
-
-    Log.trap_std("Python STDOUT", "Python STDERR", severity, severity_err) do
-      module_eval(&block)
-    end
-  end
-
-  def self.run_log_stderr(mod = nil, imports = nil, severity = 0, &block)
-    if mod
-      if Array === imports
-        pyfrom mod, :import => imports
-      elsif Hash === imports
-        pyimport mod, imports
-      else
-        pyimport mod 
-      end
-    end
-
-    Log.trap_stderr("Python STDERR", severity) do
-      module_eval(&block)
-    end
-  end
-
   def self.new_binding
     Binding.new
   end
@@ -209,3 +148,5 @@ module RbbtPython
     binding.instance_exec *args, &block
   end
 end
+
+RbbtPython.add_path Rbbt.python.find(:lib)