rbbt-util 5.26.78 → 5.26.79

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fa385d029073f92c7d003f8dd139ffd76f256cf50e8b6f9eb48146b8624c1d0f
-  data.tar.gz: 1e9d23cd39e36dcebf0d2e61f269593f031cb7a8c8b4ff900993aa10fd7b11a6
+  metadata.gz: d7b3f571fcdf2bf96368174818464415b1282a2d08e71fda696d325b1d2a84f4
+  data.tar.gz: f0403774806dc37731e0cfe77bdbdc7908bc6aee9015b657871c786bbd1f849c
 SHA512:
-  metadata.gz: 3f7f0f8aef95c5f79832f18c741f6751e24ff78aa59e739fa44b38d5ae88c5be7f77ae1c293fb07c025e6ed9ed6030354b8b279ed379a01bf970a0efb9d5bd99
-  data.tar.gz: 02070acbc4a9ab3925f0bbf85bc2b07069241d46e8ffdf72085e09bd952b3e24d73eceed255c267d2b1de9ed63fb214142413a9d3e0badb3f19b409c5880a4b6
+  metadata.gz: '0988f55f748f4917507a819e0bc367bb307893cdc18654bd3af040f4a4f1ade4b64656802c388bfeb0eb9e297e20e1392db869ff4fe9d81871eaccfbf2893e3a'
+  data.tar.gz: e0e865fc6e92866da62eef19bdbdb7fe3dd4036e6fc3f730c01f6823dd5d970477c568d7451bdeab5be76ace260552a19be2d04661a33da79630750fd4fb3143

data/etc/app.d/base.rb

@@ -3,13 +3,17 @@ register Sinatra::MultiRoute
 
 register Sinatra::RbbtRESTMain
 register Sinatra::RbbtRESTEntity
-register Sinatra::RbbtRESTFileServer # Remove to prevent serving files
 register Sinatra::RbbtRESTKnowledgeBase
 register Sinatra::RbbtRESTWorkflow
 
 helpers Sinatra::RbbtToolHelper
 helpers Sinatra::RbbtMiscHelpers
 
+if ENV["RBBT_REST_FILE_SERVER"] == 'true'
+  Log.high "Activating File Server"
+  register Sinatra::RbbtRESTFileServer 
+end
+
 #{{{ SESSIONS
 
 use Rack::Session::Cookie, :key => 'rack.session',

data/lib/rbbt/resource/path.rb

@@ -236,6 +236,7 @@ module Path
                          sub('{BASENAME}', File.basename(self)).
                          sub('{PATH}', self).
                          sub('{LIBDIR}', libdir).
+                         sub('{RESOURCE}', resource.to_s).
                          sub('{REMOVE}/', '').
                          sub('{REMOVE}', '')
 

data/lib/rbbt/resource/util.rb

@@ -25,7 +25,7 @@ module Path
   SLASH = "/"[0]
   DOT = "."[0]
   def located?
-    self.byte(0) == SLASH or (self.byte(0) == DOT and self.byte(1) == SLASH)
+    self.byte(0) == SLASH || (self.byte(0) == DOT && self.byte(1) == SLASH) || (resource != Rbbt && (Open.remote?(self) || Open.ssh?(self)))
   end
 end
 

data/lib/rbbt/resource.rb

@@ -297,6 +297,7 @@ url='#{url}'
         end
       end
     end
+    nil
   end
 end
 

data/lib/rbbt/workflow/step.rb

@@ -83,6 +83,20 @@ class Step
     end
   end
 
+  def load_dependencies_from_info
+    @dependencies = (self.info[:dependencies] || []).collect do |task,name,dep_path|
+      if Open.exists?(dep_path) || Open.exists?(dep_path + '.info')
+        Workflow._load_step dep_path
+      else
+        next if FalseClass === relocated
+        new_path = Workflow.relocate(path, dep_path)
+        relocated = true if Open.exists?(new_path) || Open.exists?(new_path + '.info')
+        Workflow._load_step new_path
+      end
+    end.compact
+    @relocated = relocated
+  end
+
 
   def inputs
     return @inputs if NamedArray === @inputs

data/lib/rbbt/workflow/util/archive.rb

@@ -49,7 +49,7 @@ class Step
     end
   end
 
-  def self.job_files_for_archive(files)
+  def self.job_files_for_archive(files, relocate = false)
     job_files = Set.new
 
     jobs = files.collect do |file|  
@@ -71,10 +71,15 @@ class Step
       seen = Set.new
       while deps.any?
         path = deps.shift
+
         dep = Step.new path
         seen << dep.path
-        dep.info[:dependencies].each do |task, name, path|
-          dep = Step.new path
+
+        dep.relocated = !!relocate
+
+        dep.load_dependencies_from_info
+
+        dep.dependencies.each do |dep|
           next if seen.include? dep.path
           deps << dep.path
           rec_dependencies << dep.path
@@ -83,20 +88,21 @@ class Step
 
       rec_dependencies.each do |path|
         next unless File.exists?(path)
+        dep = Step.new path
+        job_files << dep.path
         job_files << dep.files_dir if Dir.glob(dep.files_dir + '/*').any?
         job_files << dep.info_file if File.exists?(dep.info_file)
-        job_files << path
       end
     end
 
     job_files.to_a
   end
 
-  def self.archive(files, target = nil)
+  def self.archive(files, target = nil, relocate = true)
     target = self.path + '.tar.gz' if target.nil?
     target = File.expand_path(target) if String === target
 
-    job_files = job_files_for_archive files
+    job_files = job_files_for_archive files, relocate
     TmpFile.with_file do |tmpdir|
       job_files.each do |file|
         Step.link_job file, tmpdir
@@ -112,4 +118,117 @@ class Step
       Log.debug "Archive finished at: #{target}"
     end
   end
+
+  def self.migrate(path, search_path, options = {})
+
+    resource=Rbbt
+
+    other_rsync_args = options[:rsync]
+    relocate = options[:relocate]
+
+    paths = if options[:source]
+              SSHClient.run(options[:source], <<-EOF).split("\n")
+require 'rbbt-util'
+require 'rbbt/workflow'
+
+path = "#{path}"
+relocate = #{ relocate.to_s }
+if File.exists?(path)
+  path = #{resource.to_s}.identify(path)
+else
+  path = Path.setup(path)
+end
+files = path.glob_all
+if #{options[:recursive].to_s == 'true'}
+  files = Step.job_files_for_archive(files, relocate)
+end
+puts files * "\n"
+              EOF
+            else
+              if File.exists?(path)
+                path = resource.identify(path)
+              else
+                path = Path.setup(path)
+              end
+              files = path.glob_all
+              if options[:recursive]
+                files = Step.job_files_for_archive(files)
+              end
+              files
+            end
+
+    target = if options[:target] 
+               target = SSHClient.run(options[:target], <<-EOF).split("\n").first
+require 'rbbt-util'
+path = "var/jobs"
+resource = #{resource.to_s}
+search_path = "#{search_path}"
+puts resource[path].find(search_path)
+               EOF
+             else
+               resource['var/jobs'].find(search_path)
+             end
+
+    subpath_files = {}
+    paths.each do |source|
+      parts = source.split("/")
+      subpath = parts[0..-4] * "/"
+      source = parts[-3..-1] * "/"
+      subpath_files[subpath] ||= []
+      subpath_files[subpath] << source
+    end
+
+    subpath_files.each do |subpath, files|
+      if options[:target]
+        CMD.cmd("ssh #{options[:target]} mkdir -p '#{File.dirname(target)}'")
+      else
+        Open.mkdir File.dirname(target)
+      end
+
+      if options[:source]
+        source = [options[:source], subpath] * ":"
+      else
+        source = subpath
+      end
+      target = [options[:target], target] * ":" if options[:target]
+
+      files_and_dirs = Set.new(files )
+      files.each do |file|
+        parts = file.split("/")[0..-2]
+        while parts.any?
+          files_and_dirs << parts * "/"
+          parts.pop
+        end
+      end
+
+      TmpFile.with_file(files_and_dirs.to_a * "\n") do |tmp_include_file|
+        test_str = options[:test] ? '-nv' : ''
+
+        includes_str = "--include-from='#{tmp_include_file}'"
+        cmd = "rsync -atAXmPL --progress #{test_str} --include-from='#{tmp_include_file}' --exclude='*' #{source}/ #{target}/ #{other_rsync_args}"
+
+        cmd << " && rm -Rf #{source}" if options[:delete]
+
+        if options[:print]
+          ppp Open.read(tmp_include_file)
+          puts cmd 
+        else
+          CMD.cmd_log(cmd)
+        end
+      end
+    end
+  end
+
+  def self.purge(path, relocate = false)
+    path = [path] if String === path
+    job_files = job_files_for_archive path, relocate
+
+    job_files.each do |file|
+      begin
+        Open.rm_rf file if Open.exists?(file)
+      rescue
+        Log.warn "Could not erase '#{file}': #{$!.message}"
+      end
+    end
+  end
 end

data/share/rbbt_commands/app/start

@@ -17,6 +17,7 @@ $ rbbt app start [options] <app_name>
 -p--port* TCP port
 -s--server* Server type:  thin, webrick, unicorn, etc
 -f--finder Start server with finder functionality
+-fs--file_server Activate file serving for resources
 -R--Rserve_session* Rserve session to use, otherwise start new one
 -wd--workdir* Change the working directory of the workflow
 --views* Directory with view templates
@@ -57,9 +58,11 @@ app_dir = app_dir[app]
 server = options[:server] || 'puma'
 Misc.in_dir(app_dir) do
   require 'rack'
-  ENV["RBBT_FINDER"] = "true" if options.include?(:finder)
+  ENV["RBBT_FINDER"] = "true" if options[:finder]
   ENV["RACK_ENV"] = options[:environment] if options.include?(:environment)
   ENV["RBBT_VIEWS_DIR"] = options[:views] if options.include?(:views)
+  ENV["RBBT_REST_FILE_SERVER"] = "true" if  options[:file_server]
+
 
   if options[:stream]
     #raise "No streaming available for any server other than puma" unless options[:server] =~ /^puma/

data/share/rbbt_commands/migrate_job

@@ -32,101 +32,6 @@ if options[:help]
   exit 0
 end
 
-excludes = %w(.save .crap .source tmp filecache open-remote)
-excludes += (options[:exclude] || "").split(/,\s*/)
-excludes_str = excludes.collect{|s| "--exclude '#{s}'" } * " "
-
-test_str = options[:test] ? '-nv' : ''
-
 path, search_path, _sep, *other = ARGV
 
-resource = Rbbt
-
-paths = if options[:source]
-          SSHClient.run(options[:source], <<-EOF).split("\n")
-require 'rbbt-util'
-require 'rbbt/workflow'
-
-path = "#{path}"
-if File.exists?(path)
-  path = #{resource.to_s}.identify(path)
-else
-  path = Path.setup(path)
-end
-files = path.glob_all
-if #{options[:recursive].to_s == 'true'}
-  files = Step.job_files_for_archive(files)
-end
-puts files * "\n"
-          EOF
-        else
-          if File.exists?(path)
-            path = resource.identify(path)
-          else
-            path = Path.setup(path)
-          end
-          files = path.glob_all
-          if options[:recursive]
-            files = Step.job_files_for_archive(files)
-          end
-          files
-        end
-
-target = if options[:target] 
-           target = SSHClient.run(options[:target], <<-EOF).split("\n").first
-require 'rbbt-util'
-path = "var/jobs"
-resource = #{resource.to_s}
-search_path = "#{search_path}"
-puts resource[path].find(search_path)
-           EOF
-         else
-           resource['var/jobs'].find(search_path)
-         end
-
-subpath_files = {}
-paths.each do |source|
-  parts = source.split("/")
-  subpath = parts[0..-4] * "/"
-  source = parts[-3..-1] * "/"
-  subpath_files[subpath] ||= []
-  subpath_files[subpath] << source
-end
-
-subpath_files.each do |subpath, files|
-  if options[:target]
-    CMD.cmd("ssh #{options[:target]} mkdir -p '#{File.dirname(target)}'")
-  else
-    Open.mkdir File.dirname(target)
-  end
-
-  if options[:source]
-    source = [options[:source], subpath] * ":"
-  else
-    source = subpath
-  end
-  target = [options[:target], target] * ":" if options[:target]
-
-  files_and_dirs = Set.new(files )
-  files.each do |file|
-    parts = file.split("/")[0..-2]
-    while parts.any?
-      files_and_dirs << parts * "/"
-      parts.pop
-    end
-  end
-
-  TmpFile.with_file(files_and_dirs.to_a * "\n") do |tmp_include_file|
-    includes_str = "--include-from='#{tmp_include_file}'"
-    cmd = "rsync -atAXmPL --progress #{test_str} --include-from='#{tmp_include_file}' --exclude='*' #{source}/ #{target}/ #{other * " "}"
-
-    cmd << " && rm -Rf #{source}" if options[:delete]
-
-    if options[:print]
-      ppp Open.read(tmp_include_file)
-      puts cmd 
-    else
-      CMD.cmd_log(cmd)
-    end
-  end
-end
+Step.migrate(path, search_path, options)

data/share/rbbt_commands/purge_job

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+require 'rbbt/workflow'
+require 'rbbt/workflow/remote/ssh/get'
+
+$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Remove a job and its dependencies
+
+$ rbbt purge [options] <job_path> 
+
+You can specify extra rsync options after --
+
+-h--help Print this help
+-t--test Do a verbose dry run
+-r--relocate Include relocated dependencies
+
+EOF
+if options[:help]
+  if defined? rbbt_usage
+    rbbt_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+path, search_path, _sep, *other = ARGV
+
+raise ParameterException, "No path given" if path.nil? 
+
+Step.purge(path, options[:relocate])

data/share/rbbt_commands/workflow/server

@@ -23,6 +23,7 @@ $ rbbt workflow server [options] <Workflow>
 -R--requires* Require a list of files
 --views* Directory with view templates
 --stream Activate streaming of workflow tasks
+-fs--file_server Activate file serving for resources
 --export_all Export all workflow tasks (use with caution!)
 --export* Export workflow tasks (asynchronous)
 --export_asynchronous* Export workflow tasks as asynchronous
@@ -81,9 +82,10 @@ TmpFile.with_file do |app_dir|
     Open.write(app_dir.etc.requires.find, requires.split(/,\s*/)*"\n") if requires and not requires.empty? 
 
     require 'rack'
-    ENV["RBBT_FINDER"] = "true" if options.include?(:finder)
+    ENV["RBBT_FINDER"] = "true" if options[:finder]
     ENV["RACK_ENV"] = options[:environment] if options.include?(:environment)
     ENV["RBBT_VIEWS_DIR"] = options[:views] if options.include?(:views)
+    ENV["RBBT_REST_FILE_SERVER"] = "true" if  options[:file_server]
 
     if options[:export_all]
       ENV["RBBT_WORKFLOW_EXPORT_ALL"] = 'true'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.26.78
+  version: 5.26.79
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-09-25 00:00:00.000000000 Z
+date: 2019-10-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -367,6 +367,7 @@ files:
 - share/rbbt_commands/log
 - share/rbbt_commands/migrate
 - share/rbbt_commands/migrate_job
+- share/rbbt_commands/purge_job
 - share/rbbt_commands/resource/exists
 - share/rbbt_commands/resource/find
 - share/rbbt_commands/resource/get