scout-essentials 1.6.9 → 1.6.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a3c6a3bfa03c83171a83558fa73b4a23c5bffffb02fa036c36f4ad6417abbee9
-  data.tar.gz: 605d88dc63abe624432c9ece462ac19c33e243252667d5aa8b27a541e38582f6
+  metadata.gz: ca46feecfdca0f796ad76e9e0e8f3dcf33eb169e41605708cf2042507c968207
+  data.tar.gz: 93b589d483250fdca27ed5e888887d2135b81f253b7ad30afb97adcb8a3382f2
 SHA512:
-  metadata.gz: 9528eea7eccc93c0640f1530ecba2eeb874fd3f4bb211789f8bd830e127ff52c880cee2fb4621f92db7f855fb0d4f7d12098ace50726f76f14f27c5cd8ffb8f7
-  data.tar.gz: beaf0a93070745d6102d766931557369a74d5ada67182c7b13cde60eb5d14a8d8e71f29e8f80356910bac97fef1022e9bb21028d2a687515a04bb781dd7d61fb
+  metadata.gz: 95d450e003292a7507116cd39bc74822e12820119cea0ddfeabd8aa5a86979f17d4a8aefdf76db5f526026c7795422b8bf7a91c5ae2833a75feb48cf08c5a17b
+  data.tar.gz: 2bdaa762c5899135b5d5aadc054017b597244bf000bb5525894da4b9f9e893053221d3049c5df821726cb999cb5c3fc53e486535395b7fb54513b6a9a339d21c

data/.vimproject

@@ -66,6 +66,7 @@ scout-essentials=/$PWD filter="*.rb *.txt *.md *.conf *.yaml" {
     remote.rb
     stream.rb
     util.rb
+    sync.rb
     bgzf.rb
    }
    tmpfile.rb
@@ -76,6 +77,7 @@ scout-essentials=/$PWD filter="*.rb *.txt *.md *.conf *.yaml" {
     produce.rb
     scout.rb
     software.rb
+    sync.rb
     util.rb
    }
    config.rb

data/VERSION

@@ -1 +1 @@
-1.6.9
+1.6.11

data/lib/scout/annotation/annotation_module.rb

@@ -1,8 +1,10 @@
+require 'scout/log'
 module Annotation
   module AnnotationModule
     def annotation(*attrs)
       annotations = self.instance_variable_get("@annotations")
-      annotations.concat attrs - annotations
+      attrs -= annotations
+      annotations.concat attrs
       attrs.each do |a|
         self.attr_accessor a
       end
@@ -13,8 +15,11 @@ module Annotation
     end
 
     def included(mod)
+      attrs = self.instance_variable_get("@annotations")
       mod.instance_variable_set(:@annotations, []) unless mod.instance_variables.include?(:@annotations)
-      mod.instance_variable_get(:@annotations).concat self.instance_variable_get(:@annotations)
+      annotations = mod.instance_variable_get(:@annotations)
+      attrs -= annotations
+      annotations.concat attrs
     end
 
     def extended(obj)
@@ -24,6 +29,7 @@ module Annotation
       obj.annotation_types << self
 
       annotations = obj.instance_variable_get(:@annotations)
+      attrs -= annotations
       annotations.concat attrs
     end
 

data/lib/scout/cmd.rb

@@ -92,7 +92,7 @@ module CMD
   end
 
   def self.bash(cmd)
-    cmd = %Q(bash <<EOF\n#{cmd}\nEOF\n)
+    cmd = %Q(bash -l <<EOF\n#{cmd}\nEOF\n)
     CMD.cmd(cmd, :autojoin => true)
   end
 

data/lib/scout/indiferent_hash.rb

@@ -115,10 +115,27 @@ module IndiferentHash
   def keys_to_sym!
     string_keys = keys.select{|k| String === k}
     string_keys.each do |key|
-      self[key.to_sym] = self.delete(key)
+      begin
+        self[key.to_sym] = self.delete(key)
+      rescue
+        next
+      end
     end
   end
 
+  def keys_to_sym
+    new = IndiferentHash.setup({})
+    self.keys.each do |key|
+      begin
+        skey = key.is_a?(String) ? key.to_sym : key
+      rescue
+        skey = key
+      end
+      new[skey] = self[key]
+    end
+    new
+  end
+
   def prety_print
     Misc.format_definition_list(self, sep: "\n")
   end

data/lib/scout/open/final.rb

@@ -140,4 +140,56 @@ module Open
       nil
     end
   end
+
+  def self.ln_s(source, target, options = {})
+    source = source.find if Path === source
+    target = target.find if Path === target
+
+    target = File.join(target, File.basename(source)) if File.directory? target
+    FileUtils.mkdir_p File.dirname(target) unless File.exist?(File.dirname(target))
+    FileUtils.rm target if File.exist?(target)
+    FileUtils.rm target if File.symlink?(target)
+    FileUtils.ln_s source, target
+  end
+
+  def self.ln(source, target, options = {})
+    source = source.find if Path === source
+    target = target.find if Path === target
+    source = File.realpath(source) if File.symlink?(source)
+
+    FileUtils.mkdir_p File.dirname(target) unless File.exist?(File.dirname(target))
+    FileUtils.rm target if File.exist?(target)
+    FileUtils.rm target if File.symlink?(target)
+    FileUtils.ln source, target
+  end
+
+  def self.ln_h(source, target, options = {})
+    source = source.find if Path === source
+    target = target.find if Path === target
+
+    FileUtils.mkdir_p File.dirname(target) unless File.exist?(File.dirname(target))
+    FileUtils.rm target if File.exist?(target)
+    begin
+      CMD.cmd("ln -L '#{ source }' '#{ target }'")
+    rescue ProcessFailed
+      Log.debug "Could not hard link #{source} and #{target}: #{$!.message.gsub("\n", '. ')}"
+      CMD.cmd("cp -L '#{ source }' '#{ target }'")
+    end
+  end
+
+  def self.link(source, target, options = {})
+    begin
+      Open.ln(source, target, options)
+    rescue
+      Log.debug "Could not make regular link, trying symbolic: #{Misc.fingerprint(source)} -> #{Misc.fingerprint(target)}"
+      Open.ln_s(source, target, options)
+    end
+    nil
+  end
+
+  def self.link_dir(source, target)
+    Log.debug "Copy with hard-links #{Log.fingerprint source}->#{Log.fingerprint target}"
+    FileUtils.cp_lr(source, target)
+  end
+
 end

data/lib/scout/open/sync.rb

@@ -0,0 +1,85 @@
+module Open
+
+  def self.rsync(source, target, options = {})
+    excludes, files, hard_link, test, print, delete, source_server, target_server, other = IndiferentHash.process_options options,
+      :excludes, :files, :hard_link, :test, :print, :delete, :source, :target, :other
+
+    excludes ||= %w(.save .crap .source tmp filecache open-remote)
+    excludes = excludes.split(/,\s*/) if excludes.is_a?(String) and not excludes.include?("--exclude")
+
+    if File.directory?(source) || source.end_with?("/")
+      source += "/" unless source.end_with? '/'
+      target += "/" unless target.end_with? '/'
+    end
+
+    if source == target && ! (source_server || target_server)
+      Log.warn "Asking to rsync local file with itself"
+      return
+    end
+
+    if target_server
+      target_uri = [target_server, "'" + target + "'"] * ":" 
+    else
+      target_uri = "'" + target + "'"
+    end
+
+    if source_server
+      source_uri = [source_server, "'" + source + "'"] * ":" 
+    else
+      source_uri = "'" + source + "'"
+    end
+
+
+    if target_server
+      CMD.cmd("ssh #{target_server} mkdir -p '#{File.dirname(target)}'")
+    else
+      Open.mkdir File.dirname(target)
+    end
+
+
+    Log.low "Migrating #{source} #{files.length} files to #{target} - #{Misc.fingerprint(files)}}" if files
+
+
+    rsync_args = %w(-avztHP --copy-unsafe-links --omit-dir-times)
+    rsync_args << "--link-dest '#{source}'" if hard_link && ! source_server
+    rsync_args << excludes.collect{|s| "--exclude '#{s}'" } if excludes and excludes.any?
+    rsync_args << "-nv" if test
+    if files
+      tmp_files = TmpFile.tmp_file 'rsync_files-'
+      Open.write(tmp_files, files * "\n")
+      rsync_args << "--files-from='#{tmp_files}'"
+    end
+
+    cmd = "rsync #{rsync_args * " "} #{source_uri} #{target_uri}"
+    case other
+    when String
+      cmd << " " << other
+    when Array
+      cmd << " " << other * " "
+    end
+    cmd << " && rm -Rf #{source}" if delete && ! files
+
+    if print
+      cmd
+    else
+      CMD.cmd_log(cmd, :log => Log::HIGH)
+
+      if delete && files
+        remove_files = files.collect{|f| File.join(source, f) }
+        dirs = remove_files.select{|f| File.directory? f }
+        remove_files.each do |file|
+          next if dirs.include? file
+          Open.rm file
+        end
+
+        dirs.each do |dir|
+          FileUtils.rmdir dir if Dir.glob(dir).empty?
+        end
+      end 
+    end
+  end
+
+  def self.sync(...)
+    rsync(...)
+  end
+end

data/lib/scout/open/util.rb

@@ -125,57 +125,6 @@ module Open
     Pathname.new(File.expand_path(file)).realpath.to_s 
   end
 
-  def self.ln_s(source, target, options = {})
-    source = source.find if Path === source
-    target = target.find if Path === target
-
-    target = File.join(target, File.basename(source)) if File.directory? target
-    FileUtils.mkdir_p File.dirname(target) unless File.exist?(File.dirname(target))
-    FileUtils.rm target if File.exist?(target)
-    FileUtils.rm target if File.symlink?(target)
-    FileUtils.ln_s source, target
-  end
-
-  def self.ln(source, target, options = {})
-    source = source.find if Path === source
-    target = target.find if Path === target
-    source = File.realpath(source) if File.symlink?(source)
-
-    FileUtils.mkdir_p File.dirname(target) unless File.exist?(File.dirname(target))
-    FileUtils.rm target if File.exist?(target)
-    FileUtils.rm target if File.symlink?(target)
-    FileUtils.ln source, target
-  end
-
-  def self.ln_h(source, target, options = {})
-    source = source.find if Path === source
-    target = target.find if Path === target
-
-    FileUtils.mkdir_p File.dirname(target) unless File.exist?(File.dirname(target))
-    FileUtils.rm target if File.exist?(target)
-    begin
-      CMD.cmd("ln -L '#{ source }' '#{ target }'")
-    rescue ProcessFailed
-      Log.debug "Could not hard link #{source} and #{target}: #{$!.message.gsub("\n", '. ')}"
-      CMD.cmd("cp -L '#{ source }' '#{ target }'")
-    end
-  end
-
-  def self.link(source, target, options = {})
-    begin
-      Open.ln(source, target, options)
-    rescue
-      Log.debug "Could not make regular link, trying symbolic: #{Misc.fingerprint(source)} -> #{Misc.fingerprint(target)}"
-      Open.ln_s(source, target, options)
-    end
-    nil
-  end
-
-  def self.link_dir(source, target)
-    Log.debug "Copy with hard-links #{Log.fingerprint source}->#{Log.fingerprint target}"
-    FileUtils.cp_lr(source, target)
-  end
-
   def self.list(file)
     file = file.produce_and_find if Path === file
     Open.read(file).split("\n")

data/lib/scout/open.rb

@@ -6,6 +6,7 @@ require_relative 'open/stream'
 require_relative 'open/util'
 require_relative 'open/remote'
 require_relative 'open/lock'
+require_relative 'open/sync'
 
 module Open
   module NamedStream

data/lib/scout/path/find.rb

@@ -141,9 +141,13 @@ module Path
 
   SLASH = "/"[0]
   DOT = "."[0]
-  def located?
+  def self.located?(path)
     # OPEN RESOURCE
-    self.slice(0,1) == SLASH || (self.slice(0,1) == DOT && self.slice(1,2) == SLASH) # || (resource != Rbbt && (Open.remote?(self) || Open.ssh?(self)))
+    path.slice(0,1) == SLASH || (path.slice(0,1) == DOT && path.slice(1,2) == SLASH)
+  end
+
+  def located?
+    Path.located?(self)
   end
 
   def annotate_found_where(found, where)

data/lib/scout/resource/sync.rb

@@ -0,0 +1,24 @@
+require 'scout/open/sync'
+module Resource
+  def self.sync(path, map = nil, options = {})
+    resource = IndiferentHash.process_options options, 
+      :resource
+
+    map = 'user' if map.nil?
+    resource = path.pkgdir if resource.nil? and path.is_a?(Path) and path.pkgdir.is_a?(Resource)
+    resource = Resource.default_resource if resource.nil?
+
+    target = resource.identify(path).find(map)
+
+    if File.exist?(path)
+      real_paths = [path]
+    else
+      path = Path.setup(path, pkgdir: resource) unless path.is_a?(Path)
+      real_paths = path.directory? ? path.find_all : path.glob_all
+    end
+
+    real_paths.each do |source|
+      Open.sync(source, target, options)
+    end
+  end
+end

data/lib/scout/resource/util.rb

@@ -1,9 +1,9 @@
 module Resource
   def identify(path)
-    return path unless path.start_with?("/")
+    return path unless Path.located?(path)
     path_maps = path.path_maps if Path === path
     path_maps ||= self.path_maps || Path.path_maps
-    path = File.expand_path(path)
+    path = File.expand_path(path) if path.start_with?("/")
     path += "/" if File.directory?(path)
 
     map_order ||= (path_maps.keys & Path.basic_map_order) + (path_maps.keys - Path.basic_map_order)
@@ -30,7 +30,7 @@ module Resource
               m.named_captures.include?(c) ? m[c] : nil
             }.compact * "/"
 
-            unlocated.gsub!(/\/+/,'/')
+            #unlocated.gsub!(/\/+/,'/')
 
             if self.subdir && ! self.subdir.empty?
               subdir = self.subdir

data/lib/scout/resource.rb

@@ -5,6 +5,7 @@ require_relative 'resource/path'
 require_relative 'resource/open'
 require_relative 'resource/util'
 require_relative 'resource/software'
+require_relative 'resource/sync'
 
 module Resource
   extend Annotation
@@ -14,7 +15,7 @@ module Resource
     attr_accessor :default_resource
 
     def default_resource
-      @default_resource ||= Scout
+      @default_resource
     end
   end
 

data/lib/scout/simple_opt/get.rb

@@ -41,7 +41,7 @@ module SOPT
     IndiferentHash.setup @@current_options
     GOT_OPTIONS.merge! @@current_options
 
-    @@current_options
+    @@current_options.keys_to_sym
   end
   
   def self.get(opt_str)

data/scout-essentials.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-essentials 1.6.9 ruby lib
+# stub: scout-essentials 1.6.11 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-essentials".freeze
-  s.version = "1.6.9".freeze
+  s.version = "1.6.11".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Miguel Vazquez".freeze]
-  s.date = "2025-03-25"
+  s.date = "2025-04-10"
   s.description = "Things a scout can use anywhere".freeze
   s.email = "mikisvaz@gmail.com".freeze
   s.extra_rdoc_files = [
@@ -64,6 +64,7 @@ Gem::Specification.new do |s|
     "lib/scout/open/lock/lockfile.rb",
     "lib/scout/open/remote.rb",
     "lib/scout/open/stream.rb",
+    "lib/scout/open/sync.rb",
     "lib/scout/open/util.rb",
     "lib/scout/path.rb",
     "lib/scout/path/digest.rb",
@@ -81,6 +82,7 @@ Gem::Specification.new do |s|
     "lib/scout/resource/produce/rake.rb",
     "lib/scout/resource/scout.rb",
     "lib/scout/resource/software.rb",
+    "lib/scout/resource/sync.rb",
     "lib/scout/resource/util.rb",
     "lib/scout/simple_opt.rb",
     "lib/scout/simple_opt/accessor.rb",
@@ -110,6 +112,7 @@ Gem::Specification.new do |s|
     "test/scout/open/test_lock.rb",
     "test/scout/open/test_remote.rb",
     "test/scout/open/test_stream.rb",
+    "test/scout/open/test_sync.rb",
     "test/scout/open/test_util.rb",
     "test/scout/path/test_digest.rb",
     "test/scout/path/test_find.rb",
@@ -120,6 +123,7 @@ Gem::Specification.new do |s|
     "test/scout/resource/test_path.rb",
     "test/scout/resource/test_produce.rb",
     "test/scout/resource/test_software.rb",
+    "test/scout/resource/test_sync.rb",
     "test/scout/resource/test_util.rb",
     "test/scout/simple_opt/test_doc.rb",
     "test/scout/simple_opt/test_get.rb",

data/test/scout/open/test_sync.rb

@@ -0,0 +1,55 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestOpenSync < Test::Unit::TestCase
+  def test_rsync_basic
+    TmpFile.with_file do |source|
+      TmpFile.with_file do |target|
+        Open.write(source, 'payload')
+        Open.rsync(source, target)
+        assert_equal 'payload', Open.read(target)
+      end
+    end
+  end
+
+  def test_rsync_dir
+    TmpFile.with_path do |source|
+      TmpFile.with_path do |target|
+        Open.write(source.file, 'payload')
+        Open.rsync(source, target)
+        assert_equal 'payload', Open.read(target.file)
+      end
+    end
+  end
+
+  def test_rsync_excludes
+    TmpFile.with_path do |source|
+      TmpFile.with_path do |target|
+        Open.write(source.file, 'payload')
+        Open.write(source.tmp_dir.tmp_file, 'decoy')
+        Open.rsync(source, target, excludes: 'tmp_dir')
+        assert_equal 'payload', Open.read(target.file)
+        refute Open.exist?(target.tmp_dir.tmp_file)
+      end
+    end
+  end
+
+  def test_rsync_remote
+    TmpFile.with_path do |source|
+      Open.write(source.file, 'payload')
+      cmd = Open.rsync(source, 'remote:target', print: true)
+      assert cmd.end_with?('remote:target/\'')
+    end
+  end
+
+  def test_sync_alias
+    TmpFile.with_path do |source|
+      TmpFile.with_path do |target|
+        Open.write(source.file, 'payload')
+        Open.sync(source, target)
+        assert_equal 'payload', Open.read(target.file)
+      end
+    end
+  end
+end
+

data/test/scout/resource/test_sync.rb

@@ -0,0 +1,32 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout'
+class TestResourceSync < Test::Unit::TestCase
+  def test_sync_file
+    TmpFile.with_path do |source|
+      TmpFile.with_path do |target|
+        Open.write(source.file, 'payload')
+        Misc.in_dir target.find do
+          Resource.sync(source.file, :current)
+        end
+
+        assert_equal 'payload', Open.read(target[Resource.identify(source)].file)
+      end
+    end
+  end
+
+  def test_sync_dir
+    TmpFile.with_path do |source|
+      TmpFile.with_path do |target|
+        Open.write(source.file, 'payload')
+        Misc.in_dir target.find do
+          Resource.sync(source, :current)
+        end
+
+        assert_equal 'payload', Open.read(target[Resource.identify(source)].file)
+      end
+    end
+  end
+end
+

data/test/scout/resource/test_util.rb

@@ -10,6 +10,11 @@ class TestResourceUtil < Test::Unit::TestCase
     assert_equal 'share/data/somedir/somepath', Scout.identify(path)
   end
 
+  def test_identify_dir
+    path = Path.setup('share/data/somedir/').find
+    assert_equal 'share/data/somedir', Scout.identify(path)
+  end
+
   def test_identify_with_subdir
     m = Module.new
     m.extend Resource
@@ -35,5 +40,14 @@ class TestResourceUtil < Test::Unit::TestCase
       assert_equal path2, Resource.relocate(path1)
     end
   end
+
+  def ___test_s3
+    require 'scout-camp'
+    require 'scout/aws/s3'
+    p = Path.setup("s3://bucket/var/jobs/workflow/task/job.txt")
+    p.path_maps = {:bucket => 's3://bucket/{TOPLEVEL}/{SUBPATH}'}
+
+    assert_equal 'var/jobs/workflow/task/job.txt', Resource.identify(p)
+  end
 end
 

data/test/scout/test_resource.rb

@@ -1,6 +1,7 @@
 require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
 require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
 
+require 'scout'
 require 'scout/resource/scout'
 class TestResourceUnit < Test::Unit::TestCase
   module TestResource
@@ -22,5 +23,6 @@ class TestResourceUnit < Test::Unit::TestCase
     assert_equal 'share/databases/DATABASE/FILE', Resource.identify('/usr/local/share/scout/databases/DATABASE/FILE')
     assert_equal 'share/databases/DATABASE/FILE', Resource.identify(File.join(ENV["HOME"], '.scout/share/databases/DATABASE/FILE'))
     assert_equal 'share/databases/DATABASE/FILE', Resource.identify('/usr/local/share/scout/databases/DATABASE/FILE')
+    assert_equal 'share/databases/DATABASE/FILE', Resource.identify('/usr/local/share/scout/databases/DATABASE/FILE/')
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: scout-essentials
 version: !ruby/object:Gem::Version
-  version: 1.6.9
+  version: 1.6.11
 platform: ruby
 authors:
 - Miguel Vazquez
 bindir: bin
 cert_chain: []
-date: 2025-03-25 00:00:00.000000000 Z
+date: 2025-04-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: shoulda
@@ -174,6 +174,7 @@ files:
 - lib/scout/open/lock/lockfile.rb
 - lib/scout/open/remote.rb
 - lib/scout/open/stream.rb
+- lib/scout/open/sync.rb
 - lib/scout/open/util.rb
 - lib/scout/path.rb
 - lib/scout/path/digest.rb
@@ -191,6 +192,7 @@ files:
 - lib/scout/resource/produce/rake.rb
 - lib/scout/resource/scout.rb
 - lib/scout/resource/software.rb
+- lib/scout/resource/sync.rb
 - lib/scout/resource/util.rb
 - lib/scout/simple_opt.rb
 - lib/scout/simple_opt/accessor.rb
@@ -220,6 +222,7 @@ files:
 - test/scout/open/test_lock.rb
 - test/scout/open/test_remote.rb
 - test/scout/open/test_stream.rb
+- test/scout/open/test_sync.rb
 - test/scout/open/test_util.rb
 - test/scout/path/test_digest.rb
 - test/scout/path/test_find.rb
@@ -230,6 +233,7 @@ files:
 - test/scout/resource/test_path.rb
 - test/scout/resource/test_produce.rb
 - test/scout/resource/test_software.rb
+- test/scout/resource/test_sync.rb
 - test/scout/resource/test_util.rb
 - test/scout/simple_opt/test_doc.rb
 - test/scout/simple_opt/test_get.rb