scout-camp 0.1.10 → 0.1.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: de2b027b60ec3f0c33cb643258e365f129b217368040cc49d6df0b7232d643f9
-  data.tar.gz: a89e3ed2424d5eb99f107314bab8f6406297b1e2961d6b7fb084f652abcb4705
+  metadata.gz: 4310a29a500f7fe04fb974b8a5cb3ae5aae506a9a3663663ec6bd86270c17fd2
+  data.tar.gz: a6e9b6a396a76f9ddde67a502658b9cb37cddeb12355c1bbe244c5f1ff7c3f36
 SHA512:
-  metadata.gz: 30a2f0f6b5f1c28dfb6b2c364c78e8efa10c1dca04ad5a53ca651c62607c41ec74981a4729e885b778954e67dd5be4fcd6c27b12ea976bfc71854c55f97a15f7
-  data.tar.gz: 5c598b2cc4295153b2c2ad8cd7105c7d9b4dfb37b6a176065520523159e8de67167b5a211d06bcdef8322bd3f59b3414734ef1ac380fd80996ebf1fe07a49600
+  metadata.gz: 86dbee75e288daf9d1be188a9d13e79b9656314ef3345e24d226fd79fa987c7723c4bd1d2df488a5a2e56e04cdcb97bf69e4a2ef0f2a88bde00713978ff3b4be
+  data.tar.gz: 7d7bd20736cad39b68d9dc0406490a5ea2a5095bec45dfe550fd2e5131ecbbdf4ecfbd21ac523cfcddadd163c4988465ad6358038d38a9c3d5151a48c50cb6cb

data/.vimproject

@@ -89,6 +89,7 @@ scout-camp=/$PWD filter="*" {
 
     container_lambda=container_lambda{
      main.tf
+     data.tf
      locals.tf
      variables.tf
     }

data/VERSION

@@ -1 +1 @@
-0.1.10
+0.1.11

data/lib/scout/aws/s3.rb

@@ -21,14 +21,18 @@ module Open
       uri.start_with? 's3://'
     end
 
-    def self.claim(uri, ...)
+    def self.claim_uri(uri)
       if Path === uri and not uri.located?
-        is_s3? uri.find
+        is_s3?(uri.find)
       else
         is_s3? uri
       end
     end
 
+    def self.claim(uri, uri2=nil, ...)
+      claim_uri(uri) || (String === uri2 && claim_uri(uri2))
+    end
+
     def self.parse_s3_uri(uri)
       uri = uri.find if Path === uri and not uri.located?
       uri = uri.sub(%r{^s3://}, '')
@@ -66,13 +70,12 @@ module Open
       end
     end
 
-    def self.glob(uri, pattern="**/*")
+    def self.glob(uri, pattern="*")
       bucket, prefix = parse_s3_uri(uri)
       s3 = Aws::S3::Client.new
       matches = []
       continuation_token = nil
 
-      Log.debug "Glob: #{uri} #{pattern}"
       loop do
         resp = s3.list_objects_v2(
           bucket: bucket,
@@ -185,6 +188,7 @@ module Open
       bucket, key = parse_s3_uri(uri)
       return false if key.empty? # Can't check existence of bucket this way
 
+      key += '/' unless key.end_with?('/')
       s3 = Aws::S3::Client.new
       response = s3.list_objects_v2({
         bucket: bucket,
@@ -224,6 +228,68 @@ module Open
     def self.ln_s(source, target, options = {})
       cp(source, target)
     end
+
+    def self.sync(source, target, options = {})
+      excludes, files, hard_link, test, print, delete, other = IndiferentHash.process_options options,
+        :excludes, :files, :hard_link, :test, :print, :delete, :other
+
+      excludes ||= %w(.save .crap .source tmp filecache open-remote)
+      excludes = excludes.split(/,\s*/) if excludes.is_a?(String) and not excludes.include?("--exclude")
+
+      if File.directory?(source) || source.end_with?("/")
+        source += "/" unless source.end_with? '/'
+        target += "/" unless target.end_with? '/'
+      end
+
+      if source == target
+        Log.warn "Asking to sync with itself"
+        return
+      end
+
+      Log.low "Migrating #{source} #{files.length} files to #{target} - #{Misc.fingerprint(files)}}" if files
+
+      sync_args = %w()
+      sync_args << excludes.collect{|s| "--exclude '#{s}'" } if excludes and excludes.any?
+      sync_args << "-nv" if test
+
+      if files
+        tmp_files = TmpFile.tmp_file 's3_sync_files-'
+        Open.write(tmp_files, files * "\n")
+        sync_args << "--files-from='#{tmp_files}'"
+      end
+
+      if Open.directory?(source)
+        cmd = "aws s3 sync #{sync_args * " "} #{source} #{target}"
+      else
+        cmd = "aws s3 cp #{source} #{target}"
+      end
+      case other
+      when String
+        cmd << " " << other
+      when Array
+        cmd << " " << other * " "
+      end
+      cmd << " && rm -Rf #{source}" if delete && ! files
+
+      if print
+        cmd
+      else
+        CMD.cmd_log(cmd, :log => Log::HIGH)
+
+        if delete && files
+          remove_files = files.collect{|f| File.join(source, f) }
+          dirs = remove_files.select{|f| File.directory? f }
+          remove_files.each do |file|
+            next if dirs.include? file
+            Open.rm file
+          end
+
+          dirs.each do |dir|
+            FileUtils.rmdir dir if Dir.glob(dir).empty?
+          end
+        end
+      end
+    end
   end
 end
 

data/lib/scout/offsite/resource.rb

@@ -11,7 +11,7 @@ module Resource
       resource = path.pkgdir if resource.nil? and path.is_a?(Path) and path.pkgdir.is_a?(Resource)
       resource = Resource.default_resource if resource.nil?
 
-      if File.exist?(path)
+      if Path.located?(path)
         real_paths = [path]
       else
         path = Path.setup(path, pkgdir: resource) unless path.is_a?(Path)

data/scout-camp.gemspec

@@ -2,11 +2,11 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-camp 0.1.10 ruby lib
+# stub: scout-camp 0.1.11 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-camp".freeze
-  s.version = "0.1.10".freeze
+  s.version = "0.1.11".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]

data/scout_commands/sync

@@ -2,6 +2,7 @@
 
 require 'scout'
 require 'scout/offsite/resource'
+require 'scout/aws/s3'
 
 $0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
 

data/share/aws/lambda_function.rb

@@ -40,7 +40,7 @@ def lambda_handler(event:, context:)
         body: job.path
       }
     elsif queue
-      save_inputs = Scout.var.queue[workflow.to_s][task_name][job.name].find
+      save_inputs = Scout.var.queue[workflow.to_s][task_name][job.name].find :bucket
       job.save_inputs(save_inputs)
       {
         statusCode: 202,

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scout-camp
 version: !ruby/object:Gem::Version
-  version: 0.1.10
+  version: 0.1.11
 platform: ruby
 authors:
 - Miguel Vazquez