cnvrg 0.0.148 → 0.0.149

data/lib/cnvrg/experiment.rb

@@ -78,15 +78,15 @@ module Cnvrg
 
     end
 
-    def exec_remote(command, commit_to_run, instance_type, image_slug,scheduling_query,local_timestamp, grid,path_to_cmd)
-
+    def exec_remote(command, commit_to_run, instance_type, image_slug,scheduling_query,local_timestamp, grid,path_to_cmd,data, data_commit)
       response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/experiment/remote", 'POST', {command: command, image_slug: image_slug,
                                                                                                             commit_sha1: commit_to_run,
                                                                                                             instance_type: instance_type,
                                                                                                             scheduling_query:scheduling_query,
                                                                                                             local_timestamp:local_timestamp,
                                                                                                             grid: grid,
-                                                                                                            path_to_cmd:path_to_cmd})
+                                                                                                            path_to_cmd:path_to_cmd,dataset_slug:data,
+                                                                                                            dataset_commit: data_commit})
       return response
     end
 

data/lib/cnvrg/files.rb

@@ -206,7 +206,7 @@ module Cnvrg
 
     end
 
-    def   upload_large_files_s3(upload_resp, file_path)
+    def upload_large_files_s3(upload_resp, file_path)
       begin
         sts_path = upload_resp["result"]["path_sts"]
         s4cmd_path = upload_resp["result"]["path_s4cmd"]
@@ -225,6 +225,14 @@ module Cnvrg
         URLcrypt::key = [body].pack('H*')
 
         python_version=`python --version  > /dev/null 2>&1` ;  is_python=$?.success?
+        if is_python
+
+          s4cmd=`pip freeze |grep s4cmd > /dev/null 2>&1` ;  s4cmd_suc=$?.success?
+          if !s4cmd_suc
+            `pip install s4cmd  > /dev/null 2>&1`
+          end
+
+        end
         if !is_python
            s3 = Aws::S3::Resource.new(
                :access_key_id => URLcrypt.decrypt(upload_resp["result"]["sts_a"]),
@@ -245,7 +253,6 @@ module Cnvrg
             response = http.request s4cmd_request
             s4cmd_body = response.read_body
           end
-
           s4cmd_new_body = s4cmd_body.gsub(" self.client = self.boto3.client('s3',
                                       aws_access_key_id=aws_access_key_id,
                                       aws_secret_access_key=aws_secret_access_key)"," self.client = self.boto3.client('s3',
@@ -253,6 +260,7 @@ module Cnvrg
                                       aws_secret_access_key='#{URLcrypt.decrypt(upload_resp["result"]["sts_s"])}',
                                       aws_session_token='#{URLcrypt.decrypt(upload_resp["result"]["sts_st"])}')")
 
+
           tmp = Tempfile.new('s4cmd.py')
           tmp << s4cmd_new_body
           tmp.flush
@@ -261,8 +269,7 @@ module Cnvrg
           is_success = false
           count = 0
           while !is_success and count <3
-
-            resp = `python #{tmp.path} --max-singlepart-upload-size=#{MULTIPART_SPLIT} put -f #{file_path} s3://#{URLcrypt.decrypt(upload_resp["result"]["bucket"])}/#{upload_resp["result"]["path"]+"/"+File.basename(file_path)}  > /dev/null 2>&1`
+            resp = `python #{tmp.path} --num-threads=128 --max-singlepart-upload-size=#{MULTIPART_SPLIT} put -f #{file_path} s3://#{URLcrypt.decrypt(upload_resp["result"]["bucket"])}/#{upload_resp["result"]["path"]+"/"+File.basename(file_path)}  > /dev/null 2>&1`
             is_success =$?.success?
             count +=1
 
@@ -277,7 +284,6 @@ module Cnvrg
         if File.exist? tmp
           FileUtils.rm_rf [tmp]
         end
-        puts e
         return false
 
       end

data/lib/cnvrg/project.rb

@@ -30,12 +30,14 @@ module Cnvrg
     end
 
     def update_ignore_list(new_ignore)
-      if new_ignore.empty?
+      if new_ignore.nil? or new_ignore.empty?
         return true
       end
+      list = new_ignore.split(",")
       begin
         File.open(self.local_path+"/.cnvrgignore", "a+") do |f|
-          new_ignore.each do |i|
+          f.puts("\n")
+          list.each do |i|
             f.puts("#{i}\n")
           end
         end
@@ -49,13 +51,18 @@ module Cnvrg
       ignore_list = []
       File.open(self.local_path+"/.cnvrgignore", "r").each_line do |line|
         line = line.strip
-        if line.start_with? "#"
+        if line.start_with? "#" or ignore_list.include? line or line.empty?
           next
         end
-        if line.end_with? "/"
-          ignore_list << line.chop
-          sub_dirs = Dir.glob("#{line}/**/*").each { |x| x.gsub!("//", "/") }
-          ignore_list << sub_dirs.flatten
+        if line.end_with? "/" or File.directory?(line)
+          if line.end_with? "/"
+            ignore_list << line.chop
+          else
+            ignore_list << line
+          end
+            all_sub = Dir.glob("#{line}/**/*", File::FNM_DOTMATCH).flatten
+
+          ignore_list << all_sub.flatten
         else
           ignore_list << line
         end
@@ -72,7 +79,6 @@ module Cnvrg
       else
 
         list_dirs = [project_name,
-                     project_name + "/data",
                      project_name + "/models",
                      project_name + "/notebooks",
                      project_name + "/src",
@@ -242,15 +248,18 @@ module Cnvrg
         list_ignore = self.get_ignore_list()
         list.each do |e|
           label = e.gsub(self.local_path + "/", "")
+
           if File.directory? e
             if list_ignore.include? label
               next
             end
+
             tree_idx[label+"/"] = nil
           else
             if list_ignore.include? label
               next
             end
+
             sha1 = Digest::SHA1.file(e).hexdigest
             if old_idx.nil? or old_idx.to_h[:tree].nil?
               tree_idx[label] = {sha1: sha1, commit_time: nil}

data/lib/cnvrg/version.rb

@@ -1,4 +1,4 @@
 module Cnvrg
-    VERSION = '0.0.148'
+    VERSION = '0.0.149'
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 0.0.148
+  version: 0.0.149
 platform: ruby
 authors:
 - Yochay Ettun
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-01 00:00:00.000000000 Z
+date: 2017-05-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler