cnvrg 0.0.149 → 0.0.1410

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 89c404e15ae2c15390c06632e16fdb46725a7dab
-  data.tar.gz: 28435401db7d5d2a78ba88de8946513ffc86dca5
+  metadata.gz: 7fc2d5ed72c306c741afc72e0427f6a9a713a71e
+  data.tar.gz: 12c7dbc1a35c33aa1a090a84056b700be07f52b4
 SHA512:
-  metadata.gz: 5442ff1a286b258d5624ba1bc00d7944423a99b0b74128cd1b90dffd24511e52e9609a7119ba2d24c4bec348927f7a637fb669b7f961d780f6070103e8cb8e20
-  data.tar.gz: 678610c32a2104cab60cec8bcab410cc5d76372079b99044776e444cc7713f206cb10b87521425459e3a8c53937032665a795fd36adfa48a58c9227a1963b998
+  metadata.gz: 9cdb426120acfb6b29ca96895a98adb57035cf8ae32bd077ffd71a69f1154bf67703f615b1aee86e0053eb4c3e080db8c49653d9bf9bf6cdada8800d5fe4d2b8
+  data.tar.gz: 07b4ddfc04e5b66890aec5a95fb7a18d100e62d6f14c7e9dbaebe1d8f6c3a2f34e24df4a5c99973f7546cf2aa9c4a3d17822813f68b84939baf90a39562c1647

data/lib/cnvrg/cli.rb

@@ -584,8 +584,6 @@ module Cnvrg
           path = Dir.pwd
           @dataset = Dataset.new(path)
 
-          @dataset.generate_idx()
-
           url = @dataset.url
           check = Helpers.checkmark
           say "#{check} Link finished successfully", Thor::Shell::Color::GREEN
@@ -663,25 +661,11 @@ module Cnvrg
     end
 
     desc 'init_data_container', 'Init dataset directory', :hide => true
-    method_option :container, :type => :string, :aliases => ["--c"], :default => ""
     method_option :login_content, :type => :string, :aliases => ["--l"], :default => ""
-    method_option :owner, :type => :string, :aliases => ["--o"], :default => ""
-    method_option :dataset_slug, :type => :string, :aliases => ["--ds"], :default => ""
-    method_option :dataset_name, :type => :string, :aliases => ["--dn"], :default => ""
-    method_option :sha1, :type => :string, :aliases => ["--s"], :default => ""
-    method_option :username, :type => :string, :aliases => ["--u"], :default => ""
-    method_option :api_url, :type => :string, :aliases => ["--a"], :default => ""
-
-    def init_data_container()
+
+    def init_data_container(container)
       begin
-        container = options["container"]
         login_content = options["login_content"]
-        dataset_slug = options["dataset_slug"]
-        owner = options["owner"]
-        dataset_name = options["dataset_name"]
-        sha1 = options["sha1"]
-        username = options["username"]
-        api_url = options["api_url"]
 
         container = Docker::Container.get(container)
         command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
@@ -694,18 +678,6 @@ module Cnvrg
         container.exec(command, tty: true)
         command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
         container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "cnvrg set_remote_api_url #{owner} #{username} #{api_url}"]
-        container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "mkdir -p /home/ds/notebooks/data/.cnvrg"]
-        container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "touch /home/ds/notebooks/data/.cnvrg/config.yml"]
-        container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "touch /home/ds/notebooks/data/.cnvrgignore"]
-        container.exec(command, tty: true)
-
-        command = ["/bin/bash", "-lc", "cnvrg data_init_container #{owner} #{dataset_slug} #{dataset_name}"]
-        container.exec(command, tty: true)
-
 
       rescue SignalException
         log_end(-1)
@@ -736,12 +708,10 @@ module Cnvrg
         exit(1)
       end
     end
+
     desc 'data_snap', 'Init dataset directory', :hide => true
 
-    def data_init_container(owner,dataset_slug,dataset_name)
-      puts owner
-      puts dataset_name
-      puts dataset_slug
+    def data_init_container(owner, dataset_slug, dataset_name)
 
       if Dataset.init_container(owner, dataset_slug, dataset_name)
 
@@ -1050,16 +1020,19 @@ module Cnvrg
 
         @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
         if !@dataset.update_ignore_list(ignore)
-          say "Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW
+          say "Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::RED
+          exit(1)
         end
-        result = @dataset.compare_idx(false)
+        say "Checking dataset", Thor::Shell::Color::BLUE
+        local_idx = @dataset.generate_idx
+        result = @dataset.compare_idx(false, commit=@dataset.last_local_commit, local_idx= local_idx)
 
 
         commit = result["result"]["commit"]
         if commit != @dataset.last_local_commit and !@dataset.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
           log_end(0)
 
-          say "Remote server has an updated version, please run `cnvrg download` first, or alternatively: `cnvrg sync`", Thor::Shell::Color::YELLOW
+          say "Remote server has an updated version, please run `cnvrg data download` first", Thor::Shell::Color::YELLOW
           exit(1)
         end
 
@@ -1076,16 +1049,6 @@ module Cnvrg
         update_total = result["added"].size + result["updated_on_local"].size + result["deleted"].size
         successful_updates = []
         successful_deletions = []
-        if verbose
-          if update_total == 1
-            say "Updating #{update_total} file", Thor::Shell::Color::BLUE
-          else
-            say "Updating #{update_total} files", Thor::Shell::Color::BLUE
-          end
-        else
-          say "Syncing files", Thor::Shell::Color::BLUE unless sync
-
-        end
 
         # Start commit
         res = @files.start_commit(false)["result"]
@@ -1108,19 +1071,51 @@ module Cnvrg
           @dataset.update_idx_with_files_commits!((successful_deletions+successful_updates), commit_time)
 
           @dataset.update_idx_with_commit!(commit_sha1)
+          say "Compressing data", Thor::Shell::Color::BLUE
 
           home_dir = File.expand_path('~')
           tar_path = "#{home_dir}/.cnvrg/tmp/#{@dataset.slug}_#{commit_sha1}.tar.gz"
-          tar_files_path =  "#{home_dir}/.cnvrg/tmp/#{@dataset.slug}_#{commit_sha1}.txt"
+          tar_files_path = "#{home_dir}/.cnvrg/tmp/#{@dataset.slug}_#{commit_sha1}.txt"
           tar_files = (result["added"] + result["updated_on_local"] +["\n"]).join("\n")
-          File.open(tar_files_path, 'w') {|f| f.write tar_files}
-          create_tar(dataset_dir, tar_path, tar_files_path)
+          File.open(tar_files_path, 'w') { |f| f.write tar_files }
+          is_tar = create_tar(dataset_dir, tar_path, tar_files_path)
+          if !is_tar
+            say "ERROR: Couldn't compress data", Thor::Shell::Color::RED
+            FileUtils.rm_rf([tar_path]) if File.exist? tar_path
+            FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
+            exit(1)
+          end
+          say "Uploading data", Thor::Shell::Color::BLUE
+
           res = @files.upload_tar_file(tar_path, tar_path, commit_sha1)
-          cur_idx = @dataset.get_idx.to_h
-          res = @files.end_commit_tar(commit_sha1, cur_idx)
+
+          if res
+            cur_idx = @dataset.get_idx.to_h
+
+            res = @files.end_commit_tar(commit_sha1, cur_idx)
+            if !Cnvrg::CLI.is_response_success(res, false)
+              FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
+              FileUtils.rm_rf([tar_path]) if File.exist? tar_path
+
+
+              @files.rollback_commit(commit_sha1)
+              say "Rolling Back all changes.", Thor::Shell::Color::RED
+              exit(1)
+            end
+
+          else
+            FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
+            FileUtils.rm_rf([tar_path]) if File.exist? tar_path
+
+
+            @files.rollback_commit(commit_sha1)
+            say "Rolling Back all changes.", Thor::Shell::Color::RED
+            exit(1)
+          end
+
 
           # delete
-          FileUtils.rm_rf([tar_path,tar_files_path])
+          FileUtils.rm_rf([tar_path, tar_files_path])
 
         rescue SignalException
           log_end(-1)
@@ -1162,7 +1157,8 @@ module Cnvrg
 
 
       rescue => e
-        log_end(-1)
+        log_end(-1, e.message)
+
 
         say "Error occurd, \nAborting", Thor::Shell::Color::RED
         @files.rollback_commit(commit_sha1)
@@ -1195,11 +1191,15 @@ module Cnvrg
     desc 'list data', 'List all dataset you currently have'
 
     def list_dataset
-      verify_logged_in(true)
+      verify_logged_in(false)
       log_start(__method__, args, options)
       dataset_dir = is_cnvrg_dir(Dir.pwd)
       @dataset = Dataset.new(dataset_dir)
-      result = @dataset.list()
+      owner = @dataset.owner
+      if owner.nil? or owner.empty?
+        owner = CLI.get_owner()
+      end
+      result = @dataset.list(owner)
       list = result["result"]["list"]
 
       print_table(list)
@@ -1757,7 +1757,7 @@ module Cnvrg
 
           log_end(0)
         end
-      rescue =>e
+      rescue => e
         log_end(-1)
 
         say "Error occurd, \nAborting", Thor::Shell::Color::BLUE
@@ -2444,17 +2444,6 @@ module Cnvrg
         data_commit = options["data_commit"] || nil
         ignore = options[:ignore] || ""
 
-        if !data.nil? and !data.empty?
-          if ignore.nil? or ignore.empty?
-            ignore = "data"
-          else
-            ignore +=",data"
-          end
-        end
-        if ignore.nil? or ignore.empty?
-          ignore= ""
-        end
-
         instance_type = options["machine_type"] || nil
         schedule = options["schedule"] || ""
         if schedule.start_with? 'in'
@@ -2483,9 +2472,9 @@ module Cnvrg
           upload_output_option = "--upload_output=#{upload_output}"
         end
         options_hash = Hash[options]
-        options_hash.except!("schedule", "machine_type", "image", "upload_output", "grid", "ignore", "data", "data_commit")
+        options_hash.except!("schedule", "machine_type", "image", "upload_output", "grid", "data", "data_commit")
         exec_options = options_hash.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
-        command = "#{exec_options}  --ignore=#{ignore} #{upload_output_option} #{cmd.flatten.join(" ")}"
+        command = "#{exec_options}  #{upload_output_option} #{cmd.flatten.join(" ")}"
         commit_to_run = options["commit"] || nil
         if !schedule.nil? and !schedule.empty?
 
@@ -4355,6 +4344,7 @@ module Cnvrg
           return false
         end
       end
+
       def data_dir_include()
         all_dirs = Dir.glob("**/*/", File::FNM_DOTMATCH)
         all_dirs.flatten!

data/lib/cnvrg/data.rb

@@ -52,8 +52,6 @@ module Cnvrg
     desc 'data list', 'list of datasets'
     def list()
       cli = Cnvrg::CLI.new()
-      verbose = options["verbose"]
-      sync = options["sync"]
 
       cli.list_dataset()
 

data/lib/cnvrg/datafiles.rb

@@ -59,6 +59,8 @@ module Cnvrg
                                                                     blob_id: upload_resp["result"]["id"]})
           return true
         end
+      else
+        return false
       end
       return false
     end
@@ -290,7 +292,7 @@ module Cnvrg
           is_success = false
           count = 0
           while !is_success and count <3
-            resp = `python #{tmp.path} --num-threads=128 --max-singlepart-upload-size=#{MULTIPART_SPLIT} put -f #{file_path} s3://#{URLcrypt.decrypt(upload_resp["result"]["bucket"])}/#{upload_resp["result"]["path"]+"/"+File.basename(file_path)}  > /dev/null 2>&1`
+            resp = `python #{tmp.path} --max-singlepart-upload-size=#{MULTIPART_SPLIT} put -f #{file_path} s3://#{URLcrypt.decrypt(upload_resp["result"]["bucket"])}/#{upload_resp["result"]["path"]+"/"+File.basename(file_path)}  > /dev/null 2>&1`
             is_success =$?.success?
             count +=1
 

data/lib/cnvrg/dataset.rb

@@ -31,8 +31,8 @@ module Cnvrg
       return response
 
     end
-    def list
-      response = Cnvrg::API.request("users/#{self.owner}/datasets/list", 'GET')
+    def list(owner)
+      response = Cnvrg::API.request("users/#{owner}/datasets/list", 'GET')
       CLI.is_response_success(response)
       return response
 
@@ -150,17 +150,23 @@ module Cnvrg
 
     def self.init_container(owner, dataset_slug,dataset_name)
 
-
       cnvrgignore = Helpers.cnvrgignore_content
       begin
-
+        list_dirs = [ ".cnvrg"
+        ]
+        list_files = [
+            ".cnvrgignore",
+            ".cnvrg/config.yml"
+        ]
+        FileUtils.mkdir_p list_dirs
+        FileUtils.touch list_files
 
         config = {dataset_name: dataset_name,
                   dataset_slug: dataset_slug,
                   owner: owner}
-        File.open("/home/ds/notebooks/data/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+        File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
 
-        File.open("/home/ds/notebooks/data/.cnvrgignore", "w+") { |f| f.write cnvrgignore } unless File.exist? ".cnvrgignore"
+        File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore } unless File.exist? ".cnvrgignore"
       rescue => e
         puts e
         puts e.backtrace
@@ -227,9 +233,11 @@ module Cnvrg
       CLI.is_response_success(response)
       return response
     end
-    def compare_idx(new_branch, commit=last_local_commit)
+    def compare_idx(new_branch, commit=last_local_commit,local_idx=nil)
+      if local_idx.nil?
+        local_idx =  self.generate_idx
+      end
 
-      local_idx = self.generate_idx
       response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/status", 'POST', {idx: local_idx, new_branch: new_branch, current_commit: commit})
       CLI.is_response_success(response)
       return response

data/lib/cnvrg/files.rb

@@ -269,7 +269,7 @@ module Cnvrg
           is_success = false
           count = 0
           while !is_success and count <3
-            resp = `python #{tmp.path} --num-threads=128 --max-singlepart-upload-size=#{MULTIPART_SPLIT} put -f #{file_path} s3://#{URLcrypt.decrypt(upload_resp["result"]["bucket"])}/#{upload_resp["result"]["path"]+"/"+File.basename(file_path)}  > /dev/null 2>&1`
+            resp = `python #{tmp.path} --max-singlepart-upload-size=#{MULTIPART_SPLIT} put -f #{file_path} s3://#{URLcrypt.decrypt(upload_resp["result"]["bucket"])}/#{upload_resp["result"]["path"]+"/"+File.basename(file_path)}  > /dev/null 2>&1`
             is_success =$?.success?
             count +=1
 

data/lib/cnvrg/version.rb

@@ -1,4 +1,4 @@
 module Cnvrg
-    VERSION = '0.0.149'
+    VERSION = '0.0.1410'
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 0.0.149
+  version: 0.0.1410
 platform: ruby
 authors:
 - Yochay Ettun
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-16 00:00:00.000000000 Z
+date: 2017-05-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler