cnvrg 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 17942c336608306042bc9744c306d7761dc7e641
-  data.tar.gz: 3e931700b1a8c96abf593077d585980fcb89afc1
+  metadata.gz: 35ef0e50784c1333f4fa375ec78a0c968d883363
+  data.tar.gz: b2079022aca9ff172af33f81c1b3ee66c388bd4a
 SHA512:
-  metadata.gz: 64ae69b8efac75486edc8911f99db1c5563c84e38f2eeb33e0dcc24b75ae470782b7a3352c4b8f4e7aaf6dae3261ec71b2bd287963c396998dd1e0e876fd41d7
-  data.tar.gz: 2fecd443628c87027559088293215ef007d144a5b9dd7aef3a1ff791f3391b33f4e4b426f2b890f32a60d31ec352f2e1c0851ef8b5c1a32fd3f671b575e8cf37
+  metadata.gz: 9dbe6889b455e7a302e6909dfb99e69b963260ed3baa4b7c78df0b7ab436ca8fe8134771f256965010c4e2c0807e6f81f8268d7c01ff0d0bcc69879e3993808b
+  data.tar.gz: d624f15e9347cdd1020c9a20db0e07161c0029047dfa0859c513e415bf4654a4b79ea80f0b32915be96e5d0d454c60020b8314e1a1deaa2137a7d2fae4b6a359

data/lib/cnvrg/cli.rb

@@ -831,6 +831,8 @@ module Cnvrg
           if !download_res
             exit(1)
           end
+            Dataset.verify_cnvrgignore_exist(dataset_name,false)
+
 
 
           check = Helpers.checkmark
@@ -1286,7 +1288,11 @@ module Cnvrg
           compression_path = get_compression_path
           tar_path = "#{compression_path}#{@dataset.slug}_#{commit_sha1}.tar.gz"
           tar_files_path = "#{home_dir}/.cnvrg/tmp/#{@dataset.slug}_#{commit_sha1}.txt"
-          tar_files = (result["added"] + result["updated_on_local"]).join("\n")
+          files_to_upload = result["added"] + result["updated_on_local"]
+          if File.exist? (@dataset.local_path+"/.cnvrgignore")
+            files_to_upload << ".cnvrgignore"
+          end
+          tar_files = (files_to_upload).join("\n")
           File.open(tar_files_path, 'w') { |f| f.write tar_files }
           ignore_files_path = nil
           if !ignore_list.nil?
@@ -5258,17 +5264,17 @@ module Cnvrg
             exit(1)
           end
 
-          if !Helpers.internet_connection?
-            wait_offline = agree "Seems like you're offline, wait until your'e back online?", Thor::Shell::Color::YELLOW
-            if wait_offline
-              say "Waiting until your'e online..", Thor::Shell::Color::BLUE
-              while !Cnvrg::Helpers.internet_connection?
-              end
-            else
-              say "you seem to be offline, please check your internet connection", Thor::Shell::Color::RED
-              exit(0)
-            end
-          end
+          # if !Helpers.internet_connection?
+          #   wait_offline = agree "Seems like you're offline, wait until your'e back online?", Thor::Shell::Color::YELLOW
+          #   if wait_offline
+          #     say "Waiting until your'e online..", Thor::Shell::Color::BLUE
+          #     while !Cnvrg::Helpers.internet_connection?
+          #     end
+          #   else
+          #     say "you seem to be offline, please check your internet connection", Thor::Shell::Color::RED
+          #     exit(0)
+          #   end
+          # end
 
           config = YAML.load_file(File.expand_path('~')+"/.cnvrg/config.yml")
           version_date = config.to_h[:version_last_check]

data/lib/cnvrg/dataset.rb

@@ -4,6 +4,8 @@ module Cnvrg
     attr_reader :slug, :owner, :title, :local_path, :working_dir
 
     RemoteURL ||= "https://cnvrg.io"
+    IDXParallelThreads ||= 15
+
 
     def initialize(project_home)
       begin
@@ -178,7 +180,7 @@ module Cnvrg
       url = Cnvrg::Helpers.remote_url
       "#{url}/#{self.owner}/datasets/#{self.slug}"
     end
-    def generate_idx(list_ignore)
+    def generate_idx(list_ignore=nil)
       if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
         old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
       else
@@ -192,7 +194,7 @@ module Cnvrg
       list_ignore = self.get_ignore_list()
     end
 
-      list.each do |e|
+      Parallel.map(list, in_threads: IDXParallelThreads) do |e|
         label = e.gsub(self.local_path + "/", "")
         if File.directory? e
           if list_ignore.include? label

data/lib/cnvrg/project.rb

@@ -4,6 +4,8 @@ module Cnvrg
     attr_reader :slug, :owner, :title, :local_path, :working_dir
 
     RemoteURL ||= "https://cnvrg.io"
+    IDXParallelThreads ||= 15
+
 
     def initialize(project_home)
       begin
@@ -235,9 +237,8 @@ module Cnvrg
         return true
       end
     def self.verify_cnvrgignore_exist(project_name,remote)
-      if remote
-        path = ".cnvrgignore"
-      else
+      path = ".cnvrgignore"
+      if !File.exist? path
         path = "#{project_name}/.cnvrgignore"
       end
       ignore_exits = File.exist? path
@@ -321,7 +322,7 @@ module Cnvrg
         tree_idx = Hash.new(0)
         list = Dir.glob("#{self.local_path}/**/*", File::FNM_DOTMATCH).reject { |x| (x =~ /\/\.{1,2}$/) or (x =~ /^#{self.local_path}\/\.cnvrg\/*/) or (x =~/^#{self.local_path}\/\.cnvrgignore.conflict*/) and not (x =~/^#{self.local_path}\/\.cnvrgignore/) }
         list_ignore = self.get_ignore_list()
-        list.each do |e|
+        Parallel.map(list, in_threads: IDXParallelThreads) do |e|
           label = e.gsub(self.local_path + "/", "")
           ignore_label = label.gsub("/","//")
           if list_ignore.include? ignore_label

data/lib/cnvrg/version.rb

@@ -1,4 +1,4 @@
 module Cnvrg
-  VERSION = '0.3.0'
+  VERSION = '0.3.1'
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Yochay Ettun