cnvrg 0.6.1 → 0.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 42022fa13f8fd836b23c053f1fb307c7187603dd
-  data.tar.gz: 6fe9747627cf95a833a5efe4f3b93a35457b0cc5
+  metadata.gz: f4256d8fe0490792ce6c903208f55fb334c090fb
+  data.tar.gz: 90297a245a2df22569c0f1b11135d09d35f7d1e8
 SHA512:
-  metadata.gz: 4e9037eb3b8536c4a6c54493beb6c9826de49a90cacab42586ea113bd7d067825d6289f4335e1f45bbb02e7b0df9935aca7314e7fce110522d88fa070d406935
-  data.tar.gz: 0183b26af7ec67875fc47a3c1f4455c10a1c42e6cc33a2958456f65c174230fd17e7ce421bce0547b48ef89cc3893eb3a54e21e1d1f492d1cb2a17219c88eab1
+  metadata.gz: 0abffec7afcb1f5ed6cbbf764cea7826e2aa267e3f5e7f80ba18b5c3338604f4bdb36ad10e8009438b2d2ae437406feed79d4273a7b8f9cfb76f7b2fac43befc
+  data.tar.gz: 8ba624643cbb0772f9fb08880ddb285012a936feb31fae3ef3ca9b0059fcc95da94801e68838a273c82f357e98c42682db96fb94488b295f0a40663ff629c249

data/lib/cnvrg/api.rb

@@ -105,6 +105,36 @@ module Cnvrg
                 else
                     response
                 end
+            when 'POST_JSON'
+              conn.options.timeout = 4200
+              conn.options.open_timeout =4200
+              conn.headers['Content-Type'] = "application/json"
+              new_data = JSON.dump(data)
+
+              retries = 0
+              success = false
+
+              while !success and retries < 20
+                begin
+                  response = conn.post "#{resource}", new_data
+                  success = true
+
+                rescue => e
+                  sleep(5)
+                  retries +=1
+                end
+              end
+              if !success
+                return false
+              end
+              if response.to_hash[:status] == 404
+                return false
+              end
+              if parse_request == true
+                JSON.parse(response.body)
+              else
+                response
+              end
             when 'POST_FILE'
                 conn = Faraday.new do |fr|
                     fr.headers['Auth-Token'] = @pass

data/lib/cnvrg/cli.rb

@@ -842,20 +842,18 @@ module Cnvrg
                                            :progress_mark => '=',
                                            :format => "%b>>%i| %p%% %t",
                                            :starting_at => 0,
+                                           :total => files_count,
                                            :autofinish => true)
           while files['keys'].length > 0
-            @files.download_multiple_files_s3(files, dataset_home)
+            @files.download_multiple_files_s3(files, dataset_home, progressbar: progressbar)
             downloaded_files += files['keys'].length
-            progressbar.progress = (downloaded_files * 100 / files_count)
             files = @files.get_clone_chunk(commit: commit, latest_id: files['latest'])
           end
           if downloaded_files == files_count
             Dataset.verify_cnvrgignore_exist(dataset_name, false)
             @dataset = Dataset.new(dataset_home)
-            @dataset.write_idx(nil, commit)
+            @dataset.write_idx(nil, commit) #nil means, generate idx
             log_message("#{check} Clone finished successfully", Thor::Shell::Color::GREEN)
-          else
-            log_message(result.msg, result.msg_color)
           end
         else
           log_message("Error: Couldn't create directory: #{dataset_name}", Thor::Shell::Color::RED)
@@ -1938,10 +1936,12 @@ module Cnvrg
     method_option :tags, :type => :boolean, :aliases => ["--tags"], :desc => "upload file tags", :default => false
     method_option :parallel, :type => :numeric, :aliases => ["-p", "--parallel"], :desc => "uparallel upload at the same time", :default => 15
     def upload_data_new(new_branch,  verbose,sync,force, tags, chunk_size)
-      commit = invoke :start_commit_data,[], :new_branch=> new_branch, :direct=>false, :force =>force
+      commit = invoke :start_commit_data,[], :new_branch=> new_branch, :direct=>false, :force =>force, :chunk_size => chunk_size
       upload_res = invoke :upload_data_files,[commit],:new_branch=>new_branch, :verbose =>verbose, :force =>force, :sync =>sync, :chunk_size => chunk_size
-      invoke :end_commit_data,[commit] , :new_branch=>new_branch,  :force =>force, :success => upload_res
-
+      if !upload_res
+        return
+      end
+      invoke :end_commit_data,[commit, success: upload_res] , :new_branch=>new_branch,  :force =>force
       if tags
         log_message('Uploading Tags', Thor::Shell::Color::BLUE)
         dataset_dir = is_cnvrg_dir(Dir.pwd)
@@ -1970,6 +1970,7 @@ module Cnvrg
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
     method_option :direct, :type => :boolean, :aliases => ["-d","--direct"], :desc => "was called directed", :default => true
     method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
+    method_option :chunk_size, :type => :numeric, :aliases => ["-ch"], :default => 0
 
     def start_commit_data()
       begin
@@ -1979,30 +1980,38 @@ module Cnvrg
         direct = options[:direct]
         new_branch = options["new_branch"] || false
         force = options["force"] || false
+        chunk_size = options["chunk_size"] || false
         commit_sha1 = nil
         @dataset = Dataset.new(dataset_dir)
         @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
-        next_commit = @dataset.get_next_commit() #if there was a partial commit..
-        if next_commit.present? #keep from existing commit.
-          resp = @files.get_commit(next_commit)
-          if resp["result"]["status"].eql? "new"
-            resp = @files.start_commit(new_branch,force)
-            commit_sha1 = resp["result"]["commit_sha1"]
-            @dataset.set_next_commit(commit_sha1)
-          else
-            commit_sha1 = resp["result"]["commit_sha1"]
-          end
-        else
-          resp = @files.start_commit(new_branch,force)
-          commit_sha1 = resp["result"]["commit_sha1"]
-          @dataset.set_next_commit(commit_sha1)
+        next_commit = @dataset.get_next_commit #if there was a partial commit..
+        chunks = (@dataset.list_all_files.length.to_f / chunk_size).ceil
+        resp = @files.start_commit(new_branch, force, chunks: chunks, current_commit: next_commit || @dataset.last_local_commit)
+        if !resp['result']['can_commit']
+          log_message("Cant upload files because a new version of this dataset exists, please download it or upload with --force", Thor::Shell::Color::RED)
+          exit(1)
         end
+        #  if next_commit.present? #keep from existing commit.
+        #           resp = @files.get_commit(next_commit)
+        #           if resp["result"]["status"].eql? "new"
+        #             resp = @files.start_commit(new_branch,force, chunks: chunks)
+        #             commit_sha1 = resp["result"]["commit_sha1"]
+        #             @dataset.set_next_commit(commit_sha1)
+        #           else
+        #             commit_sha1 = resp["result"]["commit_sha1"]
+        #           end
+        #         else
+        #           resp = @files.start_commit(new_branch,force, chunks: chunks)
+        #           commit_sha1 = resp["result"]["commit_sha1"]
+        #           @dataset.set_next_commit(commit_sha1)
+        #         end
+        commit_sha1 = resp["result"]["commit_sha1"]
+        @dataset.set_next_commit(commit_sha1)
         if direct
           puts commit_sha1
         end
         return commit_sha1
       rescue => e
-        puts e
           if commit_sha1.present?
             @dataset.set_next_commit(commit_sha1)
           end
@@ -2018,9 +2027,8 @@ module Cnvrg
     desc 'end_commit', 'start data commit'
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
     method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
-    method_option :success, :type => :boolean,  :default => true, :hide=>true
 
-    def end_commit_data(commit)
+    def end_commit_data(commit, success: true)
       begin
         verify_logged_in(true)
         log_start(__method__, args, options)
@@ -2028,21 +2036,14 @@ module Cnvrg
         @dataset = Dataset.new(dataset_dir)
         @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
         force = options["force"] || false
-        resp = @files.end_commit(commit, force, options[:success])
-        if (resp and resp["result"])
-          if  !options[:success]
-            @dataset.set_next_commit(resp["result"]["commit_sha1"])
-            @dataset.remove_next_commit
-          else
-            check = Helpers.checkmark
-            @dataset.set_next_commit(commit)
-            @dataset.remove_next_commit
-            log_message("#{check} Data files were updated successfully", Thor::Shell::Color::GREEN)
-          end
-
-          end
-
 
+        resp = @files.end_commit(commit, force, success: success)
+        if (resp and resp["result"])
+          check = Helpers.checkmark
+          @dataset.set_next_commit(commit)
+          @dataset.remove_next_commit
+          log_message("#{check} Data files were updated successfully", Thor::Shell::Color::GREEN)
+        end
       rescue => e
         puts e
       rescue SignalException
@@ -2105,49 +2106,47 @@ module Cnvrg
         dataset_dir = is_cnvrg_dir(Dir.pwd)
         @dataset = Dataset.new(dataset_dir)
         @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
-        next_commit = @dataset.get_next_commit()
-        if new_commit.blank? and next_commit.blank?
+        new_commit ||= @dataset.get_next_commit()
+        if new_commit.blank?
           log_message("You must specify commit, run start_commit to create new commit", Thor::Shell::Color::RED)
-          exit(1)
+          return false
         end
-
         force = options[:force] || false
         chunk_size = options[:chunk_size]
         chunk_size = [chunk_size, 1].max
         new_branch = options["new_branch"] || false
+        new_tree = {}
         force = options["force"] || false
         parallel_threads = options["parallel"]  || ParallelThreads
-        new_tree = {}
+        all_files = @dataset.list_all_files
         progressbar = ProgressBar.create(:title => "Upload Progress",
                                          :progress_mark => '=',
                                          :format => "%b>>%i| %p%% %t",
                                          :starting_at => 0,
+                                         :total => all_files.length,
                                          :autofinish => true)
-        all_files = @dataset.list_all_files
-        files_count = all_files.count
-        finished_chunks = 0
         files_uploaded = 0
-        chunked_files  = all_files.each_slice(chunk_size)
-        chunked_files.each_with_index do |list_files,i|
-          log_message("generating chunk #{i} / #{chunked_files.size}", Thor::Shell::Color::BLUE, false)
+        all_files.each_slice(chunk_size).each do |list_files|
           temp_tree = @dataset.generate_chunked_idx(list_files, threads: parallel_threads)
+          upload_resp = @files.upload_multiple_files(new_commit, temp_tree, threads: parallel_threads, force: force, new_branch: new_branch, progressbar: progressbar)
+
+          if upload_resp.is_a? Cnvrg::Result
+            log_message(upload_resp.msg, upload_resp.msg_color)
+
+            return false
+
+          else
+            files_uploaded += upload_resp
+          end
 
-          log_message("uploading chunk #{i} / #{chunked_files.size}", Thor::Shell::Color::BLUE, false)
-          files_uploaded += @files.upload_multiple_files(next_commit, temp_tree, threads: parallel_threads, force: force)
           temp_tree.each do |k,v|
-            if v.present?
-              new_tree[k] = {sha1: v.try(:fetch, :sha1, nil), commit_time: nil}
-            else
-              new_tree[k] = nil
-            end
+            new_tree[k] = (v.present?) ? {sha1: v.try(:fetch, :sha1, nil), commit_time: nil} : nil
           end
-          @dataset.write_idx(new_tree, next_commit)
-          finished_chunks += 1
-          progressbar.progress = [(finished_chunks * chunk_size*100) / files_count, 100].min
+          @dataset.write_idx(new_tree, new_commit)
         end
         return files_uploaded > 0
       rescue Exception => e
-        log_error(e)
+        puts e
         return false
       end
 
@@ -2387,8 +2386,8 @@ module Cnvrg
           if git_output_dir.ends_with? "/"
             git_output_dir = git_output_dir[0..-2]
           end
-          spec_files_to_upload = Dir.glob("#{git_output_dir}/**/*", File::FNM_DOTMATCH).flatten
-          spec_files_to_upload = spec_files_to_upload.select{|x| !x.ends_with? "/."}
+          list  = @project.generate_output_dir(output_dir)
+          spec_files_to_upload  = list
           if spec_files_to_upload.blank?
             log_message("#{check} Project is up to date", Thor::Shell::Color::GREEN, (((options["sync"] or sync) and !direct) ? false : true))
               return true
@@ -2398,6 +2397,7 @@ module Cnvrg
         end
 
 
+
         if ignore.nil? or ignore.empty?
           ignore = ignore_list
         end
@@ -2416,6 +2416,7 @@ module Cnvrg
 
         result = @project.compare_idx(new_branch, force: force, deploy: options["deploy"],in_exp:in_exp, specific_files: spec_files_to_upload)
         commit = result["result"]["commit"]
+
         if !link
           if (result["result"]["new_version_exist"] and !force) or ((commit != @project.last_local_commit and !@project.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?) and !force)
             log_message("Remote server has an updated version, please run `cnvrg download` first, or alternatively: `cnvrg sync`", Thor::Shell::Color::BLUE)
@@ -2445,7 +2446,7 @@ module Cnvrg
           log_message("Syncing files", Thor::Shell::Color::BLUE, ((options["sync"] or sync)) ? false : true)
         end
         # Start commit
-        if in_exp
+        if in_exp || (job_slug.present? and job_type.present?)
           exp_start_commit = @project.last_local_commit
         else
           exp_start_commit = nil
@@ -2799,6 +2800,8 @@ module Cnvrg
             log_message("#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN, ((sync or options["sync"]) ? false : true))
           end
           return true
+        else
+          @dataset.update_idx_with_commit!(commit_to_download)
         end
       rescue => e
         log_message("Error occurred, \nAborting", Thor::Shell::Color::BLUE)
@@ -2811,7 +2814,7 @@ module Cnvrg
     end
 
     desc 'download in git project', 'Download other files', :hide =>true
-    def download_in_git(commit_sha1)
+    def download_in_git(*commit_sha1)
       begin
         verify_logged_in(true)
         log_start(__method__, args, options)
@@ -2826,7 +2829,11 @@ module Cnvrg
         end
 
         begin
-          download_result = @files.download_multiple_files_s3(files, project_home)
+          if files["keys"].blank?
+            log_message("No New files to download", Thor::Shell::Color::GREEN)
+            return
+          end
+          download_result = @files.download_multpile_files_s3(files, project_home)
           if download_result
             log_message("Done.\nDownloaded  finished successfully", Thor::Shell::Color::GREEN)
           else
@@ -2836,7 +2843,6 @@ module Cnvrg
 
             rescue => e
           log_error(e)
-          puts e.backtrace
           log_message("Error while trying to download ", Thor::Shell::Color::RED)
           return
         end
@@ -2850,6 +2856,8 @@ module Cnvrg
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
     method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
     method_option :ignore, :type => :string, :aliases => ["-i"], :desc => "ignore following files", :default => ""
+    method_option :git, :type => :boolean, :aliases => ["--git"], :desc => "download from git projects' commits", :default => false
+    method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :desc => "download a specific commit", :default => nil
 
     def download(sync = false, ignore_list = "", in_exp=false)
       begin
@@ -2858,6 +2866,11 @@ module Cnvrg
         project_home = get_project_home
         @project = Project.new(project_home)
         @files = Cnvrg::Files.new(@project.owner, @project.slug)
+        git = options["git"]
+        commit = options["commit"]
+        if git or @project.is_git
+          return download_in_git(commit)
+        end
         ignore = options[:ignore] || ""
         if ignore.nil? or ignore.empty?
           ignore = ignore_list
@@ -3236,7 +3249,7 @@ module Cnvrg
       if run_download
         invoke :download, [true, "", in_exp ], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true
       end
-      invoke :upload, [false, true,  direct, "",options['exp_slug'] ], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true,
+      invoke :upload, [false, true,  direct, "",in_exp,options[:force],  options["output_dir"],job_type, job_slug ], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true,
              :ignore => options[:ignore], :force => options[:force], :message => options[:message], :deploy => options["deploy"], :return_id => options["return_id"],
              :files => options["files"], :output_dir => options["output_dir"], :job_slug => job_slug, :job_type => job_type
 

data/lib/cnvrg/datafiles.rb

@@ -20,12 +20,13 @@ module Cnvrg
       @base_resource = "users/#{owner}/datasets/#{dataset_slug}/"
     end
 
-    def upload_multiple_files(commit_sha1, tree, threads: ParallelThreads, force: false, new_branch: false)
+    def upload_multiple_files(commit_sha1, tree, threads: ParallelThreads, force: false, new_branch: false, progressbar: nil)
       random_file_name = (0...8).map { (65 + rand(26)).chr }.join #needed to create a file for post_file..
       # each file have: {sha1: sha1, file_name: file_name, file_size: file_size, content_type: content_type, absolute_path, relative_path} 
 
-      #this call should also implement compare_idx... 
-      upload_resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST', {commit_sha1: commit_sha1, tree: tree, force: force, is_branch: new_branch})
+      #this call should also implement compare_idx...
+      begin
+      upload_resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST_JSON', {commit_sha1: commit_sha1, tree: tree, force: force, is_branch: new_branch})
       return Cnvrg::Result.new(false, "Failed to upload files") unless Cnvrg::CLI.is_response_success(upload_resp, false)
       results = upload_resp['result'].with_indifferent_access
       props = Cnvrg::Helpers.get_s3_props(results)
@@ -34,11 +35,20 @@ module Cnvrg
       upload_options = props[:upload_options]
       s3_bucket = Aws::S3::Resource.new(client: client).bucket(bucket)
       files = results['files']
+      progressbar.progress += tree.keys.length - files.length if progressbar.present?
       Parallel.map((files.keys), {in_threads: threads}) do |k|
         o = tree[k].merge(files[k])
         upload_single_file(o, s3_bucket, upload_options)
+        progressbar.progress += 1 if progressbar.present?
       end
+      blob_ids = files.values.map{|f| f['bv_id']}
+      dirs = tree.keys.select{|k| tree[k].nil? }
+      upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs})
       return files.keys.length
+      rescue => e
+        return Cnvrg::Result.new(false, "Failed to upload files")
+      end
+
     end
 
     def upload_single_file(file, s3_bucket, upload_options={})
@@ -734,10 +744,10 @@ module Cnvrg
       Cnvrg::CLI.is_response_success(response, true)
       return response
     end
-    def start_commit(new_branch,force=false,delete_commit=nil)
+    def start_commit(new_branch,force=false,delete_commit=nil, chunks: 0, current_commit: nil)
       begin
       response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {dataset_slug: @dataset_slug, new_branch: new_branch,force:force,
-                                                                              username: @owner,delete_commit:delete_commit})
+                                                                              username: @owner,current_commit:current_commit, total_chunks: chunks})
       Cnvrg::CLI.is_response_success(response, true)
       return response
       rescue => e
@@ -746,9 +756,9 @@ module Cnvrg
 
     end
 
-    def end_commit(commit_sha1,force, is_success)
+    def end_commit(commit_sha1,force, success: true)
       begin
-        response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1,force:force, success:is_success})
+        response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1,force:force, success: success})
         Cnvrg::CLI.is_response_success(response, true)
         return response
       rescue => e
@@ -828,7 +838,7 @@ module Cnvrg
       
     end
 
-    def download_multiple_files_s3(files, project_home, conflict: false)
+    def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil)
       begin
         props = Cnvrg::Helpers.get_s3_props(files)
         client = props[:client]
@@ -841,11 +851,11 @@ module Cnvrg
             isolation: true
         }
         Parallel.map(files["keys"], parallel_options) do |f|
-
           file_path = f["name"]
+          progressbar.progress += 1 if progressbar.present?
           if file_path.end_with? "/"
             # dir
-            if download_dir(file_path, file_path)
+            if download_dir(project_home, file_path)
               download_succ_count += 1
             else
               return Cnvrg::Result.new(false,"Could not create directory: #{file_path}")
@@ -857,6 +867,8 @@ module Cnvrg
             file_key =  Cnvrg::Helpers.decrypt(key,iv, f["path"])
             resp = false
             file_path = "#{file_path}.conflict" if conflict
+            dirname = File.dirname file_path
+            download_dir(project_home, dirname) unless dirname.eql? "."
             File.open(project_home+"/"+file_path, 'w+') do |file|
               resp = client.get_object({bucket:bucket,
                                         key:file_key}, target: file)

data/lib/cnvrg/dataset.rb

@@ -27,6 +27,9 @@ module Cnvrg
     end
 
     def last_local_commit
+      if  !File.exist? "#{self.local_path}/.cnvrg/idx.yml"
+        return nil
+      end
       idx = YAML.load_file(@local_path + "/.cnvrg/idx.yml")
       return idx[:commit]
     end
@@ -488,7 +491,6 @@ module Cnvrg
       end
       idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
       return idx_hash[:next_commit]
-
     end
     def remove_next_commit()
       if  !File.exist? "#{self.local_path}/.cnvrg/idx.yml"
@@ -501,6 +503,13 @@ module Cnvrg
       File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
 
 
+    end
+    def get_current_commit()
+      if  !File.exist? "#{self.local_path}/.cnvrg/idx.yml"
+        return nil
+      end
+      idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
+      return idx_hash[:commit]
     end
     def list_files(commit, json)
       response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/list", 'GET', {commit_sha1:commit, as_json:json})

data/lib/cnvrg/experiment.rb

@@ -141,7 +141,7 @@ module Cnvrg
       restart = false
       #TODO: remove this later
       fall_number = rand(1..20)
-      if fall_number == 5
+      if [5,15].include? fall_number
         return true
       else
         return false

data/lib/cnvrg/helpers.rb

@@ -196,6 +196,7 @@ parameters:
 
     def decrypt(key,iv,str)
       begin
+
       cipher = OpenSSL::Cipher.new("aes-256-cbc").decrypt
       cipher.key = key
       cipher.iv = Base64.decode64 iv.encode('ascii-8bit')
@@ -262,7 +263,7 @@ parameters:
             :http_open_timeout => 60, :retry_limit => 20)
         use_accelerate_endpoint = true
       else
-        endpoint = Cnvrg::Helpers.decrypt(key,iv, files["endpoint_url"])
+        endpoint = Cnvrg::Helpers.decrypt(key,iv, files["endpoint"])
         client = Aws::S3::Client.new(
             :access_key_id =>access,
             :secret_access_key => secret,

data/lib/cnvrg/project.rb

@@ -317,6 +317,22 @@ module Cnvrg
       config = YAML.load_file(@working_dir+"/.cnvrg/config.yml")
       new_config = config.except(:new_branch)
       File.open(@working_dir+"/.cnvrg/config.yml", "w+") { |f| f.write new_config.to_yaml }
+    end
+    def generate_output_dir(output_dir)
+      upload_list = []
+      list = Dir.glob("#{output_dir}/**/*", File::FNM_DOTMATCH)
+      Parallel.map(list, in_threads: IDXParallelThreads) do |e|
+        next if e.end_with? "/."
+        if File.directory? e
+
+          upload_list << e+"/"
+        else
+          upload_list << e
+          end
+      end
+      upload_list << output_dir + "/"
+      return upload_list
+
     end
 
       def generate_idx(deploy:false)

data/lib/cnvrg/version.rb

@@ -1,4 +1,4 @@
 module Cnvrg
-  VERSION = '0.6.1'
+  VERSION = '0.6.2'
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.6.2
 platform: ruby
 authors:
 - Yochay Ettun
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-02 00:00:00.000000000 Z
+date: 2018-08-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler