cnvrg 0.7.7 → 0.7.9

data/lib/cnvrg/files.rb

@@ -16,7 +16,121 @@ module Cnvrg
       @project_slug = project_slug
       @owner = owner
       @base_resource = "users/#{owner}/projects/#{project_slug}/"
-      @project_home = project_home
+      @project_home = project_home.presence || Cnvrg::CLI.get_project_home
+    end
+
+    def get_upload_options(number_of_items: 0, progress: false)
+      options = {
+          in_processes: Cnvrg::CLI::ParallelProcesses,
+          in_thread: Cnvrg::CLI::ParallelThreads,
+          isolation: true
+      }
+      if progress
+        options[:progress] = {
+            :title => "Upload Progress",
+            :progress_mark => '=',
+            :format => "%b>>%i| %p%% %t",
+            :starting_at => 0,
+            :total => number_of_items,
+            :autofinish => true
+        }
+      end
+      options
+    end
+
+    def upload_files_old(files_list, commit_sha1, progress: nil)
+      Parallel.map(files_list) do |file|
+        Cnvrg::Helpers.try_until_success{self.upload_file("#{@project_home}/#{file}", file, commit_sha1)}
+        progress.progress += 1
+      end
+    end
+
+    def upload_multiple_files(files_list, commit_sha1, progress: nil)
+      #open files on the server.
+      Cnvrg::Logger.log_info("Uploading project files")
+      return if files_list.blank?
+      if Cnvrg::Helpers.server_version < 1
+        return self.upload_files_old(files_list, commit_sha1, progress: progress)
+      end
+      files_list = files_list.map{|x| [x,self.parse_file(x)]}.to_h
+      resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST', {files: files_list, commit: commit_sha1})
+      unless Cnvrg::CLI.is_response_success(resp, false)
+        raise Exception.new("Cant upload files to the server.")
+      end
+
+      # resolve bucket
+      res = resp['result']
+      files = res['files']
+      props = Cnvrg::Helpers.get_s3_props(res)
+      client = props[:client]
+      bucket = props[:bucket]
+      upload_options = props[:upload_options]
+      s3_bucket = Aws::S3::Resource.new(client: client).bucket(bucket)
+      #upload files
+      files.keys.map do |file|
+      # Parallel.map(files.keys, self.get_upload_options) do |file|
+        resp = Cnvrg::Helpers.try_until_success{self.upload_single_file(files[file].merge(files_list[file]), s3_bucket, options: upload_options)}
+        raise Exception.new("Cant upload #{file}") unless resp
+        progress.progress += 1 if progress.present?
+      end
+
+      #save files on the server.
+      blob_ids = files.values.map {|f| f['bv_id']}
+      resp = Cnvrg::API.request(@base_resource + "upload_files_save", 'POST', {blob_ids: blob_ids, commit: commit_sha1})
+      unless Cnvrg::CLI.is_response_success(resp, false)
+        raise Exception.new("Cant save uploaded files to the server.")
+      end
+    end
+
+
+    def delete_files_from_server_old(files, commit_sha1)
+      #files are absolute path here.
+      files.each do |file|
+        if file.ends_with? '/'
+          #dir
+          self.delete_dir(file, commit_sha1)
+        else
+          #file
+          self.delete_file(file, commit_sha1)
+        end
+      end
+    end
+
+    def delete_files_from_server(files, commit_sha1)
+      #files are absolute path files here. ^^
+      if Cnvrg::Helpers.server_version < 1
+        return self.delete_files_from_server_old(files, commit_sha1)
+      end
+      #convert files to relative path
+      files = files.map{|file| file.gsub(/^#{@project_home + "/"}/, "")}
+      return if files.blank?
+      resp = Cnvrg::API.request(@base_resource + "delete_files", 'DELETE', {files: files, commit: commit_sha1})
+      unless Cnvrg::CLI.is_response_success(resp, false)
+        raise Exception.new("Cant delete the following files from the server.")
+      end
+    end
+
+    def upload_single_file(file, bucket, options: {})
+      path = file['path']
+      absolute_path = file[:absolute_path]
+      resp = bucket.object(path).
+          upload_file(absolute_path, options)
+      unless resp
+        raise Exception.new("Cant upload #{absolute_path}")
+      end
+      resp
+    end
+
+    def parse_file(file)
+      abs_path = "#{@project_home}/#{file}"
+      return {relative_path: file, absolute_path: abs_path} if file.ends_with? '/'
+      file_name = File.basename(file)
+      file_size = File.size abs_path
+      mime_type = MimeMagic.by_path(abs_path)
+      content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
+      sha1 =  OpenSSL::Digest::SHA1.file(abs_path).hexdigest
+
+      {relative_path: file, absolute_path: abs_path, file_name: file_name, file_size: file_size, content_type: content_type, sha1: sha1}
     end
 
     def upload_file(absolute_path, relative_path, commit_sha1)
@@ -30,24 +144,9 @@ module Cnvrg
                                                                                      file_size: file_size, file_content_type: content_type, sha1: sha1,
                                                                                       new_version:true,only_large:true})
 
-
       if Cnvrg::CLI.is_response_success(upload_resp, false)
-
-        path = upload_resp["result"]["path"]
         s3_res = upload_large_files_s3(upload_resp, absolute_path)
-
-        # if file_size.to_f>= Cnvrg::Files::LARGE_FILE.to_f
-        #   s3_res = upload_large_files_s3(upload_resp, absolute_path)
-        # else
-        #   s3_res = upload_small_files_s3(path, absolute_path, content_type)
-        # end
         return s3_res
-        # if s3_res
-        # update_s3_resp = Cnvrg::API.request(@base_resource + "update_s3", 'POST', {path: path, commit_id: upload_resp["result"]["commit_id"],
-        #                                                           blob_id: upload_resp["result"]["id"]})
-        # is_suc = Cnvrg::CLI.is_response_success(update_s3_resp, false)
-        #
-        # return is_suc
       end
       return false
 
@@ -248,14 +347,78 @@ module Cnvrg
       end
 
     rescue => e
-      puts e
+      Cnvrg::Logger.log_error(e)
       return false
+    end
+
+    def resolve_bucket(response)
+      begin
+        sts_path = response["path_sts"]
+        sts_body = self.download_and_read(sts_path)
+        split = sts_body.split("\n")
+        key = split[0]
+        iv = split[1]
+        access = Cnvrg::Helpers.decrypt(key, iv, response["sts_a"])
+
+        secret = Cnvrg::Helpers.decrypt(key, iv, response["sts_s"])
+
+        session = Cnvrg::Helpers.decrypt(key, iv, response["sts_st"])
+        region = Cnvrg::Helpers.decrypt(key, iv, response["region"])
+
+        bucket = Cnvrg::Helpers.decrypt(key, iv, response["bucket"])
+        Cnvrg::Logger.log_info("Resolving bucket #{bucket}, region: #{region}")
+        is_s3 = response["is_s3"]
+        if is_s3 or is_s3.nil?
+          client = Aws::S3::Client.new(
+              :access_key_id => access,
+              :secret_access_key => secret,
+              :session_token => session,
+              :region => region,
+              :use_accelerate_endpoint => true,
+              :http_open_timeout => 60, :retry_limit => 20)
+        else
+          endpoint = Cnvrg::Helpers.decrypt(key, iv, response["endpoint"])
+          client = Aws::S3::Client.new(
+              :access_key_id => access,
+              :secret_access_key => secret,
+              :region => region,
+              :endpoint => endpoint, :force_path_style => true, :ssl_verify_peer => false,
+              :use_accelerate_endpoint => false,
+              :server_side_encryption => 'AES256',
+              :http_open_timeout => 60, :retry_limit => 20)
+        end
 
+        s3 = Aws::S3::Resource.new(client: client)
+        s3.bucket(bucket)
+      rescue => e
+        Cnvrg::Logger.log_error(e)
+        Cnvrg::Logger.log_method(bind: binding)
+      end
+    end
 
+    def download_and_read(path)
+      body = nil
+      retries = 0
+      success= false
+      while !success and retries < 20
+        begin
+          if !Helpers.is_verify_ssl
+            body = open(path, {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}).read
+          else
+            body = open(path).read
+          end
+          success = true
+        rescue => e
+          retries +=1
+          sleep(1)
+        end
+      end
+      body
     end
 
     def upload_large_files_s3(upload_resp, file_path)
       begin
+        return true if upload_resp['result']['already_exists'].present?
           sts_path = upload_resp["result"]["path_sts"]
 
           retries = 0
@@ -360,13 +523,13 @@ module Cnvrg
 
       end
 
-      def delete_file(absolute_path, relative_path, commit_sha1)
-        response = Cnvrg::API.request(@base_resource + "delete_file", 'DELETE', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
+      def delete_file(relative_path, commit_sha1)
+        response = Cnvrg::API.request(@base_resource + "delete_file", 'DELETE', {relative_path: relative_path, commit_sha1: commit_sha1})
         return Cnvrg::CLI.is_response_success(response, false)
       end
 
-      def delete_dir(absolute_path, relative_path, commit_sha1)
-        response = Cnvrg::API.request(@base_resource + "delete_dir", 'DELETE', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
+      def delete_dir(relative_path, commit_sha1)
+        response = Cnvrg::API.request(@base_resource + "delete_dir", 'DELETE', {relative_path: relative_path, commit_sha1: commit_sha1})
         return Cnvrg::CLI.is_response_success(response, false)
       end
 
@@ -376,6 +539,18 @@ module Cnvrg
       end
 
 
+    def calculate_sha1(files_list)
+      files_list = files_list.map{|file| "#{@project_home}/#{file}"}
+      files_list = files_list.select{|file| !file.ends_with? '/'}
+      #TODO: parallel
+      files_list.map do |file|
+        next [file, nil] unless File.exists? file
+        sha1 = OpenSSL::Digest::SHA1.file(file).hexdigest
+        [file.gsub("#{@project_home}/", ""), sha1]
+      end.to_h
+    end
+
+
       def download_file_s3(absolute_path, relative_path, project_home, commit_sha1=nil, conflict=false)
         begin
           res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path,
@@ -456,73 +631,78 @@ module Cnvrg
 
         end
       end
-    def download_multpile_files_s3(files, project_home)
+
+    def download_files(files, commit, postfix: '', progress: nil)
+      res = Cnvrg::API.request(@base_resource + "download_files", 'POST', {files: files, commit: commit})
+      unless Cnvrg::CLI.is_response_success(res, false)
+        raise Exception.new("Cant download files from the server.")
+      end
+      self.download_multpile_files_s3(res['result'], @project_home, postfix: postfix, progress: progress)
+    end
+
+    def delete_files_local(deleted, conflicted: [], progress: nil)
+      deleted -= conflicted
+      deleted.each{|file| self.delete(file); progress.progress += 1 if progress.present?}
+      conflicted.each{|file| self.delete_conflict(file); progress.progress += 1 if progress.present?}
+    end
+
+    def download_multpile_files_s3(files, project_home, postfix: '', progress: nil)
       begin
-          props = Cnvrg::Helpers.get_s3_props(files)
-          client = props[:client]
-          iv = props[:iv]
-          key = props[:key]
-          bucket = props[:bucket]
-          download_succ_count = 0
-          parallel_options = {
-              :progress => {
-                  :title => "Download Progress",
-                  :progress_mark => '=',
-                  :format => "%b>>%i| %p%% %t",
-                  :starting_at => 0,
-                  :total => files["keys"].size,
-                  :autofinish => true
-              },
-              in_threads: Cnvrg::Helpers.parallel_threads,
-              isolation: true
-          }
-          Parallel.map(files["keys"], parallel_options) do |f|
-
-            file_path = f["name"]
-            if file_path.end_with? "/"
-              # dir
-              if download_dir(file_path, file_path, project_home)
-                download_succ_count += 1
-              else
-                return Cnvrg::Result.new(false,"Could not create directory: #{file_path}")
-                raise Parallel::Kill
-              end
+        props = Cnvrg::Helpers.get_s3_props(files)
+        client = props[:client]
+        iv = props[:iv]
+        key = props[:key]
+        bucket = props[:bucket]
+        download_succ_count = 0
+        parallel_options = {
+            in_threads: Cnvrg::Helpers.parallel_threads,
+            isolation: true
+        }
+        Parallel.map(files["keys"], parallel_options) do |f|
+
+          file_path = f["name"]
+          if file_path.end_with? "/"
+            # dir
+            if download_dir(file_path, file_path, project_home)
+              download_succ_count += 1
             else
-              # blob
-              begin
-                if not File.exists?(project_home+"/"+File.dirname(file_path))
-                  FileUtils.makedirs(project_home+"/"+File.dirname(file_path))
-                end
+              return Cnvrg::Result.new(false,"Could not create directory: #{file_path}")
+              raise Parallel::Kill
+            end
+          else
+            file_path += postfix
+            # blob
+            begin
+              if not File.exists?(project_home+"/"+File.dirname(file_path))
+                FileUtils.makedirs(project_home+"/"+File.dirname(file_path))
+              end
 
               file_key =  Cnvrg::Helpers.decrypt(key,iv, f["path"])
               resp = false
-              File.open(project_home+"/"+file_path, 'w+') do |file|
-                resp = client.get_object({bucket:bucket,
+              Cnvrg::Helpers.try_until_success(tries: 10) {
+                File.open(project_home+"/"+file_path, 'w+') do |file|
+                  resp = client.get_object({bucket:bucket,
                                           key:file_key}, target: file)
-                #TODO update idx here!
-              end
-              if resp
-                download_succ_count +=1
-              else
-                return Cnvrg::Result.new(false,"Could not create file: #{file_path}")
-              end
+                end
+              }
+              progress.progress += 1 if progress.present?
+              download_succ_count += 1
 
 
 
-              rescue => e
-                return Cnvrg::Result.new(false,"Could not create file: #{file_path}", e.message, e.backtrace)
-                raise Parallel::Kill
-              end
+            rescue => e
+              return Cnvrg::Result.new(false,"Could not create file: #{file_path}", e.message, e.backtrace)
+              raise Parallel::Kill
+            end
 
 
 
-              end
           end
+        end
         if download_succ_count == files["keys"].size
           return Cnvrg::Result.new(true,"Done.\nDownloaded #{download_succ_count} files")
         end
       rescue => e
-
         return Cnvrg::Result.new(false,"Could not download some files", e.message, e.backtrace)
       end
 
@@ -530,8 +710,7 @@ module Cnvrg
 
 
     end
-
-      def download_file(absolute_path, relative_path, project_home, conflict=false)
+    def download_file(absolute_path, relative_path, project_home, conflict=false)
         res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path})
         Cnvrg::CLI.is_response_success(res, false)
         if res["result"]
@@ -658,9 +837,9 @@ module Cnvrg
       return true
 
     end
-      def start_commit(new_branch,force:false, exp_start_commit:nil, job_slug: nil, job_type: nil, start_commit: nil)
+      def start_commit(new_branch,force:false, exp_start_commit:nil, job_slug: nil, job_type: nil, start_commit: nil, message: nil)
         response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {project_slug: @project_slug, new_branch: new_branch,force:force,
-                                                                                username: @owner, exp_start_commit:exp_start_commit, job_slug: job_slug, job_type: job_type, start_commit: start_commit})
+                                                                                username: @owner, exp_start_commit:exp_start_commit, job_slug: job_slug, job_type: job_type, start_commit: start_commit, message: message})
         Cnvrg::CLI.is_response_success(response,false)
         return response
       end
@@ -680,9 +859,17 @@ module Cnvrg
     end
 
     def delete(file)
+      file = "#{@project_home}/#{file}" unless File.exists? file
+      return unless File.exists? file
       FileUtils.rm_rf(file)
     end
 
+    def delete_conflict(file)
+      file = "#{@project_home}/#{file}" unless File.exists? file
+      return unless File.exists? file
+      File.rename(file, "#{file}.deleted")
+    end
+
     def handle_compare_idx(compared, resolver: {})
       begin
         all_files = compared.values.flatten.uniq
@@ -710,7 +897,7 @@ module Cnvrg
             self.download_file(file_path: "#{file}.conflict", key: key, iv: iv, bucket: bucket, path: files[file]['path'], client: client)
             next
           end
-          if compared['updated_on_server'].include? file or compared['added'].include? file
+          if compared['updated_on_server'].include? file
             self.download_file(file_path: file, key: key, iv: iv, bucket: bucket, path: files[file]['path'], client: client)
             next
           end

data/lib/cnvrg/flow.rb

@@ -0,0 +1,75 @@
+module Cnvrg
+  class Flows
+    def initialize(project_path, fullpath)
+      @project = Cnvrg::Project.new(project_path)
+      @fullpath = fullpath
+      @tasks = {}
+      @relations = {}
+      @title = nil
+      @slug = nil
+      @base_resource = @project.base_resource + "flows"
+
+      self.reload_flow
+    end
+
+    def self.create_flow(path, flow)
+      File.open(path, "w"){|file| file.write flow.to_yaml}
+    end
+
+    def get_flow
+      unless File.exists? @fullpath
+        raise StandardError.new("Cant find flow in #{@fullpath}")
+      end
+      YAML.load_file(@fullpath)
+    end
+
+    def set_flow(new_flow)
+      File.open(@fullpath, "w"){|file| file.write new_flow.to_yaml}
+    end
+
+    def set_flow_slug(slug)
+      flow = self.get_flow
+      flow[:slug] = slug
+      self.set_flow(flow)
+    end
+
+    def reload_flow
+      flow = self.get_flow
+      @title = flow[:title]
+      @slug = flow[:slug]
+      @relations = flow[:relations]
+      local_tasks = flow[:tasks] || {}
+      @relations.each do |relation|
+        relation.values.each do |task|
+          if local_tasks[task].present?
+            @tasks[task] = Cnvrg::Task.new(@project.local_path, content: local_tasks[task])
+          else
+            @tasks[task] = Cnvrg::Task.new(@project.local_path, path: task)
+          end
+        end
+      end
+    end
+
+    def run
+      resp = Cnvrg::API.request(@base_resource, 'POST', {data: to_api})
+      Cnvrg::CLI.is_response_success(resp, true)
+      flow_slug = resp['result']['flow']
+      self.set_flow_slug(flow_slug)
+      url = Cnvrg::Helpers.remote_url + resp['result']['url']
+      return url
+    end
+
+
+    private
+    def to_api
+      {
+          relations: @relations,
+          tasks: @tasks.keys.map{|task| [task, @tasks[task].to_api]}.to_h,
+          title: @title,
+          slug: @slug
+      }
+    end
+
+
+  end
+end