cnvrg 1.11.10 → 1.11.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 54dfcb511e29a518772481a3d99a6b30107234c4a218655babc69ddc36496d6d
-  data.tar.gz: 42ea59d4ef4fedef1afd2ba5fb8dcc7e86e439a2e9cc2dfa084409769dc7daf1
+  metadata.gz: cc3f80382d5e1ebdddea819a74e92cd00806a273dfd4d58c484e2a91a7f86932
+  data.tar.gz: cfb22dc8d1548b7c6a1f0af0e588e8c03a62bcad67b06fc8c1573116df7db31b
 SHA512:
-  metadata.gz: 5c82e60bebc3777880873744d66a366cd2a8893443c62b82606f2986afaa5812a6cf7816f0f2fd9ba35f28819baede432e65c89440c7c594e94d767b31f102c5
-  data.tar.gz: e345d41daab3f9109f47f0d56fb0fd0c1d6d0991bb43b01b5718f1b1f35c182af35989eef36030f384cbbc758524656fa1e08c75d5bbc2abccbeae5be26114a9
+  metadata.gz: 3096470b6f4aa425b2804e12b1db724c33700d209387475fbee16977c89ac75d4794f3eea304de454936fca93b5a24df8ba57391f05490fc076abc2c2f3b2a20
+  data.tar.gz: dd3dfd71604961c548d197c72cda3a6dd586103e19d38dabc05cc1af2e8262d84bab4126dfe93fe46f920c8fb345508c6626c42b212a7ad5a02b6cca0893c0d2

data/lib/cnvrg/api.rb

@@ -65,6 +65,16 @@ module Cnvrg
                     response = conn.get "#{resource}", data
                     success = true
                     Cnvrg::API.parse_version(response)
+                    if response.to_hash[:status].to_i != 200
+                      Cnvrg::Logger.log_info("Got back bad status #{response.to_hash[:status]}")
+                    end
+                    if [503, 502, 429].include?(response.to_hash[:status].to_i)
+                      Cnvrg::Logger.log_info("Got back status #{response.to_hash[:status]}, will retry in #{5 * retries} seconds")
+                      success = false
+                      sleep(5 * retries)
+                      retries +=1
+                      next
+                    end
                   rescue => e
                     Cnvrg::Logger.log_error(e)
                     sleep(5)
@@ -95,11 +105,20 @@ module Cnvrg
                         response = conn.put "#{resource}", data.to_json if method.eql? 'PUT'
                         success = true
                         Cnvrg::API.parse_version(response)
-
+                        if response.to_hash[:status].to_i != 200
+                          Cnvrg::Logger.log_info("Got back bad status #{response.to_hash[:status]}")
+                        end
+                        if [503, 502, 429].include?(response.to_hash[:status].to_i)
+                          Cnvrg::Logger.log_info("Got back status #{response.to_hash[:status]}, will retry in #{5 * retries} seconds")
+                          success = false
+                          sleep(5 * retries)
+                          retries +=1
+                          next
+                        end
                     rescue => e
                       Cnvrg::Logger.log_error(e)
-                        sleep(5)
-                        retries +=1
+                      sleep(5)
+                      retries +=1
                     end
                 end
                 if !success

data/lib/cnvrg/datafiles.rb

@@ -11,6 +11,7 @@ module Cnvrg
 
     LARGE_FILE=1024*1024*5
     MULTIPART_SPLIT=10000000
+    RETRIES = ENV['UPLOAD_FILE_RETRIES'].try(:to_i) || 10
 
     attr_reader :base_resource
 
@@ -256,6 +257,7 @@ module Cnvrg
     end
 
     def delete_file_chunk(commit_sha1, regex_list, chunk_size, offset)
+      retry_count = 0
       begin
         resp = Cnvrg::API.request(
             @base_resource + "delete_files_by_chunk",
@@ -268,13 +270,19 @@ module Cnvrg
             }
         )
         unless Cnvrg::CLI.is_response_success(resp, false)
-          Cnvrg::Logger.log_method(bind: binding)
           raise Exception.new("Got an error message from server, #{resp.try(:fetch, "message")}")
         end
         return resp["total_changes"]
       rescue => e
         Cnvrg::Logger.log_method(bind: binding)
         Cnvrg::Logger.log_error(e)
+
+        if retry_count < RETRIES
+          sleep(2**retry_count) # Exponential backoff
+          retry_count += 1
+          retry
+        end
+
         raise e
       end
     end

data/lib/cnvrg/downloader/client.rb

@@ -14,11 +14,13 @@ module Cnvrg
       end
 
       def extract_key_iv(sts_path)
-        count = 20
+        count = 0
         begin
           count += 1
           sts = open(sts_path, {ssl_verify_mode: 0}).read rescue nil
         rescue => e
+          backoff_time_seconds = backoff_time(count)
+          sleep backoff_time_seconds
           Cnvrg::Logger.log_error(e)
           retry if count <= 20
           raise StandardError.new("Cant access storage: #{e.message}")

data/lib/cnvrg/helpers/agent.rb

@@ -75,9 +75,11 @@ class Cnvrg::Helpers::Agent
 
   def exec!
     log_internal("Command: #{@command} with slug: #{@slug} started!")
-    if should_run?
+    if @command.blank?
+      @exit_status = 0
+    elsif should_run?
       send_logs(status: Status::STARTED)
-      periodic_thread
+      periodic_thread_handle = periodic_thread
       execute_command
     else
       @exit_status = 127
@@ -86,6 +88,9 @@ class Cnvrg::Helpers::Agent
     finish_log += " after #{@real_execution_retries} retries" if @real_execution_retries > 0
     log_internal(finish_log)
     send_logs(exit_status: @exit_status, status: Status::FINISHED)
+    if periodic_thread_handle.present?
+      periodic_thread_handle.join
+    end
   end
 
   def get_logs_to_send

data/lib/cnvrg/helpers/executer.rb

@@ -120,17 +120,31 @@ class Cnvrg::Helpers::Executer
   end
 
   def init
-    resp = Cnvrg::API.request(activity_url, "PUT", {stats: executer_stats})
-    machine_activity = resp["machine_activity"]
-    Cnvrg::Logger.log_info("Got back machine activity #{machine_activity}")
-    if machine_activity.present? and @machine_activity != machine_activity
-      Cnvrg::Logger.log_info("Changing to machine activity #{machine_activity}")
-      machine_activity_yml = {slug: machine_activity}
-      File.open("/conf/.machine_activity.yml", "w+") {|f| f.write machine_activity_yml.to_yaml}
-      @machine_activity = machine_activity
+    retries = 0
+    success = false
+    puts("Agent started, connecting to #{Cnvrg::API.get_api}")
+    STDOUT.flush
+    while !success and retries < 100
+      begin
+        resp = Cnvrg::API.request(activity_url, "PUT", {stats: executer_stats})
+        machine_activity = resp["machine_activity"]
+        success = true
+        puts("Connected to server")
+        STDOUT.flush
+        Cnvrg::Logger.log_info("Got back machine activity #{machine_activity}")
+        if machine_activity.present? and @machine_activity != machine_activity
+          Cnvrg::Logger.log_info("Changing to machine activity #{machine_activity}")
+          machine_activity_yml = {slug: machine_activity}
+          File.open("/conf/.machine_activity.yml", "w+") {|f| f.write machine_activity_yml.to_yaml}
+          @machine_activity = machine_activity
+        end
+      rescue => e
+        Cnvrg::Logger.log_error(e)
+        Cnvrg::Logger.info("Sleeping for #{5 * retries}")
+        sleep(5 * retries)
+        retries +=1
+      end
     end
-  rescue => e
-    Cnvrg::Logger.log_error(e)
   end
 
   def polling_thread

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '1.11.10'
+  VERSION = '1.11.12'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.11.10
+  version: 1.11.12
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-12-28 00:00:00.000000000 Z
+date: 2021-01-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler