capistrano-ops 1.0.7 → 1.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8f235992d4982cea3d663c6bd2616becfd54a4f772d199c93c76d3f27b7b359a
-  data.tar.gz: af9e2c333ff0a24fb9eff2bbc87ef38f6ac5d014b4e975ac685542ad075f8544
+  metadata.gz: 96ec2fb775bc19f6853b6194b58f544acb57957c3ccca331caea4a7de3313507
+  data.tar.gz: 320ee9acc8aaa62844ea0e693b605f4b19f0567ab1449952df4941eb986096f0
 SHA512:
-  metadata.gz: 7046680c7e931b6942785f2539215549a3bc5f20a32d4a1a2226a6bf2db684b019124cd409726bdb839295265b488abf60f340ef9311f8f794b8d576fe3b79c3
-  data.tar.gz: e0499b85de6f9bc85463433ab80fab5fccfa33489940d21d8578fa1e1a71d9ad94bcac8c7e542f4c2e8982ab5f601d986445175643646dc74769f3810c1a9157
+  metadata.gz: cdb6f282869288525433017c3ccfc48638dc0abcc22b1adca479135a43d605f2308a0e3badbccdb5820e5d223be8818b41ae6ff3adade5c621a7c56b389762b9
+  data.tar.gz: 6e42b2e1596d763b25da9578d916ad37c0feabd2bb85ba3a7a32151bb3fb49505d641a7162ccfd1f75601da73402c541ac9bdb9ddab5e36fd6362ee19613c472

data/lib/capistrano/ops/rails/lib/backup/s3.rb

@@ -6,8 +6,10 @@ module Backup
   require 'zlib'
   require 'find'
   require 'capistrano/ops/rails/lib/backup/s3_helper'
+
   class S3
     include Backup::S3Helper
+
     attr_accessor :endpoint, :region, :access_key_id, :secret_access_key, :s3_resource, :s3_client
 
     def initialize(endpoint: ENV['S3_BACKUP_ENDPOINT'], region: ENV['S3_BACKUP_REGION'], access_key_id: ENV['S3_BACKUP_KEY'],
@@ -21,7 +23,6 @@ module Backup
         access_key_id: access_key_id,
         secret_access_key: secret_access_key,
         force_path_style: true
-
       }
       configuration[:endpoint] = endpoint unless endpoint.nil?
       self.s3_resource = Aws::S3::Resource.new(configuration)
@@ -46,6 +47,7 @@ module Backup
       end
     end
 
+    # rubocop:disable Metrics/MethodLength
     def upload_file_as_stream(file_path, key)
       bucket = ENV['S3_BACKUP_BUCKET']
       # Calculate total size of the file to be uploaded
@@ -54,32 +56,50 @@ module Backup
 
       uploaded_size = 0
 
-      # Initiate multipart upload
-      multipart_upload = s3_client.create_multipart_upload(bucket: bucket, key: key)
-
       # Upload the tar.gz data from the file in parts
       part_number = 1
       parts = []
       last_logged_progress = 0
+      max_retry_time = 300 # 5 minutes in seconds
+      total_wait_time = 0
 
       begin
+        multipart_upload ||= s3_client.create_multipart_upload(bucket: bucket, key: key)
         File.open(file_path, 'rb') do |file|
           while (part = file.read(chunk_size)) # Read calculated chunk size
-            part_upload = s3_client.upload_part(
-              bucket: bucket,
-              key: key,
-              upload_id: multipart_upload.upload_id,
-              part_number: part_number,
-              body: part
-            )
-            parts << { part_number: part_number, etag: part_upload.etag }
-            uploaded_size += part.size
-            part_number += 1
+            retry_count = 0
+            begin
+              # Initiate multipart upload
+              part_upload = s3_client.upload_part(
+                bucket: bucket,
+                key: key,
+                upload_id: multipart_upload.upload_id,
+                part_number: part_number,
+                body: part
+              )
+              parts << { part_number: part_number, etag: part_upload.etag }
+              uploaded_size += part.size
+              part_number += 1
 
-            progress = (uploaded_size.to_f / total_size * 100).round
-            if progress >= last_logged_progress + 10
-              puts "Upload progress: #{progress}% complete"
-              last_logged_progress = progress
+              progress = (uploaded_size.to_f / total_size * 100).round
+              if progress >= last_logged_progress + 10
+                puts "Upload progress: #{progress}% complete"
+                last_logged_progress = progress
+              end
+            rescue StandardError => e
+              retry_count += 1
+              wait_time = 2**retry_count
+              total_wait_time += wait_time
+
+              if total_wait_time > max_retry_time
+                puts "Exceeded maximum retry time of #{max_retry_time / 60} minutes. Aborting upload."
+                raise e
+              end
+              puts "Error uploading part #{part_number}: #{e.message.split("\n").first} (Attempt #{retry_count})"
+              puts "Retry in #{wait_time} seconds"
+              sleep(wait_time) # Exponential backoff
+              puts 'Retrying upload part...'
+              retry
             end
           end
         end
@@ -109,7 +129,6 @@ module Backup
       raise e
     end
 
-    # rubocop:disable Metrics/MethodLength
     def upload_folder_as_tar_gz_stream(folder_path, key)
       bucket = ENV['S3_BACKUP_BUCKET']
 
@@ -127,31 +146,49 @@ module Backup
       # Start a thread to write the tar.gz data to the pipe
       writer_thread = start_writer_thread(folder_path, write_io)
 
-      # Initiate multipart upload
-      multipart_upload = s3_client.create_multipart_upload(bucket: bucket, key: key)
-
       # Upload the tar.gz data from the pipe in parts
       part_number = 1
       parts = []
       last_logged_progress = 0
+      max_retry_time = 300 # 5 minutes in seconds
+      total_wait_time = 0
 
       begin
+        # Initiate multipart upload
+        multipart_upload ||= s3_client.create_multipart_upload(bucket: bucket, key: key)
         while (part = read_io.read(chunk_size)) # Read calculated chunk size
-          part_upload = s3_client.upload_part(
-            bucket: bucket,
-            key: key,
-            upload_id: multipart_upload.upload_id,
-            part_number: part_number,
-            body: part
-          )
-          parts << { part_number: part_number, etag: part_upload.etag }
-          uploaded_size += part.size
-          part_number += 1
+          retry_count = 0
+          begin
+            part_upload = s3_client.upload_part(
+              bucket: bucket,
+              key: key,
+              upload_id: multipart_upload.upload_id,
+              part_number: part_number,
+              body: part
+            )
+            parts << { part_number: part_number, etag: part_upload.etag }
+            uploaded_size += part.size
+            part_number += 1
+
+            progress = (uploaded_size.to_f / total_size * 100).round
+            if progress >= last_logged_progress + 10
+              puts "Upload progress: #{progress}% complete"
+              last_logged_progress = progress
+            end
+          rescue StandardError => e
+            retry_count += 1
+            wait_time = 2**retry_count
+            total_wait_time += wait_time
 
-          progress = (uploaded_size.to_f / total_size * 100).round
-          if progress >= last_logged_progress + 10
-            puts "Upload progress: #{progress}% complete"
-            last_logged_progress = progress
+            if total_wait_time > max_retry_time
+              puts "Exceeded maximum retry time of #{max_retry_time / 60} minutes. Aborting upload."
+              raise e
+            end
+            puts "Error uploading part #{part_number}: #{e.message.split("\n").first} (Attempt #{retry_count})"
+            puts "Retry in #{wait_time} seconds"
+            sleep(wait_time) # Exponential backoff
+            puts 'Retrying upload part...'
+            retry
           end
         end
 
@@ -165,11 +202,13 @@ module Backup
         puts 'Completed multipart upload'
       rescue StandardError => e
         # Abort multipart upload in case of error
-        s3_client.abort_multipart_upload(
-          bucket: bucket,
-          key: key,
-          upload_id: multipart_upload.upload_id
-        )
+        if multipart_upload
+          s3_client.abort_multipart_upload(
+            bucket: bucket,
+            key: key,
+            upload_id: multipart_upload.upload_id
+          )
+        end
         puts "Aborted multipart upload due to error: #{e.message}"
         raise e
       ensure

data/lib/capistrano/ops/rails/lib/backup/s3_helper.rb

@@ -32,7 +32,7 @@ module Backup
 
     def calculate_chunk_size(total_size)
       max_chunks = 10_000
-      min_chunk_size = 20 * 1024 * 1024 # 20MB
+      min_chunk_size = 50 * 1024 * 1024 # 50MB
       max_chunk_size = 105 * 1024 * 1024 # 105MB
       chunk_size = [total_size / max_chunks, min_chunk_size].max
       [chunk_size, max_chunk_size].min

data/lib/capistrano/ops/version.rb

@@ -2,6 +2,6 @@
 
 module Capistrano
   module Ops
-    VERSION = '1.0.7'
+    VERSION = '1.0.9'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: capistrano-ops
 version: !ruby/object:Gem::Version
-  version: 1.0.7
+  version: 1.0.9
 platform: ruby
 authors:
 - Florian Crusius
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-12-05 00:00:00.000000000 Z
+date: 2024-12-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-s3