deb-s3-lock-fix 0.11.8.fix6 → 0.11.8.fix8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b929fb0cb051cec1edcea08170b50af78a40022ed85aa638dea7694638f50250
-  data.tar.gz: 8cfadf1a787116de9a6a4e8de4464be48d583e810b06c4b15f9c5bc8724ef336
+  metadata.gz: 1a7f40985258d5212ac5f40cf777e3c18ead8e1f73768444462a14f4b04727fe
+  data.tar.gz: d90d94de2da1ffab4439347e2f9fbe964b41b0663f58b509c110a71a91f32878
 SHA512:
-  metadata.gz: a5c2d1db7ef1b14105d7a865f2e8d72e3ae964c1ea7daacecb0ee7f68033ec4990dc16cd96e194903233fda48773658b09b671e4011f08b79c1878e4f12a71c2
-  data.tar.gz: d41baadb02c88524ca35dbbca8c407fe6edbe96e9dcd4a1dc719eaef035cf8d082ac46fb5a83f519a094fc22ee2a6ab11e90718da767c76f3099804db9847213
+  metadata.gz: 7dffaadf897accef8d6427768d393d543959843531bd2e2ce7756243580b3c6793cb917bf1f78eed47e2314ac03bc640e89b7cd252c9380aea2e258224e293d1
+  data.tar.gz: 749f27fca151c95688da92a2b7ed3686bb59df8c6118120c80ae385987091224c6bf0e00a8f3b49defd1ed4b56b1c920756b567e4676832f6cfe54bae6fbf814

data/README.md

@@ -1,8 +1,18 @@
-# deb-s3
+# deb-s3-lock-fix
 
 [![Build Status](https://travis-ci.org/deb-s3/deb-s3.svg?branch=master)](https://travis-ci.org/deb-s3/deb-s3)
 
-**This repository is a fork of [krobertson/deb-s3](https://github.com/krobertson/deb-s3).**
+**This repository is a fork of [deb-s3](https://github.com/deb-s3/deb-s3).**
+
+Note: The locking mechanism in the original deb-s3 library does not prevent race conditions.
+It relies on S3 for distributed locking, which will not work consistently due to S3's eventual
+consistency model. This fork uses DynamoDB for distributed locking, since it ensures atomic
+conditional put operations on the lock.
+
+To use this library create a DynamoDB table and export the following three env variables:
+`DEB_S3_LOCK_ACCESS_KEY_ID`
+`DEB_S3_LOCK_SECRET_ACCESS_KEY`
+`AWS_BUILDERS_REGION`
 
 `deb-s3` is a simple utility to make creating and managing APT repositories on
 S3.

data/lib/deb/s3/cli.rb

@@ -165,7 +165,7 @@ class Deb::S3::CLI < Thor
       if options[:lock]
         log("Checking for existing lock file")
         log("Locking repository for updates")
-        Deb::S3::Lock.lock(options[:codename])
+        Deb::S3::Lock.lock(options[:bucket], options[:codename])
         @lock_acquired = true
       end
 
@@ -242,18 +242,22 @@ class Deb::S3::CLI < Thor
       log("Uploading packages and new manifests to S3")
       manifests.each_value do |manifest|
         begin
-          manifest.write_to_s3 { |f| sublog("Transferring #{f}") }
+          manifest.write_to_s3 do |f|
+            sublog("Transferring #{f} at #{Time.now.strftime('%Y-%m-%d %H:%M:%S')}")
+          end
         rescue Deb::S3::Utils::AlreadyExistsError => e
           error("Uploading manifest failed because: #{e}")
         end
         release.update_manifest(manifest)
       end
-      release.write_to_s3 { |f| sublog("Transferring #{f}") }
+      release.write_to_s3 do |f|
+        sublog("Transferring #{f} at #{Time.now.strftime('%Y-%m-%d %H:%M:%S')}")
+      end
 
       log("Update complete.")
     ensure
       if options[:lock] && @lock_acquired
-        Deb::S3::Lock.unlock(options[:codename])
+        Deb::S3::Lock.unlock(options[:bucket], options[:codename])
         log("Lock released.")
       end
     end
@@ -402,7 +406,7 @@ class Deb::S3::CLI < Thor
       if options[:lock]
         log("Checking for existing lock file")
         log("Locking repository for updates")
-        Deb::S3::Lock.lock(options[:codename])
+        Deb::S3::Lock.lock(options[:bucket], options[:codename])
         @lock_acquired = true
       end
 
@@ -444,7 +448,7 @@ class Deb::S3::CLI < Thor
       log "Copy complete."
     ensure
       if options[:lock] && @lock_acquired
-        Deb::S3::Lock.unlock(options[:codename])
+        Deb::S3::Lock.unlock(options[:bucket], options[:codename])
         log("Lock released.")
       end
     end
@@ -497,7 +501,7 @@ class Deb::S3::CLI < Thor
       if options[:lock]
         log("Checking for existing lock file")
         log("Locking repository for updates")
-        Deb::S3::Lock.lock(options[:codename])
+        Deb::S3::Lock.lock(options[:bucket], options[:codename])
         @lock_acquired = true
       end
 
@@ -545,7 +549,7 @@ class Deb::S3::CLI < Thor
       end
     ensure
       if options[:lock] && @lock_acquired
-        Deb::S3::Lock.unlock(options[:codename])
+        Deb::S3::Lock.unlock(options[:bucket], options[:codename])
         log("Lock released.")
       end
     end
@@ -611,7 +615,7 @@ class Deb::S3::CLI < Thor
       if options[:lock]
         log("Checking for existing lock file")
         log("Locking repository for updates")
-        Deb::S3::Lock.lock(options[:codename])
+        Deb::S3::Lock.lock(options[:bucket], options[:codename])
         @lock_acquired = true
       end
 
@@ -675,7 +679,7 @@ class Deb::S3::CLI < Thor
       end
     ensure
       if options[:lock] && @lock_acquired
-        Deb::S3::Lock.unlock(options[:codename])
+        Deb::S3::Lock.unlock(options[:bucket], options[:codename])
         log("Lock released.")
       end
     end

data/lib/deb/s3/lock.rb

@@ -30,10 +30,10 @@ class Deb::S3::Lock
   end
 
   class << self
-    def lock(codename, max_attempts = 60, max_wait_interval = 10)
+    def lock(bucket, codename, max_attempts = 60, max_wait_interval = 10)
       uuid = SecureRandom.uuid
       lock_body = "#{Etc.getlogin}@#{Socket.gethostname}-#{uuid}"
-      lock_key = codename
+      lock_key = "#{bucket}/#{codename}"
 
       $stderr.puts("Current job's hostname with UUID: #{lock_body}")
 
@@ -51,34 +51,31 @@ class Deb::S3::Lock
           })
           return
         rescue Aws::DynamoDB::Errors::ConditionalCheckFailedException
-          lock_holder = current_lock_holder(codename)
+          lock_holder = current_lock_holder(bucket, codename)
           current_time = Time.now.strftime("%Y-%m-%d %H:%M:%S")
           $stderr.puts("[#{current_time}] Repository is locked by another user: #{lock_holder.user} at host #{lock_holder.host_with_uuid}")
           $stderr.puts("Attempting to obtain a lock after #{wait_interval} second(s).")
           sleep(wait_interval)
-        rescue => e
-          $stderr.puts("Unexpected error: #{e.message}")
-          sleep(wait_interval)
         end
       end
 
-      raise "Unable to obtain a lock after #{max_attempts}, giving up."
+      raise "Unable to obtain a lock after #{max_attempts} attemtps, giving up."
     end
 
-    def unlock(codename)
+    def unlock(bucket, codename)
       dynamodb.delete_item({
         table_name: DYNAMODB_TABLE_NAME,
         key: {
-          'lock_key' => codename
+          'lock_key' => "#{bucket}/#{codename}"
         }
       })
     end
 
-    def current_lock_holder(codename)
+    def current_lock_holder(bucket, codename)
       response = dynamodb.get_item({
         table_name: DYNAMODB_TABLE_NAME,
         key: {
-          'lock_key' => codename
+          'lock_key' => "#{bucket}/#{codename}"
         }
       })
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deb-s3-lock-fix
 version: !ruby/object:Gem::Version
-  version: 0.11.8.fix6
+  version: 0.11.8.fix8
 platform: ruby
 authors:
 - Braeden Wolf & Ken Robertson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-09-14 00:00:00.000000000 Z
+date: 2023-09-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -82,7 +82,7 @@ dependencies:
         version: '11'
 description: Fork of deb-s3 with a specific fix for locking. Original work by Ken
   Robertson.
-email: braedenwolf@outlook.com & ken@invalidlogic.com
+email: braeden.wolf@sanctuary.ai & ken@invalidlogic.com
 executables:
 - deb-s3
 extensions: []
@@ -99,7 +99,7 @@ files:
 - lib/deb/s3/templates/package.erb
 - lib/deb/s3/templates/release.erb
 - lib/deb/s3/utils.rb
-homepage: https://github.com/braedenwolf/deb-s3-lock-fix
+homepage: https://gitlab.com/sanctuaryai/precog2/infrastructure/deb-s3-lock-fix
 licenses:
 - MIT
 metadata: {}