deb-s3-lock-fix 0.11.8.fix1 → 0.11.8.fix2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 180e005ba32e429abe5a462bb81cd03d6897b02fb10e079a655008589fa687df
-  data.tar.gz: df8aeb29f17894bdfca5f82caaaf5ad6576d07de0fed3789d532de3c93486a13
+  metadata.gz: dc14627ff3d5d03feb6e36c26ba8ff41f6652ee3e085e43f750968018d662114
+  data.tar.gz: 0f628316a7fd3923201ee26ee4ab35d1d5e522d486faf08e24646c93c288572c
 SHA512:
-  metadata.gz: '09eaff5677bd51d239794dd5b5bcf9f9ac18275593cd51088f22a0bded0bdd1c2bc4c82cf771aaa8c32d7fd525c973919ac6a5fe4b85818923922e576fa75b13'
-  data.tar.gz: c463aba5d27113100687654aebd7952ca32142f2835190038dfaf401918d004cdf9bdca74e641866b4e90e5ec91cacbd94bde28c504d0ea637c798181d35e9aa
+  metadata.gz: 7b054fdfccd8dd2b573bbe2f702bd2532a5b7218dc0f049e0d8e89356aafc6d22f75054c5edd717e4bb293ff035b274e1a3ced4dc0f79d259b51f8a291e58531
+  data.tar.gz: 40cfc56aa2c87fbaadd9753562e2550f45431defcd5fede8c787ae6964dac56a11a332bffa058f62cf18b75739f734395660eba9aca0d7591da4fd06acaef891

data/lib/deb/s3/cli.rb

@@ -165,7 +165,7 @@ class Deb::S3::CLI < Thor
       if options[:lock]
         log("Checking for existing lock file")
         log("Locking repository for updates")
-        Deb::S3::Lock.lock(options[:codename], component, options[:arch], options[:cache_control])
+        Deb::S3::Lock.lock(options[:codename])
         @lock_acquired = true
       end
 
@@ -253,7 +253,7 @@ class Deb::S3::CLI < Thor
       log("Update complete.")
     ensure
       if options[:lock] && @lock_acquired
-        Deb::S3::Lock.unlock(options[:codename], component, options[:arch], options[:cache_control])
+        Deb::S3::Lock.unlock(options[:codename])
         log("Lock released.")
       end
     end
@@ -402,7 +402,7 @@ class Deb::S3::CLI < Thor
       if options[:lock]
         log("Checking for existing lock file")
         log("Locking repository for updates")
-        Deb::S3::Lock.lock(options[:codename], to_component, options[:arch], options[:cache_control])
+        Deb::S3::Lock.lock(options[:codename])
         @lock_acquired = true
       end
 
@@ -444,7 +444,7 @@ class Deb::S3::CLI < Thor
       log "Copy complete."
     ensure
       if options[:lock] && @lock_acquired
-        Deb::S3::Lock.unlock(options[:codename], component, options[:arch], options[:cache_control])
+        Deb::S3::Lock.unlock(options[:codename])
         log("Lock released.")
       end
     end
@@ -497,7 +497,7 @@ class Deb::S3::CLI < Thor
       if options[:lock]
         log("Checking for existing lock file")
         log("Locking repository for updates")
-        Deb::S3::Lock.lock(options[:codename], component, options[:arch], options[:cache_control])
+        Deb::S3::Lock.lock(options[:codename])
         @lock_acquired = true
       end
 
@@ -545,7 +545,7 @@ class Deb::S3::CLI < Thor
       end
     ensure
       if options[:lock] && @lock_acquired
-        Deb::S3::Lock.unlock(options[:codename], component, options[:arch], options[:cache_control])
+        Deb::S3::Lock.unlock(options[:codename])
         log("Lock released.")
       end
     end
@@ -611,7 +611,7 @@ class Deb::S3::CLI < Thor
       if options[:lock]
         log("Checking for existing lock file")
         log("Locking repository for updates")
-        Deb::S3::Lock.lock(options[:codename], component, options[:arch], options[:cache_control])
+        Deb::S3::Lock.lock(options[:codename])
         @lock_acquired = true
       end
 
@@ -675,7 +675,7 @@ class Deb::S3::CLI < Thor
       end
     ensure
       if options[:lock] && @lock_acquired
-        Deb::S3::Lock.unlock(options[:codename], component, options[:arch], options[:cache_control])
+        Deb::S3::Lock.unlock(options[:codename])
         log("Lock released.")
       end
     end

data/lib/deb/s3/lock.rb

@@ -1,125 +1,92 @@
-# -*- encoding : utf-8 -*-
-require "base64"
-require "digest/md5"
-require "etc"
-require "socket"
-require "tempfile"
+require "aws-sdk-dynamodb"
 require "securerandom"
+require "etc"
 
 class Deb::S3::Lock
-  attr_reader :user
-  attr_reader :host
+  DYNAMODB_TABLE_NAME = 'deb-s3-lock'
 
-  def initialize(user, host)
+  def initialize(user, host_with_uuid)
     @user = user
-    @host = host
+    @host_with_uuid = host_with_uuid
+  end
+
+  def self.dynamodb
+    @dynamodb ||= begin
+      validate_environment_variables!
+      Aws::DynamoDB::Client.new(
+        access_key_id: ENV['DEB_S3_LOCK_ACCESS_KEY_ID'],
+        secret_access_key: ENV['DEB_S3_LOCK_SECRET_ACCESS_KEY'],
+        region: ENV['AWS_REGION']
+      )
+    end
+  end
+
+  def self.validate_environment_variables!
+    %w[DEB_S3_LOCK_ACCESS_KEY_ID DEB_S3_LOCK_SECRET_ACCESS_KEY AWS_REGION].each do |var|
+      raise "Environment variable #{var} not set." unless ENV[var]
+    end
   end
 
   class << self
-    #
-    # 2-phase mutual lock mechanism based on `s3:CopyObject`.
-    #
-    # This logic isn't relying on S3's enhanced features like Object Lock
-    # because it imposes some limitation on using other features like
-    # S3 Cross-Region replication. This should work more than good enough 
-    # with S3's strong read-after-write consistency which we can presume
-    # in all region nowadays.
-    #
-    # This is relying on S3 to set object's ETag as object's MD5 if an
-    # object isn't comprized from multiple parts. We'd be able to presume
-    # it as the lock file is usually an object of some smaller bytes.
-    #
-    # acquire lock:
-    # 1. call `s3:HeadObject` on final lock object
-    #   1. If final lock object exists, restart from the beginning
-    #   2. Otherwise, call `s3:PutObject` to create initial lock object
-    # 2. Perform `s3:CopyObject` to copy from initial lock object
-    #    to final lock object with specifying ETag/MD5 of the initial
-    #    lock object
-    #   1. If copy object fails as `PreconditionFailed`, restart
-    #      from the beginning
-    #   2. Otherwise, lock has been acquired
-    #
-    # release lock:
-    # 1. remove final lock object by `s3:DeleteObject`
-    #
-    def lock(codename, component = nil, architecture = nil, cache_control = nil, max_attempts=60, max_wait_interval=10)
+    def lock(codename, max_attempts = 60, max_wait_interval = 10)
       uuid = SecureRandom.uuid
-      lockbody = "#{Etc.getlogin}@#{Socket.gethostname}-#{uuid}"
-
-      initial_lockfile = initial_lock_path(codename, component, architecture, cache_control)
-      final_lockfile = lock_path(codename, component, architecture, cache_control)
+      lock_body = "#{Etc.getlogin}@#{Socket.gethostname}-#{uuid}"
+      lock_key = codename
 
-      md5_b64 = Base64.encode64(Digest::MD5.digest(lockbody))
-      md5_hex = Digest::MD5.hexdigest(lockbody)
       max_attempts.times do |i|
-        wait_interval = [(1<<i)/10, max_wait_interval].min
-        if Deb::S3::Utils.s3_exists?(final_lockfile)
-          lock = current(codename, component, architecture, cache_control)
-          $stderr.puts("Repository is locked by another user: #{lock.user} at host #{lock.host} (phase-1)")
-          $stderr.puts("Attempting to obtain a lock after #{wait_interval} secound(s).")
-          sleep(wait_interval)
-        else
-          # upload the file
-          Deb::S3::Utils.s3.put_object(
-            bucket: Deb::S3::Utils.bucket,
-            key: Deb::S3::Utils.s3_path(initial_lockfile),
-            body: lockbody,
-            content_type: "text/plain",
-            content_md5: md5_b64,
-            metadata: {
-              "md5" => md5_hex,
+        wait_interval = [2**i, max_wait_interval].min
+
+        begin
+          dynamodb.put_item({
+            table_name: DYNAMODB_TABLE_NAME,
+            item: {
+              'lock_key' => lock_key,
+              'lock_body' => lock_body
             },
-          )
-          begin
-            Deb::S3::Utils.s3.copy_object(
-              bucket: Deb::S3::Utils.bucket,
-              key: Deb::S3::Utils.s3_path(final_lockfile),
-              copy_source: "/#{Deb::S3::Utils.bucket}/#{Deb::S3::Utils.s3_path(initial_lockfile)}",
-              copy_source_if_match: md5_hex,
-            )
-            return
-          rescue Aws::S3::Errors::PreconditionFailed => error
-            lock = current(codename, component, architecture, cache_control)
-            $stderr.puts("Repository is locked by another user: #{lock.user} at host #{lock.host} (phase-2)")
-            $stderr.puts("Attempting to obtain a lock after #{wait_interval} second(s).")
-            sleep(wait_interval)
-          end
+            condition_expression: "attribute_not_exists(lock_key)"
+          })
+          return
+        rescue Aws::DynamoDB::Errors::ConditionalCheckFailedException
+          lock_holder = current_lock_holder(codename)
+          current_time = Time.now.strftime("%Y-%m-%d %H:%M:%S")
+          $stderr.puts("[#{current_time}] Repository is locked by another user: #{lock_holder.user} at host #{lock_holder.host_with_uuid}")
+          $stderr.puts("Attempting to obtain a lock after #{wait_interval} second(s).")
+          sleep(wait_interval)
+        rescue => e
+          $stderr.puts("Unexpected error: #{e.message}")
+          sleep(wait_interval)
         end
       end
-      # TODO: throw appropriate error class
-      raise("Unable to obtain a lock after #{max_attempts}, giving up.")
+
+      raise "Unable to obtain a lock after #{max_attempts}, giving up."
     end
 
-    def unlock(codename, component = nil, architecture = nil, cache_control = nil)
-      Deb::S3::Utils.s3_remove(initial_lock_path(codename, component, architecture, cache_control))
-      Deb::S3::Utils.s3_remove(lock_path(codename, component, architecture, cache_control))
+    def unlock(codename)
+      dynamodb.delete_item({
+        table_name: DYNAMODB_TABLE_NAME,
+        key: {
+          'lock_key' => codename
+        }
+      })
     end
 
-    def current(codename, component = nil, architecture = nil, cache_control = nil)
-      lockbody = Deb::S3::Utils.s3_read(lock_path(codename, component, architecture, cache_control))
-      if lockbody
-        user, host_with_uuid = lockbody.to_s.split("@", 2)
-        lock = Deb::S3::Lock.new(user, host_with_uuid)
+    def current_lock_holder(codename)
+      response = dynamodb.get_item({
+        table_name: DYNAMODB_TABLE_NAME,
+        key: {
+          'lock_key' => codename
+        }
+      })
+
+      if response.item
+        lockdata = response.item['lock_body']
+        user, host_with_uuid = lockdata.split("@", 2)
+        Deb::S3::Lock.new(user, host_with_uuid)
       else
-        lock = Deb::S3::Lock.new("unknown", "unknown")
+        Deb::S3::Lock.new("unknown", "unknown")
       end
-      lock
-    end
-
-    private
-    def initial_lock_path(codename, component = nil, architecture = nil, cache_control = nil)
-      "dists/#{codename}/lockfile.lock"
     end
 
-    def lock_path(codename, component = nil, architecture = nil, cache_control = nil)
-      #
-      # Acquire repository lock at `codename` level to avoid race between concurrent upload attempts.
-      #
-      # * `deb-s3 upload --arch=all` touchs multiples of `dists/{codename}/{component}/binary-*/Packages*`
-      # * All `deb-s3 upload` touchs `dists/{codename}/Release`
-      #
-      "dists/#{codename}/lockfile"
-    end
+    private :dynamodb, :validate_environment_variables!
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deb-s3-lock-fix
 version: !ruby/object:Gem::Version
-  version: 0.11.8.fix1
+  version: 0.11.8.fix2
 platform: ruby
 authors:
 - Braeden Wolf & Ken Robertson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-09-13 00:00:00.000000000 Z
+date: 2023-09-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -38,6 +38,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-dynamodb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement