deb-s3 0.9.3 → 0.11.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 9e8dbe12df60c668bf2f6469b90d6cdd1cd81325
-  data.tar.gz: 0107fe0bd5e38e79a39046ebeb19566b647e3865
+SHA256:
+  metadata.gz: 63a8b704bcc52020044d1c8e2a67d5c83b20fdcb0febb012a1aa95b3a7e4e33b
+  data.tar.gz: e0c1a63f7c3528a7493a20ca65837653bd45e2c3e78815e8858becd2372efe1a
 SHA512:
-  metadata.gz: 79b208495ff06a5e68a0a6ccb6829fc8bf0b7ef99a78eb33959e688e27ac9407e9f19d1ab6af3e4c90056fb46dd6b354de43e26b1d43df16ed76a586a057c7de
-  data.tar.gz: 720fc4fc1585841872a49e1572a88107adad1b404e65be6984fff915062aa30f9e63ba31215020606e082cbba108506750c63d30a01bdcbd4437c392dec689aa
+  metadata.gz: 9fd7a53af575ee39fa3a1ea7577e5c81feea7200715a8477674022c449ed3a7cde33b8f0f84b8c096b8eddfe72a4d56c7e057bbfa4503f75063d23e744348fa6
+  data.tar.gz: d2049aeacd81d10398e901e8413421e4ea06827d426300c85fc7477b36f56fab651bc0b8402cac25f7562de3123d0ba348b9d7b43ceb6a53be36fb666acf9bf5

data/README.md

@@ -1,6 +1,8 @@
 # deb-s3
 
-[![Build Status](https://travis-ci.org/krobertson/deb-s3.svg?branch=master)](https://travis-ci.org/krobertson/deb-s3)
+[![Build Status](https://travis-ci.org/deb-s3/deb-s3.svg?branch=master)](https://travis-ci.org/deb-s3/deb-s3)
+
+**This repository is a fork of [krobertson/deb-s3](https://github.com/krobertson/deb-s3).**
 
 `deb-s3` is a simple utility to make creating and managing APT repositories on
 S3.
@@ -27,17 +29,32 @@ With `deb-s3`, there is no need for this. `deb-s3` features:
 
 ## Getting Started
 
-You can simply install it from rubygems:
+[Install the package via gem](https://github.com/deb-s3/deb-s3/packages/304683) or manually:
 
 ```console
-$ gem install deb-s3
+$ curl -sLO https://github.com/deb-s3/deb-s3/releases/download/0.11.5/deb-s3-0.11.5.gem
+$ gem install deb-s3-0.11.5.gem
 ```
 
-Or to run the code directly, just check out the repo and run Bundler to ensure
+or via APT:
+
+```console
+# Add repository key
+$ sudo wget -O /etc/apt/trusted.gpg.d/deb-s3-archive-keyring.gpg https://raw.githubusercontent.com/deb-s3/deb-s3/master/deb-s3-archive-keyring.gpg
+
+# Add repository
+$ echo "deb http://deb-s3-repo.s3.us-east-2.amazonaws.com/debian/ $(lsb_release -cs) main" | sudo tee -a /etc/apt/sources.list > /dev/null
+
+# Install package
+$ sudo apt-get update
+$ sudo apt-get install deb-s3
+```
+
+To run the code directly, just check out the repo and run bundler to ensure
 all dependencies are installed:
 
 ```console
-$ git clone https://github.com/krobertson/deb-s3.git
+$ git clone https://github.com/deb-s3/deb-s3.git
 $ cd deb-s3
 $ bundle install
 ```
@@ -64,7 +81,7 @@ Options:
   -a, [--arch=ARCH]                                        # The architecture of the package in the APT repository.
   -p, [--preserve-versions], [--no-preserve-versions]      # Whether to preserve other versions of a package in the repository when uploading one.
   -l, [--lock], [--no-lock]                                # Whether to check for an existing lock on the repository to prevent simultaneous updates
-      [--fail-if-exists], [--no-fail-if-exists]            # Whether to overwrite any existing package that has the same filename in the pool or the same name and version in the manifest.
+      [--fail-if-exists], [--no-fail-if-exists]            # Whether to overwrite any existing package that has the same filename in the pool or the same name and version in the manifest but different contents.
       [--skip-package-upload], [--no-skip-package-upload]  # Whether to skip all package uploads.This is useful when hosting .deb files outside of the bucket.
   -b, [--bucket=BUCKET]                                    # The name of the S3 bucket to upload to.
       [--prefix=PREFIX]                                    # The path prefix to use when storing on S3.
@@ -92,8 +109,9 @@ Uploads the given files to a S3 bucket as an APT repository.
 ```
 
 You can also delete packages from the APT repository. Please keep in mind that
-this does NOT delete the .deb file itself, it only removes it from the list of
-packages in the specified component, codename and architecture.
+this does NOT delete the .deb file itself (the `clean` command does that), it
+only removes it from the list of packages in the specified component, codename
+and architecture.
 
 Now to delete the package:
 ```console
@@ -139,6 +157,48 @@ Options:
 Remove the package named PACKAGE. If --versions is not specified, deleteall versions of PACKAGE. Otherwise, only the specified versions will be deleted.
 ```
 
+Dangling `.deb` files left by the `delete` command (or uploading new versions) can be removed using the `clean` command:
+
+```console
+$ deb-s3 clean --bucket my-bucket
+>> Retrieving existing manifests
+>> Searching for unreferenced packages
+   -- pool/m/my/my-deb-package-1.0.0_amd64.deb
+```
+
+```
+Usage:
+  deb-s3 clean
+
+Options:
+  -l, [--lock], [--no-lock]                          # Whether to check for an existing lock on the repository to prevent simultaneous updates
+  -b, [--bucket=BUCKET]                              # The name of the S3 bucket to upload to.
+      [--prefix=PREFIX]                              # The path prefix to use when storing on S3.
+  -o, [--origin=ORIGIN]                              # The origin to use in the repository Release file.
+      [--suite=SUITE]                                # The suite to use in the repository Release file.
+  -c, [--codename=CODENAME]                          # The codename of the APT repository.
+                                                     # Default: stable
+  -m, [--component=COMPONENT]                        # The component of the APT repository.
+                                                     # Default: main
+      [--access-key-id=ACCESS_KEY_ID]                # The access key for connecting to S3.
+      [--secret-access-key=SECRET_ACCESS_KEY]        # The secret key for connecting to S3.
+      [--session-token=SESSION_TOKEN]                # The (optional) session token for connecting to S3.
+      [--endpoint=ENDPOINT]                          # The URL endpoint to the S3 API.
+      [--s3-region=S3_REGION]                        # The region for connecting to S3.
+                                                     # Default: us-east-1
+      [--force-path-style], [--no-force-path-style]  # Use S3 path style instead of subdomains.
+      [--proxy-uri=PROXY_URI]                        # The URI of the proxy to send service requests through.
+  -v, [--visibility=VISIBILITY]                      # The access policy for the uploaded files. Can be public, private, or authenticated.
+                                                     # Default: public
+      [--sign=SIGN]                                  # GPG Sign the Release file when uploading a package, or when verifying it after removing a package. Use --sign with your GPG key ID to use a specific key (--sign=6643C242C18FE05B).
+      [--gpg-options=GPG_OPTIONS]                    # Additional command line options to pass to GPG when signing.
+  -e, [--encryption], [--no-encryption]              # Use S3 server side encryption.
+  -q, [--quiet], [--no-quiet]                        # Doesn't output information, just returns status appropriately.
+  -C, [--cache-control=CACHE_CONTROL]                # Add cache-control headers to S3 objects.
+
+Delete packages from the pool which are no longer referenced
+```
+
 You can also verify an existing APT repository on S3 using the `verify` command:
 
 ```console
@@ -179,3 +239,47 @@ Options:
 
 Verifies that the files in the package manifests exist
 ```
+
+#### Example S3 IAM Policy
+
+```
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": [
+                "s3:ListBucket"
+            ],
+            "Resource": [
+                "arn:aws:s3:::BUCKETNAME",
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "s3:PutObject",
+                "s3:GetObject",
+                "s3:DeleteObject",
+                "s3:DeleteObjectVersion",
+                "s3:GetObjectAcl",
+                "s3:GetObjectTagging",
+                "s3:GetObjectTorrent",
+                "s3:GetObjectVersion",
+                "s3:GetObjectVersionAcl",
+                "s3:GetObjectVersionTagging",
+                "s3:GetObjectVersionTorrent",
+                "s3:PutObjectAcl",
+                "s3:PutObjectTagging",
+                "s3:PutObjectVersionAcl",
+                "s3:PutObjectVersionTagging",
+                "s3:ReplicateObject",
+                "s3:RestoreObject"
+            ],
+            "Resource": [
+                "arn:aws:s3:::BUCKETNAME/*"
+            ]
+        }
+    ]
+}
+```

data/lib/deb/s3/cli.rb

@@ -1,5 +1,5 @@
 # -*- encoding : utf-8 -*-
-require "aws-sdk"
+require "aws-sdk-s3"
 require "thor"
 
 # Hack: aws requires this!
@@ -86,10 +86,11 @@ class Deb::S3::CLI < Thor
     "Can be public, private, or authenticated."
 
   class_option :sign,
-  :type     => :string,
+  :type     => :array,
   :desc     => "GPG Sign the Release file when uploading a package, " +
     "or when verifying it after removing a package. " +
-    "Use --sign with your GPG key ID to use a specific key (--sign=6643C242C18FE05B)."
+    "Use --sign with your space delimited GPG key IDs to use one ore more specific keys " +
+    "(--sign 6643C242C18FE05B 6243C322C18FE05C)."
 
   class_option :gpg_options,
   :default => "",
@@ -138,7 +139,8 @@ class Deb::S3::CLI < Thor
   :default  => false,
   :type     => :boolean,
   :desc     => "Whether to overwrite any existing package that has the same " +
-    "filename in the pool or the same name and version in the manifest."
+    "filename in the pool or the same name and version in the manifest but " +
+	"different contents."
 
   option :skip_package_upload,
   :default  => false,
@@ -162,12 +164,6 @@ class Deb::S3::CLI < Thor
     begin
       if options[:lock]
         log("Checking for existing lock file")
-        if Deb::S3::Lock.locked?(options[:codename], component, options[:arch], options[:cache_control])
-          lock = Deb::S3::Lock.current(options[:codename], component, options[:arch], options[:cache_control])
-          log("Repository is locked by another user: #{lock.user} at host #{lock.host}")
-          log("Attempting to obtain a lock")
-          Deb::S3::Lock.wait_for_lock(options[:codename], component, options[:arch], options[:cache_control])
-        end
         log("Locking repository for updates")
         Deb::S3::Lock.lock(options[:codename], component, options[:arch], options[:cache_control])
         @lock_acquired = true
@@ -204,11 +200,15 @@ class Deb::S3::CLI < Thor
         # throw an error. This is mainly the case when initializing a brand new
         # repository. With "all", we won't know which architectures they're using.
         if arch == "all" && manifests.count == 0
-          error("Package #{File.basename(file)} had architecture \"all\", " +
-                "however noexisting package lists exist. This can often happen " +
-                "if the first package you are add to a new repository is an " +
-                "\"all\" architecture file. Please use --arch [i386|amd64|armhf] or " +
-                "another platform type to upload the file.")
+          manifests['amd64'] = Deb::S3::Manifest.retrieve(options[:codename], component,'amd64', options[:cache_control], options[:fail_if_exists], options[:skip_package_upload])
+          manifests['i386'] = Deb::S3::Manifest.retrieve(options[:codename], component,'i386', options[:cache_control], options[:fail_if_exists], options[:skip_package_upload])
+          manifests['armhf'] = Deb::S3::Manifest.retrieve(options[:codename], component,'armhf', options[:cache_control], options[:fail_if_exists], options[:skip_package_upload])
+
+         # error("Package #{File.basename(file)} had architecture \"all\", " +
+         #       "however noexisting package lists exist. This can often happen " +
+         #       "if the first package you are add to a new repository is an " +
+         #       "\"all\" architecture file. Please use --arch [i386|amd64|armhf] or " +
+         #       "another platform type to upload the file.")
         end
 
         # retrieve the manifest for the arch if we don't have it already
@@ -285,7 +285,7 @@ class Deb::S3::CLI < Thor
                                             false, false)
       manifest.packages.map do |package|
         if options[:long]
-          package.generate
+          package.generate(options[:codename])
         else
           [package.name, package.full_version, package.architecture].tap do |row|
             row.each_with_index do |col, i|
@@ -330,7 +330,7 @@ class Deb::S3::CLI < Thor
       error "No such package found."
     end
 
-    puts package.generate
+    puts package.generate(options[:codename])
   end
 
   desc "copy PACKAGE TO_CODENAME TO_COMPONENT ",
@@ -346,6 +346,13 @@ class Deb::S3::CLI < Thor
     :aliases  => "-a",
     :desc     => "The architecture of the package in the APT repository."
 
+  option :lock,
+  :default  => false,
+  :type     => :boolean,
+  :aliases  => "-l",
+  :desc     => "Whether to check for an existing lock on the repository " +
+    "to prevent simultaneous updates "
+
   option :versions,
     :default  => nil,
     :type     => :array,
@@ -391,42 +398,56 @@ class Deb::S3::CLI < Thor
 
     configure_s3_client
 
-    # retrieve the existing manifests
-    log "Retrieving existing manifests"
-    from_manifest = Deb::S3::Manifest.retrieve(options[:codename],
-                                               component, arch,
+    begin
+      if options[:lock]
+        log("Checking for existing lock file")
+        log("Locking repository for updates")
+        Deb::S3::Lock.lock(options[:codename], to_component, options[:arch], options[:cache_control])
+        @lock_acquired = true
+      end
+
+      # retrieve the existing manifests
+      log "Retrieving existing manifests"
+      from_manifest = Deb::S3::Manifest.retrieve(options[:codename],
+                                                 component, arch,
+                                                 options[:cache_control],
+                                                 false, options[:skip_package_upload])
+      to_release = Deb::S3::Release.retrieve(to_codename)
+      to_manifest = Deb::S3::Manifest.retrieve(to_codename, to_component, arch,
                                                options[:cache_control],
-                                               false, options[:skip_package_upload])
-    to_release = Deb::S3::Release.retrieve(to_codename)
-    to_manifest = Deb::S3::Manifest.retrieve(to_codename, to_component, arch,
-                                             options[:cache_control],
-                                             options[:fail_if_exists],
-                                             options[:skip_package_upload])
-    packages = from_manifest.packages.select { |p|
-      p.name == package_name &&
-        (versions.nil? || versions.include?(p.full_version))
-    }
-    if packages.size == 0
-      error "No packages found in repository."
-    end
+                                               options[:fail_if_exists],
+                                               options[:skip_package_upload])
+      packages = from_manifest.packages.select { |p|
+        p.name == package_name &&
+          (versions.nil? || versions.include?(p.full_version))
+      }
+      if packages.size == 0
+        error "No packages found in repository."
+      end
+
+      packages.each do |package|
+        begin
+          to_manifest.add package, options[:preserve_versions], false
+        rescue Deb::S3::Utils::AlreadyExistsError => e
+          error("Preparing manifest failed because: #{e}")
+        end
+      end
 
-    packages.each do |package|
       begin
-        to_manifest.add package, options[:preserve_versions], false
+        to_manifest.write_to_s3 { |f| sublog("Transferring #{f}") }
       rescue Deb::S3::Utils::AlreadyExistsError => e
-        error("Preparing manifest failed because: #{e}")
+        error("Copying manifest failed because: #{e}")
       end
-    end
+      to_release.update_manifest(to_manifest)
+      to_release.write_to_s3 { |f| sublog("Transferring #{f}") }
 
-    begin
-      to_manifest.write_to_s3 { |f| sublog("Transferring #{f}") }
-    rescue Deb::S3::Utils::AlreadyExistsError => e
-      error("Copying manifest failed because: #{e}")
+      log "Copy complete."
+    ensure
+      if options[:lock] && @lock_acquired
+        Deb::S3::Lock.unlock(options[:codename], component, options[:arch], options[:cache_control])
+        log("Lock released.")
+      end
     end
-    to_release.update_manifest(to_manifest)
-    to_release.write_to_s3 { |f| sublog("Transferring #{f}") }
-
-    log "Copy complete."
   end
 
   desc "delete PACKAGE",
@@ -439,6 +460,13 @@ class Deb::S3::CLI < Thor
     :aliases  => "-a",
     :desc     => "The architecture of the package in the APT repository."
 
+  option :lock,
+  :default  => false,
+  :type     => :boolean,
+  :aliases  => "-l",
+  :desc     => "Whether to check for an existing lock on the repository " +
+    "to prevent simultaneous updates "
+
   option :versions,
     :default  => nil,
     :type     => :array,
@@ -465,30 +493,62 @@ class Deb::S3::CLI < Thor
 
     configure_s3_client
 
-    # retrieve the existing manifests
-    log("Retrieving existing manifests")
-    release  = Deb::S3::Release.retrieve(options[:codename], options[:origin], options[:suite])
-    manifest = Deb::S3::Manifest.retrieve(options[:codename], component, options[:arch], options[:cache_control], false, options[:skip_package_upload])
+    begin
+      if options[:lock]
+        log("Checking for existing lock file")
+        log("Locking repository for updates")
+        Deb::S3::Lock.lock(options[:codename], component, options[:arch], options[:cache_control])
+        @lock_acquired = true
+      end
 
-    deleted = manifest.delete_package(package, versions)
-    if deleted.length == 0
-        if versions.nil?
-            error("No packages were deleted. #{package} not found.")
+      # retrieve the existing manifests
+      log("Retrieving existing manifests")
+      release  = Deb::S3::Release.retrieve(options[:codename], options[:origin], options[:suite])
+      if arch == 'all'
+        selected_arch = release.architectures
+      else
+        selected_arch = [arch]
+      end
+      all_found = 0
+      selected_arch.each { |ar|
+        manifest = Deb::S3::Manifest.retrieve(options[:codename], component, ar, options[:cache_control], false, options[:skip_package_upload])
+
+        deleted = manifest.delete_package(package, versions)
+        all_found += deleted.length
+        if deleted.length == 0
+            if versions.nil?
+                sublog("No packages were deleted. #{package} not found in arch #{ar}.")
+                next
+            else
+                sublog("No packages were deleted. #{package} versions #{versions.join(', ')} could not be found in arch #{ar}.")
+                next
+            end
         else
-            error("No packages were deleted. #{package} versions #{versions.join(', ')} could not be found.")
+            deleted.each { |p|
+                sublog("Deleting #{p.name} version #{p.full_version} from arch #{ar}")
+            }
         end
-    else
-        deleted.each { |p|
-            sublog("Deleting #{p.name} version #{p.full_version}")
-        }
-    end
 
-    log("Uploading new manifests to S3")
-    manifest.write_to_s3 {|f| sublog("Transferring #{f}") }
-    release.update_manifest(manifest)
-    release.write_to_s3 {|f| sublog("Transferring #{f}") }
+        log("Uploading new manifests to S3")
+        manifest.write_to_s3 {|f| sublog("Transferring #{f}") }
+        release.update_manifest(manifest)
+        release.write_to_s3 {|f| sublog("Transferring #{f}") }
 
-    log("Update complete.")
+        log("Update complete.")
+      }
+      if all_found == 0
+        if versions.nil?
+          error("No packages were deleted. #{package} not found.")
+        else
+          error("No packages were deleted. #{package} versions #{versions.join(', ')} could not be found.")
+        end
+      end
+    ensure
+      if options[:lock] && @lock_acquired
+        Deb::S3::Lock.unlock(options[:codename], component, options[:arch], options[:cache_control])
+        log("Lock released.")
+      end
+    end
   end
 
 
@@ -514,9 +574,9 @@ class Deb::S3::CLI < Thor
       missing_packages = []
 
       manifest.packages.each do |p|
-        unless Deb::S3::Utils.s3_exists? p.url_filename_encoded
+        unless Deb::S3::Utils.s3_exists? p.url_filename_encoded(options[:codename])
           sublog("The following packages are missing:\n\n") if missing_packages.empty?
-          puts(p.generate)
+          puts(p.generate(options[:codename]))
           puts("")
 
           missing_packages << p
@@ -535,6 +595,92 @@ class Deb::S3::CLI < Thor
     end
   end
 
+  desc "clean", "Delete packages from the pool which are no longer referenced"
+
+  option :lock,
+  :default  => false,
+  :type     => :boolean,
+  :aliases  => "-l",
+  :desc     => "Whether to check for an existing lock on the repository " +
+    "to prevent simultaneous updates "
+
+  def clean
+    configure_s3_client
+
+    begin
+      if options[:lock]
+        log("Checking for existing lock file")
+        log("Locking repository for updates")
+        Deb::S3::Lock.lock(options[:codename], component, options[:arch], options[:cache_control])
+        @lock_acquired = true
+      end
+
+      log("Retrieving existing manifests")
+
+      # Enumerate objects under the dists/<codename>/ prefix to find any
+      # Packages files and load them....
+
+      req = Deb::S3::Utils.s3.list_objects_v2({
+        :bucket => Deb::S3::Utils.bucket,
+        :prefix => Deb::S3::Utils.s3_path("dists/#{ options[:codename] }/"),
+      })
+
+      manifests = []
+      req.contents.each do |object|
+        if match = object.key.match(/dists\/([^\/]+)\/([^\/]+)\/binary-([^\/]+)\/Packages$/)
+          codename, component, arch = match.captures
+          manifests.push(Deb::S3::Manifest.retrieve(codename, component, arch, options[:cache_control], options[:fail_if_exists], options[:skip_package_upload]))
+        end
+      end
+
+      # Iterate over the packages in each manifest and build a Set of all the
+      # referenced URLs (relative to bucket root)...
+
+      refd_urls = Set[]
+      manifests.each do |manifest|
+        manifest.packages.each do |package|
+          refd_urls.add(Deb::S3::Utils.s3_path(package.url_filename(manifest.codename)))
+        end
+      end
+
+      log("Searching for unreferenced packages")
+
+      # Enumerate objects under the pools/<codename> prefix and delete any that
+      # arent referenced by any of the manifests.
+
+      continuation_token = nil
+      while true
+        req = Deb::S3::Utils.s3.list_objects_v2({
+          :bucket => Deb::S3::Utils.bucket,
+          :prefix => Deb::S3::Utils.s3_path("pool/#{ options[:codename] }/"),
+          :continuation_token => continuation_token,
+        })
+
+        req.contents.each do |object|
+          if not refd_urls.include?(object.key)
+            sublog("Deleting #{ object.key }")
+
+            Deb::S3::Utils.s3.delete_object({
+              :bucket => Deb::S3::Utils.bucket,
+              :key => object.key,
+            })
+          end
+        end
+
+        if req.is_truncated
+          continuation_token = req.next_continuation_token
+        else
+          break
+        end
+      end
+    ensure
+      if options[:lock] && @lock_acquired
+        Deb::S3::Lock.unlock(options[:codename], component, options[:arch], options[:cache_control])
+        log("Lock released.")
+      end
+    end
+  end
+
   private
 
   def component

data/lib/deb/s3/lock.rb

@@ -1,58 +1,122 @@
 # -*- encoding : utf-8 -*-
-require "tempfile"
-require "socket"
+require "base64"
+require "digest/md5"
 require "etc"
+require "socket"
+require "tempfile"
 
 class Deb::S3::Lock
-  attr_accessor :user
-  attr_accessor :host
+  attr_reader :user
+  attr_reader :host
 
-  def initialize
-    @user = nil
-    @host = nil
+  def initialize(user, host)
+    @user = user
+    @host = host
   end
 
   class << self
-    def locked?(codename, component = nil, architecture = nil, cache_control = nil)
-      Deb::S3::Utils.s3_exists?(lock_path(codename, component, architecture, cache_control))
-    end
+    #
+    # 2-phase mutual lock mechanism based on `s3:CopyObject`.
+    #
+    # This logic isn't relying on S3's enhanced features like Object Lock
+    # because it imposes some limitation on using other features like
+    # S3 Cross-Region replication. This should work more than good enough 
+    # with S3's strong read-after-write consistency which we can presume
+    # in all region nowadays.
+    #
+    # This is relying on S3 to set object's ETag as object's MD5 if an
+    # object isn't comprized from multiple parts. We'd be able to presume
+    # it as the lock file is usually an object of some smaller bytes.
+    #
+    # acquire lock:
+    # 1. call `s3:HeadObject` on final lock object
+    #   1. If final lock object exists, restart from the beginning
+    #   2. Otherwise, call `s3:PutObject` to create initial lock object
+    # 2. Perform `s3:CopyObject` to copy from initial lock object
+    #    to final lock object with specifying ETag/MD5 of the initial
+    #    lock object
+    #   1. If copy object fails as `PreconditionFailed`, restart
+    #      from the beginning
+    #   2. Otherwise, lock has been acquired
+    #
+    # release lock:
+    # 1. remove final lock object by `s3:DeleteObject`
+    #
+    def lock(codename, component = nil, architecture = nil, cache_control = nil, max_attempts=60, max_wait_interval=10)
+      lockbody = "#{Etc.getlogin}@#{Socket.gethostname}"
+      initial_lockfile = initial_lock_path(codename, component, architecture, cache_control)
+      final_lockfile = lock_path(codename, component, architecture, cache_control)
 
-    def wait_for_lock(codename, component = nil, architecture = nil, cache_control = nil, max_attempts=60, wait=10)
-      attempts = 0
-      while self.locked?(codename, component, architecture, cache_control) do
-        attempts += 1
-        throw "Unable to obtain a lock after #{max_attempts}, giving up." if attempts > max_attempts
-        sleep(wait)
+      md5_b64 = Base64.encode64(Digest::MD5.digest(lockbody))
+      md5_hex = Digest::MD5.hexdigest(lockbody)
+      max_attempts.times do |i|
+        wait_interval = [(1<<i)/10, max_wait_interval].min
+        if Deb::S3::Utils.s3_exists?(final_lockfile)
+          lock = current(codename, component, architecture, cache_control)
+          $stderr.puts("Repository is locked by another user: #{lock.user} at host #{lock.host} (phase-1)")
+          $stderr.puts("Attempting to obtain a lock after #{wait_interval} secound(s).")
+          sleep(wait_interval)
+        else
+          # upload the file
+          Deb::S3::Utils.s3.put_object(
+            bucket: Deb::S3::Utils.bucket,
+            key: Deb::S3::Utils.s3_path(initial_lockfile),
+            body: lockbody,
+            content_type: "text/plain",
+            content_md5: md5_b64,
+            metadata: {
+              "md5" => md5_hex,
+            },
+          )
+          begin
+            Deb::S3::Utils.s3.copy_object(
+              bucket: Deb::S3::Utils.bucket,
+              key: Deb::S3::Utils.s3_path(final_lockfile),
+              copy_source: "/#{Deb::S3::Utils.bucket}/#{Deb::S3::Utils.s3_path(initial_lockfile)}",
+              copy_source_if_match: md5_hex,
+            )
+            return
+          rescue Aws::S3::Errors::PreconditionFailed => error
+            lock = current(codename, component, architecture, cache_control)
+            $stderr.puts("Repository is locked by another user: #{lock.user} at host #{lock.host} (phase-2)")
+            $stderr.puts("Attempting to obtain a lock after #{wait_interval} second(s).")
+            sleep(wait_interval)
+          end
+        end
       end
-    end
-
-    def lock(codename, component = nil, architecture = nil, cache_control = nil)
-      lockfile = Tempfile.new("lockfile")
-      lockfile.write("#{Etc.getlogin}@#{Socket.gethostname}")
-      lockfile.close
-
-      Deb::S3::Utils.s3_store(lockfile.path,
-                              lock_path(codename, component, architecture, cache_control),
-                              "text/plain",
-                              cache_control)
+      # TODO: throw appropriate error class
+      raise("Unable to obtain a lock after #{max_attempts}, giving up.")
     end
 
     def unlock(codename, component = nil, architecture = nil, cache_control = nil)
+      Deb::S3::Utils.s3_remove(initial_lock_path(codename, component, architecture, cache_control))
       Deb::S3::Utils.s3_remove(lock_path(codename, component, architecture, cache_control))
     end
 
     def current(codename, component = nil, architecture = nil, cache_control = nil)
-      lock_content = Deb::S3::Utils.s3_read(lock_path(codename, component, architecture, cache_control))
-      lock_content = lock_content.split('@')
-      lock = Deb::S3::Lock.new
-      lock.user = lock_content[0]
-      lock.host = lock_content[1] if lock_content.size > 1
+      lockbody = Deb::S3::Utils.s3_read(lock_path(codename, component, architecture, cache_control))
+      if lockbody
+        user, host = lockbody.to_s.split("@", 2)
+        lock = Deb::S3::Lock.new(user, host)
+      else
+        lock = Deb::S3::Lock.new("unknown", "unknown")
+      end
       lock
     end
 
     private
+    def initial_lock_path(codename, component = nil, architecture = nil, cache_control = nil)
+      "dists/#{codename}/lockfile.lock"
+    end
+
     def lock_path(codename, component = nil, architecture = nil, cache_control = nil)
-      "dists/#{codename}/#{component}/binary-#{architecture}/lockfile"
+      #
+      # Acquire repository lock at `codename` level to avoid race between concurrent upload attempts.
+      #
+      # * `deb-s3 upload --arch=all` touchs multiples of `dists/{codename}/{component}/binary-*/Packages*`
+      # * All `deb-s3 upload` touchs `dists/{codename}/Release`
+      #
+      "dists/#{codename}/lockfile"
     end
   end
 end

data/lib/deb/s3/manifest.rb

@@ -62,11 +62,11 @@ class Deb::S3::Manifest
       packages.each { |p|
         next unless p.name == pkg.name && \
                     p.full_version == pkg.full_version && \
-                    File.basename(p.url_filename) != \
-                    File.basename(pkg.url_filename)
+                    File.basename(p.url_filename(@codename)) != \
+                    File.basename(pkg.url_filename(@codename))
         raise AlreadyExistsError,
               "package #{pkg.name}_#{pkg.full_version} already exists " \
-              "with different filename (#{p.url_filename})"
+              "with different filename (#{p.url_filename(@codename)})"
       }
     end
     if preserve_versions
@@ -96,7 +96,7 @@ class Deb::S3::Manifest
   end
 
   def generate
-    @packages.collect { |pkg| pkg.generate }.join("\n")
+    @packages.collect { |pkg| pkg.generate(@codename) }.join("\n")
   end
 
   def write_to_s3
@@ -105,8 +105,8 @@ class Deb::S3::Manifest
     unless self.skip_package_upload
       # store any packages that need to be stored
       @packages_to_be_upload.each do |pkg|
-        yield pkg.url_filename if block_given?
-        s3_store(pkg.filename, pkg.url_filename, 'application/octet-stream; charset=binary', self.cache_control, self.fail_if_exists)
+        yield pkg.url_filename(@codename) if block_given?
+        s3_store(pkg.filename, pkg.url_filename(@codename), 'application/octet-stream; charset=binary', self.cache_control, self.fail_if_exists)
       end
     end
 
@@ -116,7 +116,7 @@ class Deb::S3::Manifest
     pkgs_temp.close
     f = "dists/#{@codename}/#{@component}/binary-#{@architecture}/Packages"
     yield f if block_given?
-    s3_store(pkgs_temp.path, f, 'text/plain; charset=UTF-8', self.cache_control)
+    s3_store(pkgs_temp.path, f, 'text/plain; charset=utf-8', self.cache_control)
     @files["#{@component}/binary-#{@architecture}/Packages"] = hashfile(pkgs_temp.path)
     pkgs_temp.unlink
 

data/lib/deb/s3/package.rb

@@ -2,6 +2,7 @@
 require "digest/sha1"
 require "digest/sha2"
 require "digest/md5"
+require "open3"
 require "socket"
 require "tmpdir"
 require "uri"
@@ -30,7 +31,6 @@ class Deb::S3::Package
   attr_accessor :attributes
 
   # hashes
-  attr_accessor :url_filename
   attr_accessor :sha1
   attr_accessor :sha256
   attr_accessor :md5
@@ -57,22 +57,36 @@ class Deb::S3::Package
 
     def extract_control(package)
       if system("which dpkg > /dev/null 2>&1")
-        `dpkg -f #{package}`
+        output, status = Open3.capture2("dpkg", "-f", package)
+        output
       else
+	# use ar to determine control file name (control.ext)
+        package_files = `ar t #{package}`
+        control_file = package_files.split("\n").select do |file|
+          file.start_with?("control.")
+        end.first
+        if control_file === "control.tar.gz"
+	  compression = "z"
+        elsif control_file === "control.tar.zst"
+          compression = "I zstd"
+	else
+	  compression = "J"
+        end
+
         # ar fails to find the control.tar.gz tarball within the .deb
         # on Mac OS. Try using ar to list the control file, if found,
         # use ar to extract, otherwise attempt with tar which works on OS X.
-        extract_control_tarball_cmd = "ar p #{package} control.tar.gz"
+        extract_control_tarball_cmd = "ar p #{package} #{control_file}"
 
         begin
-          safesystem("ar t #{package} control.tar.gz &> /dev/null")
+          safesystem("ar t #{package} #{control_file} &> /dev/null")
         rescue SafeSystemError
           warn "Failed to find control data in .deb with ar, trying tar."
-          extract_control_tarball_cmd = "tar zxf #{package} --to-stdout control.tar.gz"
+          extract_control_tarball_cmd = "tar -#{compression} -xf #{package} --to-stdout #{control_file}"
         end
 
         Dir.mktmpdir do |path|
-          safesystem("#{extract_control_tarball_cmd} | tar -zxf - -C #{path}")
+          safesystem("#{extract_control_tarball_cmd} | tar -#{compression} -xf - -C #{path}")
           File.read(File.join(path, "control"))
         end
       end
@@ -122,20 +136,19 @@ class Deb::S3::Package
     [[epoch, version].compact.join(":"), iteration].compact.join("-")
   end
 
-  def filename=(f)
-    @filename = f
-    @filename
+  def url_filename=(f)
+    @url_filename = f
   end
-
-  def url_filename
-    @url_filename || "pool/#{self.name[0]}/#{self.name[0..1]}/#{File.basename(self.filename)}"
+  
+  def url_filename(codename)
+    @url_filename || "pool/#{codename}/#{self.name[0]}/#{self.name[0..1]}/#{File.basename(self.filename)}"
   end
 
-  def url_filename_encoded
-    @url_filename || "pool/#{self.name[0]}/#{self.name[0..1]}/#{s3_escape(File.basename(self.filename))}"
+  def url_filename_encoded(codename)
+    @url_filename || "pool/#{codename}/#{self.name[0]}/#{self.name[0..1]}/#{s3_escape(File.basename(self.filename))}"
   end
 
-  def generate
+  def generate(codename)
     template("package.erb").result(binding)
   end
 
@@ -242,7 +255,7 @@ class Deb::S3::Package
 
     # Packages manifest fields
     filename = fields.delete('Filename')
-    self.url_filename = filename && URI.unescape(filename)
+    self.url_filename = filename && CGI.unescape(filename)
     self.sha1 = fields.delete('SHA1')
     self.sha256 = fields.delete('SHA256')
     self.md5 = fields.delete('MD5sum')

data/lib/deb/s3/release.rb

@@ -98,11 +98,11 @@ class Deb::S3::Release
     release_tmp.puts self.generate
     release_tmp.close
     yield self.filename if block_given?
-    s3_store(release_tmp.path, self.filename, 'text/plain; charset=UTF-8', self.cache_control)
+    s3_store(release_tmp.path, self.filename, 'text/plain; charset=utf-8', self.cache_control)
 
     # sign the file, if necessary
     if Deb::S3::Utils.signing_key
-      key_param = Deb::S3::Utils.signing_key != "" ? "--default-key=#{Deb::S3::Utils.signing_key}" : ""
+      key_param = Deb::S3::Utils.signing_key.any? ? "-u #{Deb::S3::Utils.signing_key.join(" -u ")}" : ""
       if system("gpg -a #{key_param} --digest-algo SHA256 #{Deb::S3::Utils.gpg_options} -s --clearsign #{release_tmp.path}")
         local_file = release_tmp.path+".asc"
         remote_file = "dists/#{@codename}/InRelease"

data/lib/deb/s3/templates/package.erb

@@ -40,7 +40,7 @@ Origin: <%= attributes[:deb_origin] %>
 <% end -%>
 Priority: <%= attributes[:deb_priority] %>
 Homepage: <%= url or "http://nourlgiven.example.com/" %>
-Filename: <%= url_filename_encoded %>
+Filename: <%= url_filename_encoded(codename) %>
 <% if size -%>
 Size: <%= size %>
 <% end -%>

data/lib/deb/s3/utils.rb

@@ -1,8 +1,6 @@
 # -*- encoding : utf-8 -*-
-require "base64"
 require "digest/md5"
 require "erb"
-require "tmpdir"
 
 module Deb::S3::Utils
   module_function
@@ -41,7 +39,7 @@ module Deb::S3::Utils
   def template(path)
     template_file = File.join(File.dirname(__FILE__), "templates", path)
     template_code = File.read(template_file)
-    ERB.new(template_code, nil, "-")
+    ERB.new(template_code, trim_mode: "-")
   end
 
   def s3_path(path)
@@ -70,7 +68,7 @@ module Deb::S3::Utils
       :key => s3_path(path),
     )[:body].read
   rescue Aws::S3::Errors::NoSuchKey
-    false
+    nil
   end
 
   def s3_store(path, filename=nil, content_type='application/octet-stream; charset=binary', cache_control=nil, fail_if_exists=false)
@@ -82,7 +80,7 @@ module Deb::S3::Utils
     # check if the object already exists
     if obj != false
       return if (file_md5.to_s == obj[:etag].gsub('"', '') or file_md5.to_s == obj[:metadata]['md5'])
-      raise AlreadyExistsError, "file #{obj.public_url} already exists with different contents" if fail_if_exists
+      raise AlreadyExistsError, "file #{filename} already exists with different contents" if fail_if_exists
     end
 
     options = {

data/lib/deb/s3.rb

@@ -1,6 +1,6 @@
 # -*- encoding : utf-8 -*-
 module Deb
   module S3
-    VERSION = "0.9.3"
+    VERSION = "0.11.5"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deb-s3
 version: !ruby/object:Gem::Version
-  version: 0.9.3
+  version: 0.11.5
 platform: ruby
 authors:
 - Ken Robertson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-05-17 00:00:00.000000000 Z
+date: 2022-11-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -16,28 +16,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.19.0
+        version: '1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.19.0
+        version: '1'
 - !ruby/object:Gem::Dependency
-  name: aws-sdk
+  name: aws-sdk-s3
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '1'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -96,15 +96,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 1.9.3
+      version: 2.7.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 3.3.7
 signing_key: 
 specification_version: 4
 summary: Easily create and manage an APT repository on S3.