deb-s3 0.5.1 → 0.6.0

data/README.md

@@ -59,25 +59,39 @@ Usage:
   deb-s3 upload FILES
 
 Options:
-  -a, [--arch=ARCH]              # The architecture of the package in the APT repository.
-      [--sign=SIGN]              # Sign the Release file. Use --sign with your key ID to use a specific key.
-  -p, [--preserve-versions]      # Whether to preserve other versions of a package in the repository when uploading one.
-  -b, [--bucket=BUCKET]          # The name of the S3 bucket to upload to.
-  -c, [--codename=CODENAME]      # The codename of the APT repository.
-                                 # Default: stable
-  -m, [--component=COMPONENT]    # The component of the APT repository.
-                                 # Default: main
-      [--access-key=ACCESS_KEY]  # The access key for connecting to S3.
-                                 # Default: $AMAZON_ACCESS_KEY_ID
-      [--secret-key=SECRET_KEY]  # The secret key for connecting to S3.
-                                 # Default: $AMAZON_SECRET_ACCESS_KEY
-      [--endpoint=ENDPOINT]      # The region endpoint for connecting to S3.
-  -v, [--visibility=VISIBILITY]  # The access policy for the uploaded files. Can be public, private, or authenticated.
-                                 # Default: public
+  -a, [--arch=ARCH]                     # The architecture of the package in the APT repository.
+      [--sign=SIGN]                     # Sign the Release file. Use --sign with your key ID to use a specific key.
+  -p, [--preserve-versions]             # Whether to preserve other versions of a package in the repository when uploading one.
+  -b, [--bucket=BUCKET]                 # The name of the S3 bucket to upload to.
+  -c, [--codename=CODENAME]             # The codename of the APT repository.
+                                        # Default: stable
+  -m, [--component=COMPONENT]           # The component of the APT repository.
+                                        # Default: main
+      [--access-key-id=ACCESS_KEY]      # The access key for connecting to S3.
+      [--secret-access-key=SECRET_KEY]  # The secret key for connecting to S3.
+  -v, [--visibility=VISIBILITY]         # The access policy for the uploaded files. Can be public, private, or authenticated.
+                                        # Default: public
 
 Uploads the given files to a S3 bucket as an APT repository.
 ```
 
+You can also delete packages from the APT repository. Please keep in mind that
+this does NOT delete the .deb file itself, it only removes it from the list of
+packages in the specified component, codename and architecture.
+
+Now to delete the package:
+```console
+$ deb-s3 delete --arch amd64 --bucket my-bucket --versions 1.0.0 my-deb-package
+>> Retrieving existing manifests
+   -- Deleting my-deb-package version 1.0.0
+>> Uploading new manifests to S3
+   -- Transferring dists/stable/main/binary-amd64/Packages
+   -- Transferring dists/stable/main/binary-amd64/Packages.gz
+   -- Transferring dists/stable/Release
+>> Update complete.
+
+````
+
 You can also verify an existing APT repository on S3 using the `verify` command:
 
 ```console
@@ -93,20 +107,17 @@ Usage:
   deb-s3 verify
 
 Options:
-  -f, [--fix-manifests]          # Whether to fix problems in manifests when verifying.
-      [--sign=SIGN]              # Sign the Release file. Use --sign with your key ID to use a specific key.
-  -b, [--bucket=BUCKET]          # The name of the S3 bucket to upload to.
-  -c, [--codename=CODENAME]      # The codename of the APT repository.
-                                 # Default: stable
-  -m, [--component=COMPONENT]    # The component of the APT repository.
-                                 # Default: main
-      [--access-key=ACCESS_KEY]  # The access key for connecting to S3.
-                                 # Default: $AMAZON_ACCESS_KEY_ID
-      [--secret-key=SECRET_KEY]  # The secret key for connecting to S3.
-                                 # Default: $AMAZON_SECRET_ACCESS_KEY
-      [--endpoint=ENDPOINT]      # The region endpoint for connecting to S3.
-  -v, [--visibility=VISIBILITY]  # The access policy for the uploaded files. Can be public, private, or authenticated.
-                                 # Default: public
+  -f, [--fix-manifests]                 # Whether to fix problems in manifests when verifying.
+      [--sign=SIGN]                     # Sign the Release file. Use --sign with your key ID to use a specific key.
+  -b, [--bucket=BUCKET]                 # The name of the S3 bucket to upload to.
+  -c, [--codename=CODENAME]             # The codename of the APT repository.
+                                        # Default: stable
+  -m, [--component=COMPONENT]           # The component of the APT repository.
+                                        # Default: main
+      [--access-key-id=ACCESS_KEY]      # The access key for connecting to S3.
+      [--secret-access-key=SECRET_KEY]  # The secret key for connecting to S3.
+  -v, [--visibility=VISIBILITY]         # The access policy for the uploaded files. Can be public, private, or authenticated.
+                                        # Default: public
 
 Verifies that the files in the package manifests exist
 ```

data/lib/deb/s3.rb

@@ -1,5 +1,5 @@
 module Deb
   module S3
-    VERSION = "0.5.1"
+    VERSION = "0.6.0"
   end
 end

data/lib/deb/s3/cli.rb

@@ -1,6 +1,9 @@
-require 'aws/s3'
+require "aws"
 require "thor"
 
+# Hack: aws requires this!
+require "json"
+
 require "deb/s3"
 require "deb/s3/utils"
 require "deb/s3/manifest"
@@ -8,75 +11,77 @@ require "deb/s3/package"
 require "deb/s3/release"
 
 class Deb::S3::CLI < Thor
-
   class_option :bucket,
-    :type     => :string,
-    :aliases  => "-b",
-    :desc     => "The name of the S3 bucket to upload to."
+  :type     => :string,
+  :aliases  => "-b",
+  :desc     => "The name of the S3 bucket to upload to."
+
+  class_option :prefix,
+  :type     => :string,
+  :desc     => "The path prefix to use when storing on S3."
 
   class_option :codename,
-    :default  => "stable",
-    :type     => :string,
-    :aliases  => "-c",
-    :desc     => "The codename of the APT repository."
+  :default  => "stable",
+  :type     => :string,
+  :aliases  => "-c",
+  :desc     => "The codename of the APT repository."
 
   class_option :component,
-    :default  => "main",
-    :type     => :string,
-    :aliases  => "-m",
-    :desc     => "The component of the APT repository."
+  :default  => "main",
+  :type     => :string,
+  :aliases  => "-m",
+  :desc     => "The component of the APT repository."
 
   class_option :section,
-    :type     => :string,
-    :aliases  => "-s",
-    :hide     => true
+  :type     => :string,
+  :aliases  => "-s",
+  :hide     => true
 
-  class_option :access_key,
-    :default  => "$AMAZON_ACCESS_KEY_ID",
-    :type     => :string,
-    :desc     => "The access key for connecting to S3."
+  class_option :access_key_id,
+  :type     => :string,
+  :desc     => "The access key for connecting to S3."
 
-  class_option :secret_key,
-    :default  => "$AMAZON_SECRET_ACCESS_KEY",
-    :type     => :string,
-    :desc     => "The secret key for connecting to S3."
+  class_option :secret_access_key,
+  :type     => :string,
+  :desc     => "The secret key for connecting to S3."
 
   class_option :endpoint,
-    :type     => :string,
-    :desc     => "The region endpoint for connecting to S3."
+  :type     => :string,
+  :desc     => "The region endpoint for connecting to S3.",
+  :default  => "s3.amazonaws.com"
 
   class_option :visibility,
-    :default  => "public",
-    :type     => :string,
-    :aliases  => "-v",
-    :desc     => "The access policy for the uploaded files. " +
-                 "Can be public, private, or authenticated."
+  :default  => "public",
+  :type     => :string,
+  :aliases  => "-v",
+  :desc     => "The access policy for the uploaded files. " +
+    "Can be public, private, or authenticated."
 
   class_option :sign,
-    :type     => :string,
-    :desc     => "Sign the Release file when uploading a package," +
-                 "or when verifying it after removing a package." +
-                 "Use --sign with your key ID to use a specific key."
+  :type     => :string,
+  :desc     => "Sign the Release file when uploading a package," +
+    "or when verifying it after removing a package." +
+    "Use --sign with your key ID to use a specific key."
 
   class_option :gpg_options,
-    :default => "",
-    :type    => :string,
-    :desc    => "Additional command line options to pass to GPG when signing"
+  :default => "",
+  :type    => :string,
+  :desc    => "Additional command line options to pass to GPG when signing"
 
   desc "upload FILES",
-    "Uploads the given files to a S3 bucket as an APT repository."
+  "Uploads the given files to a S3 bucket as an APT repository."
 
   option :arch,
-    :type     => :string,
-    :aliases  => "-a",
-    :desc     => "The architecture of the package in the APT repository."
+  :type     => :string,
+  :aliases  => "-a",
+  :desc     => "The architecture of the package in the APT repository."
 
   option :preserve_versions,
-    :default  => false,
-    :type     => :boolean,
-    :aliases  => "-p",
-    :desc     => "Whether to preserve other versions of a package " +
-                 "in the repository when uploading one."
+  :default  => false,
+  :type     => :boolean,
+  :aliases  => "-p",
+  :desc     => "Whether to preserve other versions of a package " +
+    "in the repository when uploading one."
 
   def upload(*files)
     component = options[:component]
@@ -132,13 +137,82 @@ class Deb::S3::CLI < Thor
     log("Update complete.")
   end
 
+  desc "delete PACKAGE",
+    "Remove the package named PACKAGE. If --versions is not specified, delete" +
+    "all versions of PACKAGE. Otherwise, only the specified versions will be " +
+    "deleted."
+
+  option :arch,
+    :type     => :string,
+    :aliases  => "-a",
+    :desc     => "The architecture of the package in the APT repository."
+
+  option :versions,
+    :default  => nil,
+    :type     => :array,
+    :desc     => "The space-delimited versions of PACKAGE to delete. If not" +
+    "specified, ALL VERSIONS will be deleted. Fair warning." +
+    "E.g. --versions \"0.1 0.2 0.3\""
+
+  def delete(package)
+    component = options[:component]
+    if options[:section]
+      component = options[:section]
+      warn("===> WARNING: The --section/-s argument is deprecated, please use --component/-m.")
+    end
+
+    if package.nil?
+      error("You must specify a package name.")
+    end
+
+    versions = options[:versions]
+    if versions.nil?
+      warn("===> WARNING: Deleting all versions of #{package}")
+    else
+      log("Versions to delete: #{versions.join(', ')}")
+    end
+
+    arch = options[:arch]
+    if arch.nil?
+      error("You must specify the architecture of the package to remove.")
+    end
+
+    configure_s3_client
+
+    # retrieve the existing manifests
+    log("Retrieving existing manifests")
+    release  = Deb::S3::Release.retrieve(options[:codename])
+    manifest = Deb::S3::Manifest.retrieve(options[:codename], component, options[:arch])
+
+    deleted = manifest.delete_package(package, versions)
+    if deleted.length == 0
+        if versions.nil?
+            error("No packages were deleted. #{package} not found.")
+        else
+            error("No packages were deleted. #{package} versions #{versions.join(', ')} could not be found.")
+        end
+    else
+        deleted.each { |p|
+            sublog("Deleting #{p.name} version #{p.version}")
+        }
+    end
+
+    log("Uploading new manifests to S3")
+    manifest.write_to_s3 {|f| sublog("Transferring #{f}") }
+    release.update_manifest(manifest)
+    release.write_to_s3 {|f| sublog("Transferring #{f}") }
+
+    log("Update complete.")
+  end
+
+
   desc "verify", "Verifies that the files in the package manifests exist"
 
   option :fix_manifests,
-    :default  => false,
-    :type     => :boolean,
-    :aliases  => "-f",
-    :desc     => "Whether to fix problems in manifests when verifying."
+  :default  => false,
+  :type     => :boolean,
+  :aliases  => "-f",
+  :desc     => "Whether to fix problems in manifests when verifying."
 
   def verify
     component = options[:component]
@@ -152,7 +226,7 @@ class Deb::S3::CLI < Thor
     log("Retrieving existing manifests")
     release = Deb::S3::Release.retrieve(options[:codename])
 
-    %w[i386 amd64 all].each do |arch|
+    %w[amd64 armel i386 all].each do |arch|
       log("Checking for missing packages in: #{options[:codename]}/#{options[:component]} #{arch}")
       manifest = Deb::S3::Manifest.retrieve(options[:codename], component, arch)
       missing_packages = []
@@ -194,48 +268,44 @@ class Deb::S3::CLI < Thor
     exit 1
   end
 
-  def access_key
-    if options[:access_key] == "$AMAZON_ACCESS_KEY_ID"
-      ENV["AMAZON_ACCESS_KEY_ID"]
-    else
-      options[:access_key]
-    end
-  end
+  def provider
+    access_key_id     = options[:access_key_id]
+    secret_access_key = options[:secret_access_key]
 
-  def secret_key
-    if options[:secret_key] == "$AMAZON_SECRET_ACCESS_KEY"
-      ENV["AMAZON_SECRET_ACCESS_KEY"]
-    else
-      options[:secret_key]
+    if access_key_id.nil? ^ secret_access_key.nil?
+      error("If you specify one of --access-key-id or --secret-access-key, you must specify the other.")
     end
+
+    static_credentials = {}
+    static_credentials[:access_key_id]     = access_key_id     if access_key_id
+    static_credentials[:secret_access_key] = secret_access_key if secret_access_key
+
+    AWS::Core::CredentialProviders::DefaultProvider.new(static_credentials)
   end
 
   def configure_s3_client
-    error("No access key given for S3. Please specify one.") unless access_key
-    error("No secret access key given for S3. Please specify one.") unless secret_key
     error("No value provided for required options '--bucket'") unless options[:bucket]
 
-    AWS::S3::Base.establish_connection!(
-      :access_key_id     => access_key,
-      :secret_access_key => secret_key
-    )
-
-    AWS::S3::DEFAULT_HOST.replace options[:endpoint] if options[:endpoint]
+    settings = { :s3_endpoint => options[:endpoint] }
+    settings.merge!(provider.credentials)
 
+    Deb::S3::Utils.s3          = AWS::S3.new(settings)
     Deb::S3::Utils.bucket      = options[:bucket]
     Deb::S3::Utils.signing_key = options[:sign]
     Deb::S3::Utils.gpg_options = options[:gpg_options]
+    Deb::S3::Utils.prefix      = options[:prefix]
 
     # make sure we have a valid visibility setting
-    Deb::S3::Utils.access_policy = case options[:visibility]
-    when "public"
-      :public_read
-    when "private"
-      :private
-    when "authenticated"
-      :authenticated_read
-    else
-      error("Invalid visibility setting given. Can be public, private, or authenticated.")
-    end
+    Deb::S3::Utils.access_policy =
+      case options[:visibility]
+      when "public"
+        :public_read
+      when "private"
+        :private
+      when "authenticated"
+        :authenticated_read
+      else
+        error("Invalid visibility setting given. Can be public, private, or authenticated.")
+      end
   end
 end

data/lib/deb/s3/manifest.rb

@@ -55,6 +55,22 @@ class Deb::S3::Manifest
     pkg
   end
 
+  def delete_package(pkg, versions=nil)
+    deleted = []
+    new_packages = @packages.select { |p|
+        # Include packages we didn't name
+        if p.name != pkg
+           p
+        # Also include the packages not matching a specified version
+        elsif (!versions.nil? and p.name == pkg and !versions.include? p.version)
+            p
+        end
+    }
+    deleted = @packages - new_packages
+    @packages = new_packages
+    deleted
+  end
+
   def generate
     @packages.collect { |pkg| pkg.generate }.join("\n")
   end

data/lib/deb/s3/package.rb

@@ -214,7 +214,7 @@ class Deb::S3::Package
 
   # from fpm
   def extract_info(control)
-    parse = lambda do |field| 
+    parse = lambda do |field|
       value = control[/^#{field}: .*/]
       if value.nil?
         return nil
@@ -239,6 +239,7 @@ class Deb::S3::Package
     self.url = parse.call("Homepage")
     self.vendor = parse.call("Vendor") || self.vendor
     self.attributes[:deb_priority] = parse.call("Priority")
+    self.attributes[:deb_origin] = parse.call("Origin")
     self.attributes[:deb_installed_size] = parse.call("Installed-Size")
 
     # Packages manifest fields

data/lib/deb/s3/templates/package.erb

@@ -11,7 +11,7 @@ Depends: <%= dependencies.collect { |d| fix_dependency(d) }.flatten.join(", ") %
 <% if !conflicts.empty? -%>
 Conflicts: <%= conflicts.join(", ") %>
 <% end -%>
-<% if attributes[:deb_pre_depends_given?] -%>
+<% if attributes[:deb_pre_depends] -%>
 Pre-Depends: <%= attributes[:deb_pre_depends].collect { |d| fix_dependency(d) }.flatten.join(", ") %>
 <% end -%>
 <% if !provides.empty? -%>
@@ -22,13 +22,16 @@ Provides: <%= provides.map {|p| p.split(" ").first}.join ", " %>
 <% if !replaces.empty? -%>
 Replaces: <%= replaces.join(", ") %>
 <% end -%>
-<% if attributes[:deb_recommends_given?] -%>
+<% if attributes[:deb_recommends] -%>
 Recommends: <%= attributes[:deb_recommends].collect { |d| fix_dependency(d) }.flatten.join(", ") %>
 <% end -%>
-<% if attributes[:deb_suggests_given?] -%>
+<% if attributes[:deb_suggests] -%>
 Suggests: <%= attributes[:deb_suggests].collect { |d| fix_dependency(d) }.flatten.join(", ") %>
 <% end -%>
 Section: <%= category %>
+<% if attributes[:deb_origin] -%>
+Origin: <%= attributes[:deb_origin] %>
+<% end -%>
 Priority: <%= attributes[:deb_priority] %>
 Homepage: <%= url or "http://nourlgiven.example.com/" %>
 Filename: <%= url_filename_encoded %>
@@ -50,7 +53,7 @@ Description: <%= firstline %>
 <% if remainder.any? -%>
 <%= remainder.collect { |l| l =~ /^ *$/ ? " ." : " #{l}" }.join("\n") %>
 <% end -%>
-<% if attributes[:deb_field_given?] -%>
+<% if attributes[:deb_field] -%>
 <%   attributes[:deb_field].each do |field, value| -%>
 <%= field %>: <%= value %>
 <%   end -%>

data/lib/deb/s3/utils.rb

@@ -1,8 +1,12 @@
+require "base64"
+require "digest/md5"
 require "erb"
 require "tmpdir"
 
 module Deb::S3::Utils
   module_function
+  def s3; @s3 end
+  def s3= v; @s3 = v end
   def bucket; @bucket end
   def bucket= v; @bucket = v end
   def access_policy; @access_policy end
@@ -11,6 +15,8 @@ module Deb::S3::Utils
   def signing_key= v; @signing_key = v end
   def gpg_options; @gpg_options end
   def gpg_options= v; @gpg_options = v end
+  def prefix; @prefix end
+  def prefix= v; @prefix = v end
 
   def safesystem(*args)
     success = system(*args)
@@ -32,6 +38,10 @@ module Deb::S3::Utils
     ERB.new(template_code, nil, "-")
   end
 
+  def s3_path(path)
+    File.join(*[Deb::S3::Utils.prefix, path].compact)
+  end
+
   # from fog, Fog::AWS.escape
   def s3_escape(string)
     string.gsub(/([^a-zA-Z0-9_.\-~]+)/) {
@@ -40,27 +50,29 @@ module Deb::S3::Utils
   end
 
   def s3_exists?(path)
-    AWS::S3::S3Object.exists?(path, Deb::S3::Utils.bucket)
+    Deb::S3::Utils.s3.buckets[Deb::S3::Utils.bucket].objects[s3_path(path)].exists?
   end
 
   def s3_read(path)
     return nil unless s3_exists?(path)
-    s = ""
-    AWS::S3::S3Object.stream(path, Deb::S3::Utils.bucket) do |chunk|
-      s += chunk
-    end
-    s
+    Deb::S3::Utils.s3.buckets[Deb::S3::Utils.bucket].objects[s3_path(path)].read
   end
 
   def s3_store(path, filename=nil)
     filename = File.basename(path) unless filename
-    File.open(path) do |file|
-      AWS::S3::S3Object.store(filename, file,
-        Deb::S3::Utils.bucket, :access => Deb::S3::Utils.access_policy)
+    obj = Deb::S3::Utils.s3.buckets[Deb::S3::Utils.bucket].objects[s3_path(filename)]
+
+    # check if the object already exists
+    if obj.exists?
+      file_md5 = Digest::MD5.file(path)
+      return if file_md5.to_s == obj.etag.gsub('"', '')
     end
+
+    # upload the file
+    obj.write(Pathname.new(path), :acl => Deb::S3::Utils.access_policy)
   end
 
   def s3_remove(path)
-    AWS::S3::S3Object.delete(path, Deb::S3::Utils.bucket) if s3_exists?(path)
+    Deb::S3::Utils.s3.buckets[Deb::S3::Utils.bucket].objects[s3_path(path)].delete if s3_exists?(path)
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: deb-s3
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.6.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-02 00:00:00.000000000 Z
+date: 2013-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -28,13 +28,13 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.18.0
 - !ruby/object:Gem::Dependency
-  name: aws-s3
+  name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.6.3
+        version: '1.18'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -42,7 +42,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.6.3
+        version: '1.18'
 description: Easily create and manage an APT repository on S3.
 email: ken@invalidlogic.com
 executables: