dpkg-s3 0.3.1 → 0.4.0

data/lib/dpkg/s3/release.rb

@@ -1,161 +1,159 @@
-# -*- encoding : utf-8 -*-
-require "tempfile"
-
-class Dpkg::S3::Release
-  include Dpkg::S3::Utils
-
-  attr_accessor :codename
-  attr_accessor :origin
-  attr_accessor :suite
-  attr_accessor :architectures
-  attr_accessor :components
-  attr_accessor :cache_control
-
-  attr_accessor :files
-  attr_accessor :policy
-
-  def initialize
-    @origin = nil
-    @suite = nil
-    @codename = nil
-    @architectures = []
-    @components = []
-    @cache_control = ""
-    @files = {}
-    @policy = :public_read
-  end
-
-  class << self
-    def retrieve(codename, origin=nil, suite=nil, cache_control=nil)
-      if s = Dpkg::S3::Utils.s3_read("dists/#{codename}/Release")
-        rel = self.parse_release(s)
-      else
-        rel = self.new
+# frozen_string_literal: true
+
+require 'tempfile'
+
+module Dpkg
+  module S3
+    # Release is resposible of creating/retrieving and rebuilding the debian Release manifest with
+    # standard information required when publishing the package to a debian repository
+    class Release
+      include Dpkg::S3::Utils
+
+      attr_accessor :codename, :origin, :suite, :architectures, :components, :cache_control, :files, :policy
+
+      def initialize
+        @origin = nil
+        @suite = nil
+        @codename = nil
+        @architectures = []
+        @components = []
+        @cache_control = ''
+        @files = {}
+        @policy = :public_read
       end
-      rel.codename = codename
-      rel.origin = origin unless origin.nil?
-      rel.suite = suite unless suite.nil?
-      rel.cache_control = cache_control
-      rel
-    end
-
-    def parse_release(str)
-      rel = self.new
-      rel.parse(str)
-      rel
-    end
-  end
 
-  def filename
-    "dists/#{@codename}/Release"
-  end
+      class << self
+        def retrieve(codename, origin = nil, suite = nil, cache_control = nil)
+          rel = if (s = Dpkg::S3::Utils.s3_read("dists/#{codename}/Release"))
+                  parse_release(s)
+                else
+                  new
+                end
+          rel.codename = codename
+          rel.origin = origin unless origin.nil?
+          rel.suite = suite unless suite.nil?
+          rel.cache_control = cache_control
+          rel
+        end
 
-  def parse(str)
-    parse = lambda do |field|
-      value = str[/^#{field}: .*/]
-      if value.nil?
-        return nil
-      else
-        return value.split(": ",2).last
+        def parse_release(str)
+          rel = new
+          rel.parse(str)
+          rel
+        end
       end
-    end
 
-    # grab basic fields
-    self.codename = parse.call("Codename")
-    self.origin = parse.call("Origin") || nil
-    self.suite = parse.call("Suite") || nil
-    self.architectures = (parse.call("Architectures") || "").split(/\s+/)
-    self.components = (parse.call("Components") || "").split(/\s+/)
-
-    # find all the hashes
-    str.scan(/^\s+([^\s]+)\s+(\d+)\s+(.+)$/).each do |(hash,size,name)|
-      self.files[name] ||= { :size => size.to_i }
-      case hash.length
-      when 32
-        self.files[name][:md5] = hash
-      when 40
-        self.files[name][:sha1] = hash
-      when 64
-        self.files[name][:sha256] = hash
+      def filename
+        "dists/#{@codename}/Release"
       end
-    end
-  end
 
-  def generate
-    template("release.erb").result(binding)
-  end
+      def parse(str)
+        parse = lambda do |field|
+          value = str[/^#{field}: .*/]
+          return nil if value.nil?
 
-  def write_to_s3
-    # validate some other files are present
-    if block_given?
-      self.validate_others { |f| yield f }
-    else
-      self.validate_others
-    end
+          return value.split(': ', 2).last
+        end
 
-    # generate the Release file
-    release_tmp = Tempfile.new("Release")
-    release_tmp.puts self.generate
-    release_tmp.close
-    yield self.filename if block_given?
-    s3_store(release_tmp.path, self.filename, 'text/plain; charset=utf-8', self.cache_control)
-
-    # sign the file, if necessary
-    if Dpkg::S3::Utils.signing_key
-      key_param = Dpkg::S3::Utils.signing_key != "" ? "--default-key=#{Dpkg::S3::Utils.signing_key}" : ""
-      if system("gpg -a #{key_param} --digest-algo SHA256 #{Dpkg::S3::Utils.gpg_options} -s --clearsign #{release_tmp.path}")
-        local_file = release_tmp.path+".asc"
-        remote_file = "dists/#{@codename}/InRelease"
-        yield remote_file if block_given?
-        raise "Unable to locate InRelease file" unless File.exists?(local_file)
-        s3_store(local_file, remote_file, 'application/pgp-signature; charset=us-ascii', self.cache_control)
-        File.unlink(local_file)
-      else
-        raise "Signing the InRelease file failed."
+        # grab basic fields
+        self.codename = parse.call('Codename')
+        self.origin = parse.call('Origin') || nil
+        self.suite = parse.call('Suite') || nil
+        self.architectures = (parse.call('Architectures') || '').split(/\s+/)
+        self.components = (parse.call('Components') || '').split(/\s+/)
+
+        # find all the hashes
+        str.scan(/^\s+([^\s]+)\s+(\d+)\s+(.+)$/).each do |(hash, size, name)|
+          files[name] ||= { size: size.to_i }
+          case hash.length
+          when 32
+            files[name][:md5] = hash
+          when 40
+            files[name][:sha1] = hash
+          when 64
+            files[name][:sha256] = hash
+          end
+        end
       end
-      if system("gpg -a #{key_param} --digest-algo SHA256 #{Dpkg::S3::Utils.gpg_options} -b #{release_tmp.path}")
-        local_file = release_tmp.path+".asc"
-        remote_file = self.filename+".gpg"
-        yield remote_file if block_given?
-        raise "Unable to locate Release signature file" unless File.exists?(local_file)
-        s3_store(local_file, remote_file, 'application/pgp-signature; charset=us-ascii', self.cache_control)
-        File.unlink(local_file)
-      else
-        raise "Signing the Release file failed."
+
+      def generate
+        template('release.erb').result(binding)
       end
-    else
-      # remove an existing Release.gpg, if it was there
-      s3_remove(self.filename+".gpg")
-    end
 
-    release_tmp.unlink
-  end
+      def write_to_s3(&block)
+        # validate some other files are present
+        if block_given?
+          validate_others(&block)
+        else
+          validate_others
+        end
 
-  def update_manifest(manifest)
-    self.components << manifest.component unless self.components.include?(manifest.component)
-    self.architectures << manifest.architecture unless self.architectures.include?(manifest.architecture)
-    self.files.merge!(manifest.files)
-  end
+        # generate the Release file
+        release_tmp = Tempfile.new('Release')
+        release_tmp.puts generate
+        release_tmp.close
+        yield filename if block_given?
+        s3_store(release_tmp.path, filename, 'text/plain; charset=utf-8', cache_control)
+
+        # sign the file, if necessary
+        if Dpkg::S3::Utils.signing_key
+          key_param = Dpkg::S3::Utils.signing_key == '' ? '' : "--default-key=#{Dpkg::S3::Utils.signing_key}"
+          gpg_clear = "gpg -a #{key_param} --digest-algo SHA256 #{Dpkg::S3::Utils.gpg_options} -s --clearsign #{release_tmp.path}" # rubocop:disable Layout/LineLength
+          gpg_sign = "gpg -a #{key_param} --digest-algo SHA256 #{Dpkg::S3::Utils.gpg_options} -b #{release_tmp.path}"
+          raise 'Signing the InRelease file failed.' unless system(gpg_clear)
+
+          local_file = "#{release_tmp.path}.asc"
+          remote_file = "dists/#{@codename}/InRelease"
+          yield remote_file if block_given?
+          raise 'Unable to locate InRelease file' unless File.exist?(local_file)
+
+          s3_store(local_file, remote_file, 'application/pgp-signature; charset=us-ascii', cache_control)
+          File.unlink(local_file)
+
+          raise 'Signing the Release file failed.' unless system(gpg_sign)
+
+          local_file = "#{release_tmp.path}.asc"
+          remote_file = "#{filename}.gpg"
+          yield remote_file if block_given?
+          raise 'Unable to locate Release signature file' unless File.exist?(local_file)
+
+          s3_store(local_file, remote_file, 'application/pgp-signature; charset=us-ascii', cache_control)
+          File.unlink(local_file)
+        else
+          # remove an existing Release.gpg, if it was there
+          s3_remove("#{filename}.gpg")
+        end
 
-  def validate_others
-    to_apply = []
-    self.components.each do |comp|
-      %w(amd64 i386 armhf).each do |arch|
-        next if self.files.has_key?("#{comp}/binary-#{arch}/Packages")
+        release_tmp.unlink
+      end
 
-        m = Dpkg::S3::Manifest.new
-        m.codename = self.codename
-        m.component = comp
-        m.architecture = arch
-        if block_given?
-          m.write_to_s3 { |f| yield f }
-        else
-          m.write_to_s3
+      def update_manifest(manifest)
+        components << manifest.component unless components.include?(manifest.component)
+        architectures << manifest.architecture unless architectures.include?(manifest.architecture)
+        files.merge!(manifest.files)
+      end
+
+      def validate_others(&block)
+        to_apply = []
+        components.each do |comp|
+          %w[amd64 i386 armhf].each do |arch|
+            next if files.key?("#{comp}/binary-#{arch}/Packages")
+
+            m = Dpkg::S3::Manifest.new
+            m.codename = codename
+            m.component = comp
+            m.architecture = arch
+            if block_given?
+              m.write_to_s3(&block)
+            else
+              m.write_to_s3
+            end
+            to_apply << m
+          end
         end
-        to_apply << m
+
+        to_apply.each { |m| update_manifest(m) }
       end
     end
-
-    to_apply.each { |m| self.update_manifest(m) }
   end
 end

data/lib/dpkg/s3/templates/package.erb

@@ -40,7 +40,7 @@ Origin: <%= attributes[:deb_origin] %>
 <% end -%>
 Priority: <%= attributes[:deb_priority] %>
 Homepage: <%= url or "http://nourlgiven.example.com/" %>
-Filename: <%= url_filename(codename) %>
+Filename: <%= url_filename_encoded(codename) %>
 <% if size -%>
 Size: <%= size %>
 <% end -%>

data/lib/dpkg/s3/utils.rb

@@ -1,117 +1,115 @@
-# -*- encoding : utf-8 -*-
-require "base64"
-require "digest/md5"
-require "erb"
-require "tmpdir"
-
-module Dpkg::S3::Utils
-  module_function
-  def s3; @s3 end
-  def s3= v; @s3 = v end
-  def bucket; @bucket end
-  def bucket= v; @bucket = v end
-  def access_policy; @access_policy end
-  def access_policy= v; @access_policy = v end
-  def signing_key; @signing_key end
-  def signing_key= v; @signing_key = v end
-  def gpg_options; @gpg_options end
-  def gpg_options= v; @gpg_options = v end
-  def prefix; @prefix end
-  def prefix= v; @prefix = v end
-  def encryption; @encryption end
-  def encryption= v; @encryption = v end
-
-  class SafeSystemError < RuntimeError; end
-  class AlreadyExistsError < RuntimeError; end
-
-  def safesystem(*args)
-    success = system(*args)
-    if !success
-      raise SafeSystemError, "'system(#{args.inspect})' failed with error code: #{$?.exitstatus}"
-    end
-    return success
-  end
-
-  def debianize_op(op)
-    # Operators in debian packaging are <<, <=, =, >= and >>
-    # So any operator like < or > must be replaced
-    {:< => "<<", :> => ">>"}[op.to_sym] or op
-  end
-
-  def template(path)
-    template_file = File.join(File.dirname(__FILE__), "templates", path)
-    template_code = File.read(template_file)
-    ERB.new(template_code, nil, "-")
-  end
-
-  def s3_path(path)
-    File.join(*[Dpkg::S3::Utils.prefix, path].compact)
-  end
-
-  # from fog, Fog::AWS.escape
-  def s3_escape(string)
-    string.gsub(/([^a-zA-Z0-9_.\-~+]+)/) {
-      "%" + $1.unpack("H2" * $1.bytesize).join("%").upcase
-    }
-  end
-
-  def s3_exists?(path)
-    Dpkg::S3::Utils.s3.head_object(
-      :bucket => Dpkg::S3::Utils.bucket,
-      :key => s3_path(path),
-    )
-  rescue Aws::S3::Errors::NotFound
-    false
-  end
-
-  def s3_read(path)
-    Dpkg::S3::Utils.s3.get_object(
-      :bucket => Dpkg::S3::Utils.bucket,
-      :key => s3_path(path),
-    )[:body].read
-  rescue Aws::S3::Errors::NoSuchKey
-    false
-  end
-
-  def s3_store(path, filename=nil, content_type='application/x-debian-package', cache_control=nil, fail_if_exists=false)
-    filename = File.basename(path) unless filename
-    obj = s3_exists?(filename)
-
-    file_md5 = Digest::MD5.file(path)
-
-    # check if the object already exists
-    if obj != false
-      return if (file_md5.to_s == obj[:etag].gsub('"', '') or file_md5.to_s == obj[:metadata]['md5'])
-      raise AlreadyExistsError, "file #{filename} already exists with different contents" if fail_if_exists
-    end
-
-    options = {
-      :bucket => Dpkg::S3::Utils.bucket,
-      :key => s3_path(filename),
-      :acl => Dpkg::S3::Utils.access_policy,
-      :content_type => content_type,
-      :metadata => { "md5" => file_md5.to_s },
-    }
-    if !cache_control.nil?
-      options[:cache_control] = cache_control
-    end
-
-    # specify if encryption is required
-    options[:server_side_encryption] = "AES256" if Dpkg::S3::Utils.encryption
-
-    # upload the file
-    File.open(path) do |f|
-      options[:body] = f
-      Dpkg::S3::Utils.s3.put_object(options)
-    end
-  end
-
-  def s3_remove(path)
-    if s3_exists?(path)
-      Dpkg::S3::Utils.s3.delete_object(
-        :bucket =>Dpkg::S3::Utils.bucket,
-        :key => s3_path(path),
-      )
+# frozen_string_literal: true
+
+require 'base64'
+require 'digest/md5'
+require 'erb'
+require 'tmpdir'
+
+# Dpkg is the root module for all storage modules including S3
+module Dpkg
+  # S3 storage module resposible of handling packages on S3 including upload, delete
+  module S3
+    # Utils contains functions will be used in Package and Release modules
+    module Utils
+      extend self
+
+      attr_accessor :s3, :bucket, :access_policy, :signing_key, :gpg_options, :prefix, :encryption
+
+      class SafeSystemError < RuntimeError; end
+
+      class AlreadyExistsError < RuntimeError; end
+
+      def safesystem(*args)
+        success = system(*args)
+        unless success
+          raise SafeSystemError,
+                "'system(#{args.inspect})' failed with error code: #{$CHILD_STATUS.exitstatus}"
+        end
+
+        success
+      end
+
+      def debianize_op(operator)
+        # Operators in debian packaging are <<, <=, =, >= and >>
+        # So any operator like < or > must be replaced
+        { :< => '<<', :> => '>>' }[operator.to_sym] or operator
+      end
+
+      def template(path)
+        template_file = File.join(File.dirname(__FILE__), 'templates', path)
+        template_code = File.read(template_file)
+        ERB.new(template_code, nil, '-')
+      end
+
+      def s3_path(path)
+        File.join(*[Dpkg::S3::Utils.prefix, path].compact)
+      end
+
+      # from fog, Fog::AWS.escape
+      def s3_escape(string)
+        string.gsub(/([^a-zA-Z0-9_.\-~+]+)/) do
+          "%#{Regexp.last_match(1).unpack('H2' * Regexp.last_match(1).bytesize).join('%').upcase}"
+        end
+      end
+
+      def s3_exists?(path)
+        Dpkg::S3::Utils.s3.head_object(
+          bucket: Dpkg::S3::Utils.bucket,
+          key: s3_path(path)
+        )
+      rescue Aws::S3::Errors::NotFound
+        false
+      end
+
+      def s3_read(path)
+        Dpkg::S3::Utils.s3.get_object(
+          bucket: Dpkg::S3::Utils.bucket,
+          key: s3_path(path)
+        )[:body].read
+      rescue Aws::S3::Errors::NoSuchKey
+        false
+      end
+
+      def s3_store(path, filename = nil, content_type = 'application/x-debian-package',
+                   cache_control = nil, fail_if_exists: false)
+        filename ||= File.basename(path)
+        obj = s3_exists?(filename)
+
+        file_md5 = Digest::MD5.file(path)
+
+        # check if the object already exists
+        if obj != false
+          return if (file_md5.to_s == obj[:etag].gsub('"', '')) || (file_md5.to_s == obj[:metadata]['md5'])
+          raise AlreadyExistsError, "file #{filename} already exists with different contents" if fail_if_exists
+        end
+
+        options = {
+          bucket: Dpkg::S3::Utils.bucket,
+          key: s3_path(filename),
+          acl: Dpkg::S3::Utils.access_policy,
+          content_type: content_type,
+          metadata: { 'md5' => file_md5.to_s }
+        }
+        options[:cache_control] = cache_control unless cache_control.nil?
+
+        # specify if encryption is required
+        options[:server_side_encryption] = 'AES256' if Dpkg::S3::Utils.encryption
+
+        # upload the file
+        File.open(path) do |f|
+          options[:body] = f
+          Dpkg::S3::Utils.s3.put_object(options)
+        end
+      end
+
+      def s3_remove(path)
+        return unless s3_exists?(path)
+
+        Dpkg::S3::Utils.s3.delete_object(
+          bucket: Dpkg::S3::Utils.bucket,
+          key: s3_path(path)
+        )
+      end
     end
   end
 end