deb-s3-lock-fix 0.11.8.fix0.1

data/lib/deb/s3/lock.rb

@@ -0,0 +1,96 @@
+require "aws-sdk-dynamodb"
+require "securerandom"
+require "etc"
+require "time"
+
+class Deb::S3::Lock
+  attr_reader :user, :host_with_uuid
+
+  DYNAMODB_TABLE_NAME = 'deb-s3-lock'
+
+  def initialize(user, host_with_uuid)
+    @user = user
+    @host_with_uuid = host_with_uuid
+  end
+
+  def self.dynamodb
+    @dynamodb ||= begin
+      validate_environment_variables!
+      Aws::DynamoDB::Client.new(
+        access_key_id: ENV['DEB_S3_ACCESS_KEY_ID'],
+        secret_access_key: ENV['DEB_S3_SECRET_ACCESS_KEY'],
+        region: ENV['AWS_BUILDERS_REGION']
+      )
+    end
+  end
+
+  def self.validate_environment_variables!
+    %w[DEB_S3_LOCK_ACCESS_KEY_ID DEB_S3_LOCK_SECRET_ACCESS_KEY AWS_BUILDERS_REGION].each do |var|
+      raise "Environment variable #{var} not set." unless ENV[var]
+    end
+  end
+
+  class << self
+    def lock(codename, max_attempts = 60, max_wait_interval = 10)
+      uuid = SecureRandom.uuid
+      lock_body = "#{Etc.getlogin}@#{Socket.gethostname}-#{uuid}"
+      lock_key = codename
+
+      $stderr.puts("Current job's hostname with UUID: #{lock_body}")
+
+      max_attempts.times do |i|
+        wait_interval = [2**i, max_wait_interval].min
+
+        expiration_time = Time.now.to_i + 45
+        begin
+          dynamodb.put_item({
+            table_name: DYNAMODB_TABLE_NAME,
+            item: {
+              'lock_key' => "52",
+              'lock_body' => lock_body,
+              'ttl' => expiration_time
+            },
+            condition_expression: "attribute_not_exists(lock_key)"
+          })
+          return
+        rescue Aws::DynamoDB::Errors::ConditionalCheckFailedException
+          lock_holder = current_lock_holder(codename)
+          current_time = Time.now.strftime("%Y-%m-%d %H:%M:%S")
+          $stderr.puts("[#{current_time}] Repository is locked by another user: #{lock_holder.user} at host #{lock_holder.host_with_uuid}")
+          $stderr.puts("Attempting to obtain a lock after #{wait_interval} second(s).")
+          sleep(wait_interval)
+        end
+      end
+
+      raise "Unable to obtain a lock after #{max_attempts} attemtps, giving up."
+    end
+
+    def unlock(codename)
+      # dynamodb.delete_item({
+      #   table_name: DYNAMODB_TABLE_NAME,
+      #   key: {
+      #     'lock_key' => codename
+      #   }
+      # })
+    end
+
+    def current_lock_holder(codename)
+      response = dynamodb.get_item({
+        table_name: DYNAMODB_TABLE_NAME,
+        key: {
+          'lock_key' => codename
+        }
+      })
+
+      if response.item
+        lockdata = response.item['lock_body']
+        user, host_with_uuid = lockdata.split("@", 2)
+        Deb::S3::Lock.new(user, host_with_uuid)
+      else
+        Deb::S3::Lock.new("unknown", "unknown")
+      end
+    end
+
+    private :dynamodb, :validate_environment_variables!
+  end
+end

data/lib/deb/s3/manifest.rb

@@ -0,0 +1,144 @@
+# -*- encoding : utf-8 -*-
+require "tempfile"
+require "zlib"
+require 'deb/s3/utils'
+require 'deb/s3/package'
+
+class Deb::S3::Manifest
+  include Deb::S3::Utils
+
+  attr_accessor :codename
+  attr_accessor :component
+  attr_accessor :cache_control
+  attr_accessor :architecture
+  attr_accessor :fail_if_exists
+  attr_accessor :skip_package_upload
+
+  attr_accessor :files
+
+  attr_reader :packages
+  attr_reader :packages_to_be_upload
+
+  def initialize
+    @packages = []
+    @packages_to_be_upload = []
+    @component = nil
+    @architecture = nil
+    @files = {}
+    @cache_control = ""
+    @fail_if_exists = false
+    @skip_package_upload = false
+  end
+
+  class << self
+    def retrieve(codename, component, architecture, cache_control, fail_if_exists, skip_package_upload=false)
+      m = if s = Deb::S3::Utils.s3_read("dists/#{codename}/#{component}/binary-#{architecture}/Packages")
+        self.parse_packages(s)
+      else
+        self.new
+      end
+
+      m.codename = codename
+      m.component = component
+      m.architecture = architecture
+      m.cache_control = cache_control
+      m.fail_if_exists = fail_if_exists
+      m.skip_package_upload = skip_package_upload
+      m
+    end
+
+    def parse_packages(str)
+      m = self.new
+      str.split("\n\n").each do |s|
+        next if s.chomp.empty?
+        m.packages << Deb::S3::Package.parse_string(s)
+      end
+      m
+    end
+  end
+
+  def add(pkg, preserve_versions, needs_uploading=true)
+    if self.fail_if_exists
+      packages.each { |p|
+        next unless p.name == pkg.name && \
+                    p.full_version == pkg.full_version && \
+                    File.basename(p.url_filename(@codename)) != \
+                    File.basename(pkg.url_filename(@codename))
+        raise AlreadyExistsError,
+              "package #{pkg.name}_#{pkg.full_version} already exists " \
+              "with different filename (#{p.url_filename(@codename)})"
+      }
+    end
+    if preserve_versions
+      packages.delete_if { |p| p.name == pkg.name && p.full_version == pkg.full_version }
+    else
+      packages.delete_if { |p| p.name == pkg.name }
+    end
+    packages << pkg
+    packages_to_be_upload << pkg if needs_uploading
+    pkg
+  end
+
+  def delete_package(pkg, versions=nil)
+    deleted = []
+    new_packages = @packages.select { |p|
+        # Include packages we didn't name
+        if p.name != pkg
+           p
+        # Also include the packages not matching a specified version
+        elsif (!versions.nil? and p.name == pkg and !versions.include?(p.version) and !versions.include?("#{p.version}-#{p.iteration}") and !versions.include?(p.full_version))
+            p
+        end
+    }
+    deleted = @packages - new_packages
+    @packages = new_packages
+    deleted
+  end
+
+  def generate
+    @packages.collect { |pkg| pkg.generate(@codename) }.join("\n")
+  end
+
+  def write_to_s3
+    manifest = self.generate
+
+    unless self.skip_package_upload
+      # store any packages that need to be stored
+      @packages_to_be_upload.each do |pkg|
+        yield pkg.url_filename(@codename) if block_given?
+        s3_store(pkg.filename, pkg.url_filename(@codename), 'application/octet-stream; charset=binary', self.cache_control, self.fail_if_exists)
+      end
+    end
+
+    # generate the Packages file
+    pkgs_temp = Tempfile.new("Packages")
+    pkgs_temp.write manifest
+    pkgs_temp.close
+    f = "dists/#{@codename}/#{@component}/binary-#{@architecture}/Packages"
+    yield f if block_given?
+    s3_store(pkgs_temp.path, f, 'text/plain; charset=utf-8', self.cache_control)
+    @files["#{@component}/binary-#{@architecture}/Packages"] = hashfile(pkgs_temp.path)
+    pkgs_temp.unlink
+
+    # generate the Packages.gz file
+    gztemp = Tempfile.new("Packages.gz")
+    gztemp.close
+    Zlib::GzipWriter.open(gztemp.path) { |gz| gz.write manifest }
+    f = "dists/#{@codename}/#{@component}/binary-#{@architecture}/Packages.gz"
+    yield f if block_given?
+    s3_store(gztemp.path, f, 'application/x-gzip; charset=binary', self.cache_control)
+    @files["#{@component}/binary-#{@architecture}/Packages.gz"] = hashfile(gztemp.path)
+    gztemp.unlink
+
+    nil
+  end
+
+  def hashfile(path)
+    {
+      :size   => File.size(path),
+      :sha1   => Digest::SHA1.file(path).hexdigest,
+      :sha256 => Digest::SHA2.file(path).hexdigest,
+      :md5    => Digest::MD5.file(path).hexdigest
+    }
+  end
+end

data/lib/deb/s3/package.rb

@@ -0,0 +1,309 @@
+# -*- encoding : utf-8 -*-
+require "digest/sha1"
+require "digest/sha2"
+require "digest/md5"
+require "open3"
+require "socket"
+require "tmpdir"
+require "uri"
+
+require 'deb/s3/utils'
+
+class Deb::S3::Package
+  include Deb::S3::Utils
+
+  attr_accessor :name
+  attr_accessor :version
+  attr_accessor :epoch
+  attr_accessor :iteration
+  attr_accessor :maintainer
+  attr_accessor :vendor
+  attr_accessor :url
+  attr_accessor :category
+  attr_accessor :license
+  attr_accessor :architecture
+  attr_accessor :description
+
+  attr_accessor :dependencies
+
+  # Any other attributes specific to this package.
+  # This is where you'd put rpm, deb, or other specific attributes.
+  attr_accessor :attributes
+
+  # hashes
+  attr_accessor :sha1
+  attr_accessor :sha256
+  attr_accessor :md5
+  attr_accessor :size
+
+  attr_accessor :filename
+
+  class << self
+    include Deb::S3::Utils
+
+    def parse_file(package)
+      p = self.new
+      p.extract_info(extract_control(package))
+      p.apply_file_info(package)
+      p.filename = package
+      p
+    end
+
+    def parse_string(s)
+      p = self.new
+      p.extract_info(s)
+      p
+    end
+
+    def extract_control(package)
+      if system("which dpkg > /dev/null 2>&1")
+        output, status = Open3.capture2("dpkg", "-f", package)
+        output
+      else
+	# use ar to determine control file name (control.ext)
+        package_files = `ar t #{package}`
+        control_file = package_files.split("\n").select do |file|
+          file.start_with?("control.")
+        end.first
+        if control_file === "control.tar.gz"
+	  compression = "z"
+        elsif control_file === "control.tar.zst"
+          compression = "I zstd"
+	else
+	  compression = "J"
+        end
+
+        # ar fails to find the control.tar.gz tarball within the .deb
+        # on Mac OS. Try using ar to list the control file, if found,
+        # use ar to extract, otherwise attempt with tar which works on OS X.
+        extract_control_tarball_cmd = "ar p #{package} #{control_file}"
+
+        begin
+          safesystem("ar t #{package} #{control_file} &> /dev/null")
+        rescue SafeSystemError
+          warn "Failed to find control data in .deb with ar, trying tar."
+          extract_control_tarball_cmd = "tar -#{compression} -xf #{package} --to-stdout #{control_file}"
+        end
+
+        Dir.mktmpdir do |path|
+          safesystem("#{extract_control_tarball_cmd} | tar -#{compression} -xf - -C #{path}")
+          File.read(File.join(path, "control"))
+        end
+      end
+    end
+  end
+
+  def initialize
+    @attributes = {}
+
+    # Reference
+    # http://www.debian.org/doc/manuals/maint-guide/first.en.html
+    # http://wiki.debian.org/DeveloperConfiguration
+    # https://github.com/jordansissel/fpm/issues/37
+    if ENV.include?("DEBEMAIL") and ENV.include?("DEBFULLNAME")
+      # Use DEBEMAIL and DEBFULLNAME as the default maintainer if available.
+      @maintainer = "#{ENV["DEBFULLNAME"]} <#{ENV["DEBEMAIL"]}>"
+    else
+      # TODO(sissel): Maybe support using 'git config' for a default as well?
+      # git config --get user.name, etc can be useful.
+      #
+      # Otherwise default to user@currenthost
+      @maintainer = "<#{ENV["USER"]}@#{Socket.gethostname}>"
+    end
+
+    @name = nil
+    @architecture = "native"
+    @description = "no description given"
+    @version = nil
+    @epoch = nil
+    @iteration = nil
+    @url = nil
+    @category = "default"
+    @license = "unknown"
+    @vendor = "none"
+    @sha1 = nil
+    @sha256 = nil
+    @md5 = nil
+    @size = nil
+    @filename = nil
+    @url_filename = nil
+
+    @dependencies = []
+  end
+
+  def full_version
+    return nil if [epoch, version, iteration].all?(&:nil?)
+    [[epoch, version].compact.join(":"), iteration].compact.join("-")
+  end
+
+  def url_filename=(f)
+    @url_filename = f
+  end
+  
+  def url_filename(codename)
+    @url_filename || "pool/#{codename}/#{self.name[0]}/#{self.name[0..1]}/#{File.basename(self.filename)}"
+  end
+
+  def generate(codename)
+    template("package.erb").result(binding)
+  end
+
+  # from fpm
+  def parse_depends(data)
+    return [] if data.nil? or data.empty?
+    # parse dependencies. Debian dependencies come in one of two forms:
+    # * name
+    # * name (op version)
+    # They are all on one line, separated by ", "
+
+    dep_re = /^([^ ]+)(?: \(([>=<]+) ([^)]+)\))?$/
+    return data.split(/, */).collect do |dep|
+      m = dep_re.match(dep)
+      if m
+        name, op, version = m.captures
+        # this is the proper form of dependency
+        if op && version && op != "" && version != ""
+          "#{name} (#{op} #{version})".strip
+        else
+          name.strip
+        end
+      else
+        # Assume normal form dependency, "name op version".
+        dep
+      end
+    end
+  end # def parse_depends
+
+  # from fpm
+  def fix_dependency(dep)
+    # Deb dependencies are: NAME (OP VERSION), like "zsh (> 3.0)"
+    # Convert anything that looks like 'NAME OP VERSION' to this format.
+    if dep =~ /[\(,\|]/
+      # Don't "fix" ones that could appear well formed already.
+    else
+      # Convert ones that appear to be 'name op version'
+      name, op, version = dep.split(/ +/)
+      if !version.nil?
+        # Convert strings 'foo >= bar' to 'foo (>= bar)'
+        dep = "#{name} (#{debianize_op(op)} #{version})"
+      end
+    end
+
+    name_re = /^[^ \(]+/
+    name = dep[name_re]
+    if name =~ /[A-Z]/
+      dep = dep.gsub(name_re) { |n| n.downcase }
+    end
+
+    if dep.include?("_")
+      dep = dep.gsub("_", "-")
+    end
+
+    # Convert gem ~> X.Y.Z to '>= X.Y.Z' and << X.Y+1.0
+    if dep =~ /\(~>/
+      name, version = dep.gsub(/[()~>]/, "").split(/ +/)[0..1]
+      nextversion = version.split(".").collect { |v| v.to_i }
+      l = nextversion.length
+      nextversion[l-2] += 1
+      nextversion[l-1] = 0
+      nextversion = nextversion.join(".")
+      return ["#{name} (>= #{version})", "#{name} (<< #{nextversion})"]
+    elsif (m = dep.match(/(\S+)\s+\(!= (.+)\)/))
+      # Append this to conflicts
+      self.conflicts += [dep.gsub(/!=/,"=")]
+      return []
+    elsif (m = dep.match(/(\S+)\s+\(= (.+)\)/)) and
+        self.attributes[:deb_ignore_iteration_in_dependencies?]
+      # Convert 'foo (= x)' to 'foo (>= x)' and 'foo (<< x+1)'
+      # but only when flag --ignore-iteration-in-dependencies is passed.
+      name, version = m[1..2]
+      nextversion = version.split('.').collect { |v| v.to_i }
+      nextversion[-1] += 1
+      nextversion = nextversion.join(".")
+      return ["#{name} (>= #{version})", "#{name} (<< #{nextversion})"]
+    else
+      # otherwise the dep is probably fine
+      return dep.rstrip
+    end
+  end # def fix_dependency
+
+  # from fpm
+  def extract_info(control)
+    fields = parse_control(control)
+
+    # Parse 'epoch:version-iteration' in the version string
+    full_version = fields.delete('Version')
+    if full_version !~ /^(?:([0-9]+):)?(.+?)(?:-(.*))?$/
+      raise "Unsupported version string '#{full_version}'"
+    end
+    self.epoch, self.version, self.iteration = $~.captures
+
+    self.architecture = fields.delete('Architecture')
+    self.category = fields.delete('Section')
+    self.license = fields.delete('License') || self.license
+    self.maintainer = fields.delete('Maintainer')
+    self.name = fields.delete('Package')
+    self.url = fields.delete('Homepage')
+    self.vendor = fields.delete('Vendor') || self.vendor
+    self.attributes[:deb_priority] = fields.delete('Priority')
+    self.attributes[:deb_origin] = fields.delete('Origin')
+    self.attributes[:deb_installed_size] = fields.delete('Installed-Size')
+
+    # Packages manifest fields
+    self.url_filename = fields.delete('Filename')
+    self.sha1 = fields.delete('SHA1')
+    self.sha256 = fields.delete('SHA256')
+    self.md5 = fields.delete('MD5sum')
+    self.size = fields.delete('Size')
+    self.description = fields.delete('Description')
+
+    #self.config_files = config_files
+
+    self.dependencies += Array(parse_depends(fields.delete('Depends')))
+
+    self.attributes[:deb_recommends] = fields.delete('Recommends')
+    self.attributes[:deb_suggests]   = fields.delete('Suggests')
+    self.attributes[:deb_enhances]   = fields.delete('Enhances')
+    self.attributes[:deb_pre_depends] = fields.delete('Pre-Depends')
+
+    self.attributes[:deb_breaks]    = fields.delete('Breaks')
+    self.attributes[:deb_conflicts] = fields.delete('Conflicts')
+    self.attributes[:deb_provides]  = fields.delete('Provides')
+    self.attributes[:deb_replaces]  = fields.delete('Replaces')
+
+    self.attributes[:deb_field] = Hash[fields.map { |k, v|
+      [k.sub(/\AX[BCS]{0,3}-/, ''), v]
+    }]
+  end # def extract_info
+
+  def apply_file_info(file)
+    self.size = File.size(file)
+    self.sha1 = Digest::SHA1.file(file).hexdigest
+    self.sha256 = Digest::SHA2.file(file).hexdigest
+    self.md5 = Digest::MD5.file(file).hexdigest
+  end
+
+  def parse_control(control)
+    field = nil
+    value = ""
+    {}.tap do |fields|
+      control.each_line do |line|
+        if line =~ /^(\s+)(\S.*)$/
+          indent, rest = $1, $2
+          # Continuation
+          if indent.size == 1 && rest == "."
+            value << "\n"
+            rest = ""
+          elsif value.size > 0
+            value << "\n"
+          end
+          value << rest
+        elsif line =~ /^([-\w]+):(.*)$/
+          fields[field] = value if field
+          field, value = $1, $2.strip
+        end
+      end
+      fields[field] = value if field
+    end
+  end
+end

data/lib/deb/s3/release.rb

@@ -0,0 +1,161 @@
+# -*- encoding : utf-8 -*-
+require "tempfile"
+
+class Deb::S3::Release
+  include Deb::S3::Utils
+
+  attr_accessor :codename
+  attr_accessor :origin
+  attr_accessor :suite
+  attr_accessor :architectures
+  attr_accessor :components
+  attr_accessor :cache_control
+
+  attr_accessor :files
+  attr_accessor :policy
+
+  def initialize
+    @origin = nil
+    @suite = nil
+    @codename = nil
+    @architectures = []
+    @components = []
+    @cache_control = ""
+    @files = {}
+    @policy = :public_read
+  end
+
+  class << self
+    def retrieve(codename, origin=nil, suite=nil, cache_control=nil)
+      if s = Deb::S3::Utils.s3_read("dists/#{codename}/Release")
+        rel = self.parse_release(s)
+      else
+        rel = self.new
+      end
+      rel.codename = codename
+      rel.origin = origin unless origin.nil?
+      rel.suite = suite unless suite.nil?
+      rel.cache_control = cache_control
+      rel
+    end
+
+    def parse_release(str)
+      rel = self.new
+      rel.parse(str)
+      rel
+    end
+  end
+
+  def filename
+    "dists/#{@codename}/Release"
+  end
+
+  def parse(str)
+    parse = lambda do |field|
+      value = str[/^#{field}: .*/]
+      if value.nil?
+        return nil
+      else
+        return value.split(": ",2).last
+      end
+    end
+
+    # grab basic fields
+    self.codename = parse.call("Codename")
+    self.origin = parse.call("Origin") || nil
+    self.suite = parse.call("Suite") || nil
+    self.architectures = (parse.call("Architectures") || "").split(/\s+/)
+    self.components = (parse.call("Components") || "").split(/\s+/)
+
+    # find all the hashes
+    str.scan(/^\s+([^\s]+)\s+(\d+)\s+(.+)$/).each do |(hash,size,name)|
+      self.files[name] ||= { :size => size.to_i }
+      case hash.length
+      when 32
+        self.files[name][:md5] = hash
+      when 40
+        self.files[name][:sha1] = hash
+      when 64
+        self.files[name][:sha256] = hash
+      end
+    end
+  end
+
+  def generate
+    template("release.erb").result(binding)
+  end
+
+  def write_to_s3
+    # validate some other files are present
+    if block_given?
+      self.validate_others { |f| yield f }
+    else
+      self.validate_others
+    end
+
+    # generate the Release file
+    release_tmp = Tempfile.new("Release")
+    release_tmp.puts self.generate
+    release_tmp.close
+    yield self.filename if block_given?
+    s3_store(release_tmp.path, self.filename, 'text/plain; charset=utf-8', self.cache_control)
+
+    # sign the file, if necessary
+    if Deb::S3::Utils.signing_key
+      key_param = Deb::S3::Utils.signing_key.any? ? "-u #{Deb::S3::Utils.signing_key.join(" -u ")}" : ""
+      if system("gpg -a #{key_param} --digest-algo SHA256 #{Deb::S3::Utils.gpg_options} -s --clearsign #{release_tmp.path}")
+        local_file = release_tmp.path+".asc"
+        remote_file = "dists/#{@codename}/InRelease"
+        yield remote_file if block_given?
+        raise "Unable to locate InRelease file" unless File.exist?(local_file)
+        s3_store(local_file, remote_file, 'application/pgp-signature; charset=us-ascii', self.cache_control)
+        File.unlink(local_file)
+      else
+        raise "Signing the InRelease file failed."
+      end
+      if system("gpg -a #{key_param} --digest-algo SHA256 #{Deb::S3::Utils.gpg_options} -b #{release_tmp.path}")
+        local_file = release_tmp.path+".asc"
+        remote_file = self.filename+".gpg"
+        yield remote_file if block_given?
+        raise "Unable to locate Release signature file" unless File.exist?(local_file)
+        s3_store(local_file, remote_file, 'application/pgp-signature; charset=us-ascii', self.cache_control)
+        File.unlink(local_file)
+      else
+        raise "Signing the Release file failed."
+      end
+    else
+      # remove an existing Release.gpg, if it was there
+      s3_remove(self.filename+".gpg")
+    end
+
+    release_tmp.unlink
+  end
+
+  def update_manifest(manifest)
+    self.components << manifest.component unless self.components.include?(manifest.component)
+    self.architectures << manifest.architecture unless self.architectures.include?(manifest.architecture)
+    self.files.merge!(manifest.files)
+  end
+
+  def validate_others
+    to_apply = []
+    self.components.each do |comp|
+      %w(amd64 i386 armhf).each do |arch|
+        next if self.files.has_key?("#{comp}/binary-#{arch}/Packages")
+
+        m = Deb::S3::Manifest.new
+        m.codename = self.codename
+        m.component = comp
+        m.architecture = arch
+        if block_given?
+          m.write_to_s3 { |f| yield f }
+        else
+          m.write_to_s3
+        end
+        to_apply << m
+      end
+    end
+
+    to_apply.each { |m| self.update_manifest(m) }
+  end
+end