dpkg-s3 0.1.0

data/lib/dpkg/s3/release.rb

@@ -0,0 +1,161 @@
+# -*- encoding : utf-8 -*-
+require "tempfile"
+
+class Dpkg::S3::Release
+  include Dpkg::S3::Utils
+
+  attr_accessor :codename
+  attr_accessor :origin
+  attr_accessor :suite
+  attr_accessor :architectures
+  attr_accessor :components
+  attr_accessor :cache_control
+
+  attr_accessor :files
+  attr_accessor :policy
+
+  def initialize
+    @origin = nil
+    @suite = nil
+    @codename = nil
+    @architectures = []
+    @components = []
+    @cache_control = ""
+    @files = {}
+    @policy = :public_read
+  end
+
+  class << self
+    def retrieve(codename, origin=nil, suite=nil, cache_control=nil)
+      if s = Dpkg::S3::Utils.s3_read("dists/#{codename}/Release")
+        rel = self.parse_release(s)
+      else
+        rel = self.new
+      end
+      rel.codename = codename
+      rel.origin = origin unless origin.nil?
+      rel.suite = suite unless suite.nil?
+      rel.cache_control = cache_control
+      rel
+    end
+
+    def parse_release(str)
+      rel = self.new
+      rel.parse(str)
+      rel
+    end
+  end
+
+  def filename
+    "dists/#{@codename}/Release"
+  end
+
+  def parse(str)
+    parse = lambda do |field|
+      value = str[/^#{field}: .*/]
+      if value.nil?
+        return nil
+      else
+        return value.split(": ",2).last
+      end
+    end
+
+    # grab basic fields
+    self.codename = parse.call("Codename")
+    self.origin = parse.call("Origin") || nil
+    self.suite = parse.call("Suite") || nil
+    self.architectures = (parse.call("Architectures") || "").split(/\s+/)
+    self.components = (parse.call("Components") || "").split(/\s+/)
+
+    # find all the hashes
+    str.scan(/^\s+([^\s]+)\s+(\d+)\s+(.+)$/).each do |(hash,size,name)|
+      self.files[name] ||= { :size => size.to_i }
+      case hash.length
+      when 32
+        self.files[name][:md5] = hash
+      when 40
+        self.files[name][:sha1] = hash
+      when 64
+        self.files[name][:sha256] = hash
+      end
+    end
+  end
+
+  def generate
+    template("release.erb").result(binding)
+  end
+
+  def write_to_s3
+    # validate some other files are present
+    if block_given?
+      self.validate_others { |f| yield f }
+    else
+      self.validate_others
+    end
+
+    # generate the Release file
+    release_tmp = Tempfile.new("Release")
+    release_tmp.puts self.generate
+    release_tmp.close
+    yield self.filename if block_given?
+    s3_store(release_tmp.path, self.filename, 'text/plain; charset=utf-8', self.cache_control)
+
+    # sign the file, if necessary
+    if Dpkg::S3::Utils.signing_key
+      key_param = Dpkg::S3::Utils.signing_key != "" ? "--default-key=#{Dpkg::S3::Utils.signing_key}" : ""
+      if system("gpg -a #{key_param} --digest-algo SHA256 #{Dpkg::S3::Utils.gpg_options} -s --clearsign #{release_tmp.path}")
+        local_file = release_tmp.path+".asc"
+        remote_file = "dists/#{@codename}/InRelease"
+        yield remote_file if block_given?
+        raise "Unable to locate InRelease file" unless File.exists?(local_file)
+        s3_store(local_file, remote_file, 'application/pgp-signature; charset=us-ascii', self.cache_control)
+        File.unlink(local_file)
+      else
+        raise "Signing the InRelease file failed."
+      end
+      if system("gpg -a #{key_param} --digest-algo SHA256 #{Dpkg::S3::Utils.gpg_options} -b #{release_tmp.path}")
+        local_file = release_tmp.path+".asc"
+        remote_file = self.filename+".gpg"
+        yield remote_file if block_given?
+        raise "Unable to locate Release signature file" unless File.exists?(local_file)
+        s3_store(local_file, remote_file, 'application/pgp-signature; charset=us-ascii', self.cache_control)
+        File.unlink(local_file)
+      else
+        raise "Signing the Release file failed."
+      end
+    else
+      # remove an existing Release.gpg, if it was there
+      s3_remove(self.filename+".gpg")
+    end
+
+    release_tmp.unlink
+  end
+
+  def update_manifest(manifest)
+    self.components << manifest.component unless self.components.include?(manifest.component)
+    self.architectures << manifest.architecture unless self.architectures.include?(manifest.architecture)
+    self.files.merge!(manifest.files)
+  end
+
+  def validate_others
+    to_apply = []
+    self.components.each do |comp|
+      %w(amd64 i386 armhf).each do |arch|
+        next if self.files.has_key?("#{comp}/binary-#{arch}/Packages")
+
+        m = Dpkg::S3::Manifest.new
+        m.codename = self.codename
+        m.component = comp
+        m.architecture = arch
+        if block_given?
+          m.write_to_s3 { |f| yield f }
+        else
+          m.write_to_s3
+        end
+        to_apply << m
+      end
+    end
+
+    to_apply.each { |m| self.update_manifest(m) }
+  end
+end

data/lib/dpkg/s3/templates/package.erb

@@ -0,0 +1,66 @@
+Package: <%= name %>
+Version: <%= "#{epoch}:" if epoch %><%= version %><%= "-" + iteration.to_s if iteration %>
+License: <%= license %>
+Vendor: <%= vendor %>
+Architecture: <%= architecture %>
+Maintainer: <%= maintainer  %>
+Installed-Size: <%= attributes[:deb_installed_size] %>
+<% if !dependencies.empty? and !attributes[:no_depends?] -%>
+Depends: <%= dependencies.collect { |d| fix_dependency(d) }.flatten.join(", ") %>
+<% end -%>
+<% if attributes[:deb_conflicts] -%>
+Conflicts: <%= attributes[:deb_conflicts] %>
+<% end -%>
+<% if attributes[:deb_breaks] -%>
+Breaks: <%= attributes[:deb_breaks] %>
+<% end -%>
+<% if attributes[:deb_pre_depends] -%>
+Pre-Depends: <%= attributes[:deb_pre_depends] %>
+<% end -%>
+<% if attributes[:deb_provides] -%>
+<%# Turn each provides from 'foo = 123' to simply 'foo' because Debian :\ -%>
+<%#  http://www.debian.org/doc/debian-policy/ch-relationships.html -%>
+Provides: <%= attributes[:deb_provides] %>
+<% end -%>
+<% if attributes[:deb_replaces] -%>
+Replaces: <%= attributes[:deb_replaces] %>
+<% end -%>
+<% if attributes[:deb_recommends] -%>
+Recommends: <%= attributes[:deb_recommends] %>
+<% end -%>
+<% if attributes[:deb_suggests] -%>
+Suggests: <%= attributes[:deb_suggests] %>
+<% end -%>
+<% if attributes[:deb_enhances] -%>
+Enhances: <%= attributes[:deb_enhances] %>
+<% end -%>
+Section: <%= category %>
+<% if attributes[:deb_origin] -%>
+Origin: <%= attributes[:deb_origin] %>
+<% end -%>
+Priority: <%= attributes[:deb_priority] %>
+Homepage: <%= url or "http://nourlgiven.example.com/" %>
+Filename: <%= url_filename_encoded(codename) %>
+<% if size -%>
+Size: <%= size %>
+<% end -%>
+<% if sha1 -%>
+SHA1: <%= sha1 %>
+<% end -%>
+<% if sha256 -%>
+SHA256: <%= sha256 %>
+<% end -%>
+<% if md5 -%>
+MD5sum: <%= md5 %>
+<% end -%>
+<% lines = (description or "no description given").split("\n") -%>
+<% firstline, *remainder = lines -%>
+Description: <%= firstline %>
+<% if remainder.any? -%>
+<%= remainder.collect { |l| l =~ /^ *$/ ? " ." : " #{l}" }.join("\n") %>
+<% end -%>
+<% if attributes[:deb_field] -%>
+<%   attributes[:deb_field].each do |field, value| -%>
+<%= field %>: <%= value %>
+<%   end -%>
+<% end -%>

data/lib/dpkg/s3/templates/release.erb

@@ -0,0 +1,20 @@
+<% if origin -%>
+Origin: <%= origin %>
+<% end -%>
+Codename: <%= codename %>
+Date: <%= Time.now.utc.strftime("%a, %d %b %Y %T %Z") %>
+Architectures: <%= architectures.join(" ") %>
+Components: <%= components.join(" ") %>
+Suite: <%= suite %>
+MD5Sum:
+<% files.sort.each do |f,p| -%>
+ <%= p[:md5] %> <%= p[:size].to_s.rjust(16) %> <%= f %>
+<% end -%>
+SHA1:
+<% files.sort.each do |f,p| -%>
+ <%= p[:sha1] %> <%= p[:size].to_s.rjust(16) %> <%= f %>
+<% end -%>
+SHA256:
+<% files.sort.each do |f,p| -%>
+ <%= p[:sha256] %> <%= p[:size].to_s.rjust(16) %> <%= f %>
+<% end -%>

data/lib/dpkg/s3/utils.rb

@@ -0,0 +1,117 @@
+# -*- encoding : utf-8 -*-
+require "base64"
+require "digest/md5"
+require "erb"
+require "tmpdir"
+
+module Dpkg::S3::Utils
+  module_function
+  def s3; @s3 end
+  def s3= v; @s3 = v end
+  def bucket; @bucket end
+  def bucket= v; @bucket = v end
+  def access_policy; @access_policy end
+  def access_policy= v; @access_policy = v end
+  def signing_key; @signing_key end
+  def signing_key= v; @signing_key = v end
+  def gpg_options; @gpg_options end
+  def gpg_options= v; @gpg_options = v end
+  def prefix; @prefix end
+  def prefix= v; @prefix = v end
+  def encryption; @encryption end
+  def encryption= v; @encryption = v end
+
+  class SafeSystemError < RuntimeError; end
+  class AlreadyExistsError < RuntimeError; end
+
+  def safesystem(*args)
+    success = system(*args)
+    if !success
+      raise SafeSystemError, "'system(#{args.inspect})' failed with error code: #{$?.exitstatus}"
+    end
+    return success
+  end
+
+  def debianize_op(op)
+    # Operators in debian packaging are <<, <=, =, >= and >>
+    # So any operator like < or > must be replaced
+    {:< => "<<", :> => ">>"}[op.to_sym] or op
+  end
+
+  def template(path)
+    template_file = File.join(File.dirname(__FILE__), "templates", path)
+    template_code = File.read(template_file)
+    ERB.new(template_code, nil, "-")
+  end
+
+  def s3_path(path)
+    File.join(*[Dpkg::S3::Utils.prefix, path].compact)
+  end
+
+  # from fog, Fog::AWS.escape
+  def s3_escape(string)
+    string.gsub(/([^a-zA-Z0-9_.\-~+]+)/) {
+      "%" + $1.unpack("H2" * $1.bytesize).join("%").upcase
+    }
+  end
+
+  def s3_exists?(path)
+    Dpkg::S3::Utils.s3.head_object(
+      :bucket => Dpkg::S3::Utils.bucket,
+      :key => s3_path(path),
+    )
+  rescue Aws::S3::Errors::NotFound
+    false
+  end
+
+  def s3_read(path)
+    Dpkg::S3::Utils.s3.get_object(
+      :bucket => Dpkg::S3::Utils.bucket,
+      :key => s3_path(path),
+    )[:body].read
+  rescue Aws::S3::Errors::NoSuchKey
+    false
+  end
+
+  def s3_store(path, filename=nil, content_type='application/octet-stream; charset=binary', cache_control=nil, fail_if_exists=false)
+    filename = File.basename(path) unless filename
+    obj = s3_exists?(filename)
+
+    file_md5 = Digest::MD5.file(path)
+
+    # check if the object already exists
+    if obj != false
+      return if (file_md5.to_s == obj[:etag].gsub('"', '') or file_md5.to_s == obj[:metadata]['md5'])
+      raise AlreadyExistsError, "file #{filename} already exists with different contents" if fail_if_exists
+    end
+
+    options = {
+      :bucket => Dpkg::S3::Utils.bucket,
+      :key => s3_path(filename),
+      :acl => Dpkg::S3::Utils.access_policy,
+      :content_type => content_type,
+      :metadata => { "md5" => file_md5.to_s },
+    }
+    if !cache_control.nil?
+      options[:cache_control] = cache_control
+    end
+
+    # specify if encryption is required
+    options[:server_side_encryption] = "AES256" if Dpkg::S3::Utils.encryption
+
+    # upload the file
+    File.open(path) do |f|
+      options[:body] = f
+      Dpkg::S3::Utils.s3.put_object(options)
+    end
+  end
+
+  def s3_remove(path)
+    if s3_exists?(path)
+      Dpkg::S3::Utils.s3.delete_object(
+        :bucket =>Dpkg::S3::Utils.bucket,
+        :key => s3_path(path),
+      )
+    end
+  end
+end

metadata

@@ -0,0 +1,110 @@
+--- !ruby/object:Gem::Specification
+name: dpkg-s3
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Gamunu Balagalla
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-05-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.19.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.19.0
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '11'
+description: Easily create and manage an APT repository on S3.
+email: gamunu.balagalla@outlook.com
+executables:
+- dpkg-s3
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- bin/dpkg-s3
+- lib/dpkg/s3.rb
+- lib/dpkg/s3/cli.rb
+- lib/dpkg/s3/lock.rb
+- lib/dpkg/s3/manifest.rb
+- lib/dpkg/s3/package.rb
+- lib/dpkg/s3/release.rb
+- lib/dpkg/s3/templates/package.erb
+- lib/dpkg/s3/templates/release.erb
+- lib/dpkg/s3/utils.rb
+homepage: https://github.com/gamunu/dpkg-s3
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 1.9.3
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: Easily create and manage an APT repository on S3.
+test_files: []