reaper-man 0.0.1 → 0.1.0

data/lib/reaper-man/package_list/deb.rb

@@ -0,0 +1,150 @@
+require 'reaper-man'
+
+module ReaperMan
+  class PackageList
+    class Processor
+      # Package list process for debian packages
+      class Deb < Processor
+
+        # @return [String]
+        attr_reader :origin
+        # @return [String]
+        attr_reader :dist
+        # @return [String]
+        attr_reader :component
+        # @return [Array<String>] architectures
+        attr_reader :all_map
+        # @return [String] prefix for package file location
+        attr_reader :package_root
+        # @return [String] namespace for packages
+        attr_reader :package_bucket
+
+        # default package root prefix
+        DEFAULT_ROOT = 'pool'
+        # default namespace for packages
+        DEFAULT_BUCKET = 'public'
+        # default architectures to define
+        DEFAULT_ALL_MAP = ['amd64', 'i386']
+
+        # Create new instance
+        #
+        # @param args [Hash]
+        # @option args [String] :origin
+        # @option args [String] :codename
+        # @option args [String] :component
+        # @option args [String] :package_root
+        # @option args [String] :package_bucket
+        # @option args [Array<String>] :all_map
+        def initialize(args={})
+          @origin = args[:origin].to_s
+          @dist = args[:codename].to_s
+          @component = args[:component].to_s
+          @package_root = args.fetch(:package_root, DEFAULT_ROOT)
+          @package_bucket = args.fetch(:package_bucket, DEFAULT_BUCKET)
+          if(dist.empty? || component.empty?)
+            raise 'Both `codename` and `component` must contain valid values'
+          end
+          @all_map = args.fetch(:all_map, DEFAULT_ALL_MAP)
+        end
+
+        # Add a package to the list
+        #
+        # @param conf [Hash]
+        # @param package [String] path to package
+        def add(hash, package)
+          info = extract_fields(package)
+          info.merge!(generate_checksums(package))
+          filenames = inject_package(hash, info, package)
+          filenames
+        end
+
+        # Remove package from the list
+        #
+        # @param conf [Hash] configuration hash
+        # @param package_name [String] name
+        # @param version [String]
+        def remove(hash, package_name, version, args={})
+          hash = hash.to_smash
+          arch = [args.fetch(:arch, all_map)].flatten.compact
+          deleted = false
+          arch.each do |arch_name|
+            arch_name = "binary-#{arch_name}"
+            if(hash.get(:apt, origin, dist, :components, component, arch_name, package_name))
+              if(version)
+                deleted = hash[:apt][origin][dist][:components][component][arch_name][package_name].delete(version)
+              else
+                deleted = hash[:apt][origin][dist][:components][component][arch_name].delete(package_name)
+              end
+            end
+          end
+          !!deleted
+        end
+
+        # Extract package metadata
+        #
+        # @param package [String] path to package
+        # @return [Hash]
+        def extract_fields(package)
+          content = shellout("dpkg-deb -f '#{package}'")
+          Smash[content.stdout.scan(/([^\s][^:]+):\s+(([^\n]|\n\s)+)/).map{|a| a.slice(0,2)}]
+        end
+
+        # Insert package information into package list
+        #
+        # @param hash [Hash] package list contents
+        # @param info [Hash] package information
+        # @param package [String] path to package file
+        # @return [Array<String>] package paths within package list contents
+        def inject_package(hash, info, package)
+          arch = info['Architecture']
+          arch = arch == 'all' ? all_map : [arch]
+          arch.map do |arch|
+            package_file_name = File.join(
+              package_root, package_bucket, origin,
+              dist, component, "binary-#{arch}",
+              File.basename(package)
+            )
+            hash.deep_merge!(
+              'apt' => {
+                origin => {
+                  dist => {
+                    'components' => {
+                      component => {
+                        "binary-#{arch}" => {
+                          info['Package'] => {
+                            info['Version'] => info.merge!(
+                              'Filename' => package_file_name,
+                              'Size' => File.size(package)
+                            )
+                          }
+                        },
+                        "binary-i386" => {
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            )
+            package_file_name
+          end
+        end
+
+        # Generate required checksums for given package
+        #
+        # @param package [String] path to package file
+        # @return [Hash] checksums
+        def generate_checksums(package)
+          File.open(package, 'r') do |pkg|
+            {
+              'MD5sum' => checksum(pkg.rewind && pkg, :md5),
+              'SHA1' => checksum(pkg.rewind && pkg, :sha1),
+              'SHA256' => checksum(pkg.rewind && pkg, :sha256)
+            }
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/reaper-man/package_list/gem.rb

@@ -0,0 +1,80 @@
+require 'reaper-man'
+require 'rubygems/package'
+
+module ReaperMan
+  class PackageList
+    class Processor
+      class Gem < Processor
+
+        # Add a package to the list
+        #
+        # @param conf [Hash]
+        # @param package [String] path to package
+        def add(hash, package)
+          info = extract_fields(package)
+          filenames = inject_package(hash, info, package)
+          filenames
+        end
+
+        # Remove package from the list
+        #
+        # @param conf [Hash] configuration hash
+        # @param package_name [String] name
+        # @param version [String]
+        def remove(hash, package_name, version, args={})
+          deleted = false
+          if(hash['rubygems'][package_name])
+            if(version)
+              deleted = hash['rubygems'][package_name].delete(version)
+            else
+              deleted = hash['rubygems'].delete(package_name)
+            end
+          end
+          !!deleted
+        end
+
+        # Extract package metadata
+        #
+        # @param package [String] path to package
+        # @return [Hash]
+        def extract_fields(package)
+          spec = ::Gem::Package.open(File.open(package)){|pack| pack.metadata}
+          fields = Smash[
+            spec.to_yaml_properties.map do |var_name|
+              [var_name.to_s.tr('@', ''), spec.instance_variable_get(var_name)]
+            end
+          ]
+          fields['dependencies'] = fields['dependencies'].map do |dep|
+            [dep.name, dep.requirement.to_s]
+          end
+          fields
+        end
+
+        # Insert package information into package list
+        #
+        # @param hash [Hash] package list contents
+        # @param info [Hash] package information
+        # @param package [String] path to package file
+        # @return [Array<String>] package paths within package list contents
+        def inject_package(hash, info, package)
+          package_path = File.join(
+            'rubygems', 'gems', "#{info['name']}-#{info['version']}.gem"
+          )
+          classification = info['version'].prerelease? ? 'prerelease' : 'release'
+          info['version'] = info['version'].version
+          hash.deep_merge!(
+            'rubygem' => {
+              classification => {
+                info['name'] => {
+                  info['version'].to_s => info.merge('package_path' => package_path)
+                }
+              }
+            }
+          )
+          package_path
+        end
+
+      end
+    end
+  end
+end

data/lib/reaper-man/package_list/rpm.rb

@@ -0,0 +1,11 @@
+require 'reaper-man'
+
+module ReaperMan
+  class PackageList
+    class Processor
+      class Rpm < Processor
+
+      end
+    end
+  end
+end

data/lib/reaper-man/package_list.rb

@@ -0,0 +1,127 @@
+require 'multi_json'
+require 'reaper-man'
+
+module ReaperMan
+  # Package list for repository
+  class PackageList
+    # Package list modification processor
+    class Processor
+      autoload :Rpm, 'reaper-man/package_list/rpm'
+      autoload :Deb, 'reaper-man/package_list/deb'
+      autoload :Gem, 'reaper-man/package_list/gem'
+
+      include Utils::Process
+      include Utils::Checksum
+
+      # Add a package to the list
+      #
+      # @param conf [Hash]
+      # @param package [String] path to package
+      def add(conf, package)
+        raise NoMethodError.new 'Not implemented'
+      end
+
+      # Remove package from the list
+      #
+      # @param conf [Hash] configuration hash
+      # @param package_name [String] name
+      # @param version [String]
+      def remove(conf, package_name, version=nil)
+        raise NoMethodError.new 'Not implemented'
+      end
+    end
+
+    # @return [String] path to list file
+    attr_reader :path
+    # @return [Hash] configuration
+    attr_reader :options
+    # @return [Time] package list mtime
+    attr_reader :init_mtime
+    # @return [Hash] content of package list
+    attr_reader :content
+
+    # Create new instance
+    #
+    # @param path [String] path to package list
+    # @param args [Hash] configuration
+    def initialize(path, args={})
+      @path = path
+      @options = args.dup
+      @content = Smash.new
+      init_list!
+    end
+
+    # Add package to package list file
+    #
+    # @param package [Array<String>] path to package file
+    def add_package(package)
+      [package_handler(File.extname(package).tr('.', '')).add(content, package)].flatten.compact
+    end
+
+    # Remove package from the package list file
+    #
+    # @param package [String] name of package
+    # @param version [String] version of file
+    def remove_package(package, version=nil)
+      ext = File.extname(package).tr('.', '')
+      if(ext.empty?)
+        ext = %w(deb) # rpm)
+      else
+        ext = [ext]
+      end
+      ext.each do |ext_name|
+        package_handler(ext_name).remove(content, package, version)
+      end
+    end
+
+    # @return [String] serialized content
+    def serialize
+      MultiJson.dump(content)
+    end
+
+    # Write contents to package list file
+    #
+    # @return [Integer] number of bytes written
+    def write!
+      new_file = !File.exists?(path)
+      File.open(path, File::CREAT|File::RDWR) do |file|
+        file.flock(File::LOCK_EX)
+        if(!new_file && init_mtime != file.mtime)
+          file.rewind
+          content.deep_merge!(
+            MultiJson.load(
+              file.read
+            )
+          )
+          file.rewind
+        end
+        pos = file.write MultiJson.dump(content, :pretty => true)
+        file.truncate(pos)
+      end
+    end
+
+    private
+
+    # @return [Processor] processor for give package type
+    def package_handler(pkg_ext)
+      Processor.const_get(pkg_ext.capitalize).new(options)
+    end
+
+    # Initialize the package list file
+    #
+    # @return [Hash] loaded file contents
+    def init_list!
+      write! unless File.exist?(path)
+      @init_mtime = File.mtime(path)
+      content.deep_merge!(
+        MultiJson.load(
+          File.open(path, 'r') do |file|
+            file.flock(File::LOCK_SH)
+            file.read
+          end
+        )
+      )
+    end
+
+  end
+end

data/lib/reaper-man/signer/deb.rb

@@ -0,0 +1,47 @@
+require 'reaper-man'
+
+module ReaperMan
+  class Signer
+    # Signing methods for deb files
+    module Deb
+
+      # command to use for file signing
+      SIGN_COMMAND = File.join(
+        File.expand_path(File.join(File.dirname(__FILE__), '..')),
+        'util-scripts/auto-debsigs'
+      )
+
+      # Sign given files
+      #
+      # @param pkgs [String] list of file paths
+      # @return [TrueClass]
+      def package(*pkgs)
+        pkgs = valid_packages(*pkgs)
+        pkgs.each_slice(sign_chunk_size) do |pkgs|
+          if(key_password)
+            shellout(
+              "#{SIGN_COMMAND} #{sign_type} #{key_id} #{pkgs.join(' ')}",
+              :environment => {
+                'REAPER_KEY_PASSWORD' => key_password
+              }
+            )
+          else
+            shellout(%w(debsigs --sign="#{sign_type}" --default-key="#{key_id}" #{pkgs.join(' ')}))
+          end
+        end
+        true
+      end
+
+      # Filter only valid paths for signing (.deb extensions)
+      #
+      # @param pkgs [String] list of file paths
+      # @return [Array<String>]
+      def valid_packages(*pkgs)
+        pkgs.find_all do |pkg|
+          File.extname(pkg) == '.deb'
+        end
+      end
+
+    end
+  end
+end

data/lib/reaper-man/signer/rubygems.rb

@@ -0,0 +1,17 @@
+require 'reaper-man'
+
+module ReaperMan
+  class Signer
+    module Rubygems
+
+      def package(*pkgs)
+        nil
+      end
+
+      def valid_packages(*pkgs)
+        nil
+      end
+
+    end
+  end
+end

data/lib/reaper-man/signer.rb

@@ -0,0 +1,62 @@
+require 'reaper-man'
+
+module ReaperMan
+  # File signer
+  class Signer
+
+    autoload :Rpm, 'reaper-man/signer/rpm'
+    autoload :Deb, 'reaper-man/signer/deb'
+    autoload :Rubygems, 'reaper-man/signer/rubygems'
+
+    include Utils::Process
+
+    attr_reader :key_id
+    attr_reader :sign_chunk_size
+    attr_reader :sign_type
+    attr_reader :package_system
+    attr_reader :key_password
+
+    # Create new instance
+    #
+    # @param args [Hash]
+    # @option args [String] :signing_key
+    # @option args [String] :signing_chunk_size (defaults to 1)
+    # @option args [String] :signing_type (defaults to 'origin')
+    # @option args [String] :key_password (defaults to `ENV['REAPER_KEY_PASSWORD']`)
+    # @option args [String] :package_system
+    def initialize(args={})
+      args = args.to_smash
+      @key_id = args[:signing_key]
+      @sign_chunk_size = args.fetch(:signing_chunk_size, 1)
+      @sign_type = args.fetch(:signing_type, 'origin')
+      @key_password = args.fetch(:key_password, ENV['REAPER_KEY_PASSWORD'])
+      @package_system = args[:package_system]
+      case package_system.to_sym
+      when :deb, :apt
+        extend Deb
+      when :rpm, :yum
+        extend Rpm
+      when :gem, :rubygems
+        extend Rubygems
+      else
+        raise TypeError.new "Unknown packaging type requested (#{package_system})"
+      end
+    end
+
+    # Sign the file
+    #
+    # @param src [String] path to source file
+    # @param dst [String] path for destination file
+    # @return [String] destination file path
+    def file(src, dst=nil)
+      opts = ['--detach-sign', '--armor']
+      dst ||= src.sub(/#{Regexp.escape(File.extname(src))}$/, '.gpg')
+      opts << "--output '#{dst}'"
+      cmd = (['gpg'] + opts + [src]).join(' ')
+      shellout!(cmd)
+      dst
+    end
+
+  end
+
+end

data/lib/reaper-man/util-scripts/auto-debsigs

@@ -0,0 +1,30 @@
+#!/usr/bin/expect -f
+
+log_user 0
+
+set timeout 3
+
+set signtype [lindex $argv 0]
+set signkey [lindex $argv 1]
+set package [join [lrange $argv 2 [llength $argv]] " "]
+
+spawn /bin/bash
+expect {
+  -re ".*\$ *$" {
+    send "/usr/bin/debsigs --sign=\"$signtype\" --default-key=\"$signkey\" $package\r"
+  }
+  timeout { exit 1 }
+}
+expect {
+  -re {Enter passphrase: *} {
+    send "$env(REAPER_KEY_PASSWORD)\r"
+  }
+  timeout { exit 2 }
+}
+expect {
+  -re {\$ *} {
+    send "exit $?\rexit $?\r"
+  }
+  timeout { exit 3 }
+}
+expect eof

data/lib/reaper-man/utils/process.rb

@@ -0,0 +1,114 @@
+require 'reaper-man'
+require 'childprocess'
+require 'shellwords'
+require 'tempfile'
+
+module ReaperMan
+  module Utils
+    # Shellout helper
+    module Process
+
+      # NOTE: This is extracted from the elecksee gem and some
+      # features removed that are not required here. Should be
+      # wrapped up into standalone gem so it's more reusable.
+
+      class CommandFailed < StandardError
+        attr_accessor :original, :result
+        def initialize(orig, result=nil)
+          @original = orig
+          @result = result
+          super(orig.to_s)
+        end
+      end
+
+      class Timeout < CommandFailed
+      end
+
+      class CommandResult
+        attr_reader :original, :stdout, :stderr
+        def initialize(result)
+          @original = result
+          if(result.class.ancestors.map(&:to_s).include?('ChildProcess::AbstractProcess'))
+            extract_childprocess
+          elsif(result.class.to_s == 'Mixlib::ShellOut')
+            extract_shellout
+          else
+            raise TypeError.new("Unknown process result type received: #{result.class}")
+          end
+        end
+
+        def extract_childprocess
+          original.io.stdout.rewind
+          original.io.stderr.rewind
+          @stdout = original.io.stdout.read
+          @stderr = original.io.stderr.read
+          original.io.stdout.delete
+          original.io.stderr.delete
+        end
+
+        def extract_shellout
+          @stdout = original.stdout
+          @stderr = original.stderr
+        end
+      end
+
+      # Simple helper to shell out
+      def shellout(cmd, args={})
+        result = nil
+        if(defined?(Mixlib))
+          cmd_type = :mixlib_shellout
+        else
+          cmd_type = :childprocess
+        end
+        com_block = nil
+        case cmd_type
+        when :childprocess
+          com_block = lambda{ child_process_command(cmd, args) }
+        when :mixlib_shellout
+          require 'mixlib/shellout'
+          com_block = lambda{ mixlib_shellout_command(cmd, args) }
+        else
+          raise ArgumentError.new("Unknown shellout helper provided: #{cmd_type}")
+        end
+        result = defined?(Bundler) ? Bundler.with_clean_env{ com_block.call } : com_block.call
+        result == false ? false : CommandResult.new(result)
+      end
+
+      def child_process_command(cmd, args)
+        s_out = Tempfile.new('stdout')
+        s_err = Tempfile.new('stderr')
+        s_out.sync
+        s_err.sync
+        c_proc = ChildProcess.build(*Shellwords.split(cmd))
+        c_proc.environment.merge(args.fetch(:environment, {}))
+        c_proc.io.stdout = s_out
+        c_proc.io.stderr = s_err
+        c_proc.start
+        begin
+          c_proc.poll_for_exit(args[:timeout] || 10)
+        rescue ChildProcess::TimeoutError
+          c_proc.stop
+        ensure
+          raise CommandFailed.new("Command failed: #{cmd}", CommandResult.new(c_proc)) if c_proc.crashed?
+        end
+        c_proc
+      end
+
+      def mixlib_shellout_command(cmd, args)
+        shlout = nil
+        begin
+          shlout = Mixlib::ShellOut.new(cmd,
+            :timeout => args[:timeout] || 10,
+            :environment => args.fetch(:environment, {})
+          )
+          shlout.run_command
+          shlout.error!
+          shlout
+        rescue Mixlib::ShellOut::ShellCommandFailed, CommandFailed, Mixlib::ShellOut::CommandTimeout => e
+          raise CommandFailed.new(e, CommandResult.new(shlout))
+        end
+      end
+
+    end
+  end
+end

data/lib/reaper-man/utils.rb

@@ -0,0 +1,27 @@
+require 'digest/sha1'
+require 'digest/sha2'
+require 'digest/md5'
+
+require 'reaper-man'
+
+module ReaperMan
+  # Helper utilities
+  module Utils
+
+    autoload :Process, 'reaper-man/utils/process'
+
+    # Checksum helper
+    module Checksum
+
+      def checksum(io, type)
+        digest = Digest.const_get(type.to_s.upcase).new
+        while(data = io.read(2048))
+          digest << data
+        end
+        digest.hexdigest
+      end
+
+    end
+
+  end
+end

data/lib/reaper-man.rb

@@ -0,0 +1,13 @@
+require 'multi_json'
+
+module ReaperMan
+  autoload :Cli, 'reaper-man/cli'
+  autoload :Command, 'reaper-man/command'
+  autoload :Config, 'reaper-man/config'
+  autoload :Error, 'reaper-man/error'
+  autoload :Generator, 'reaper-man/generator'
+  autoload :PackageList, 'reaper-man/package_list'
+  autoload :Signer, 'reaper-man/signer'
+  autoload :Version, 'reaper-man/version'
+  autoload :Utils, 'reaper-man/utils'
+end