reaper-man 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7dc241935585b38014666047f83d37a8a7e171d1
-  data.tar.gz: 2e5bfd4f228ea7f741aa5c1acae67b34e9ed7c62
+  metadata.gz: 5e552f3927ecc938f4e80fe58a77e2ed46e3330b
+  data.tar.gz: 58680f7d1b1be0d7043340f423a086450ff6bfc5
 SHA512:
-  metadata.gz: 3e27625be0b88e8a3fd9bd74586e3575ef908b7386812f19642465a5302046c23b75f147016fe5170f3653b765dabc9cfbd3df2b2b1b485d44810cd28035bb54
-  data.tar.gz: 53bea3cc72a468f8e1ec26ca7be2f9a6a5161559aa9eb03e0192edea7f6ae7209f3c8896ec5593ab96d118f6d40b8d1844edac729e4c71674c6bdf6126db2336
+  metadata.gz: 8c977b039699149eba6dcbcf70be527426de5d986c9a0acc8f3cedd878c0e108d6f41e94e883b039c648c8178f13bd7e44c219c0f004f09bc2335cf6569b5875
+  data.tar.gz: 91b0add2951920e67f1364f8e2900e0875fed65f5b3c6b7a568fe249f497ae47469277c85ea0824486b7239ae81c60368ca6082406777088e210f1113c0ede39

data/CHANGELOG.md

@@ -0,0 +1,2 @@
+# v0.1.0
+* Initial release

data/CONTRIBUTING.md

@@ -0,0 +1,25 @@
+# Contributing
+
+## Branches
+
+### `master` branch
+
+The master branch is the current stable released version.
+
+### `develop` branch
+
+The develop branch is the current edge of development.
+
+## Pull requests
+
+* https://github.com/hw-labs/reaper-man
+
+Please base all pull requests of the `develop` branch. Merges to
+`master` only occur through the `develop` branch. Pull requests
+based on `master` will likely be cherry picked.
+
+## Issues
+
+Need to report an issue? Use the github issues:
+
+* https://github.com/hw-labs/reaper-man/issues

data/LICENSE

@@ -0,0 +1,13 @@
+   Copyright 2015 Chris Roberts
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -0,0 +1,55 @@
+## Reaper Man
+#### "What can the harvest hope for, if not for the care of the Reaper Man?"
+##### - Terry Pratchett
+
+Grow code, harvest packages
+
+### Harvests
+
+Package repository is maintained via `JSON` file. Packages are added or removed
+from the `JSON` registry. Repository generation will result in skeleton repository
+with the proper `Release` and `Packages` files. Package paths will be not exist
+within the generated repository. Resolving that issue is left to the reader.
+
+### Support
+
+#### Enabled
+
+* deb/apt
+* gem/rubygems
+
+#### In Progress
+
+* rpm/yum
+
+### Usage
+
+#### Add package to registry
+
+```
+> reaper-man package add my_pkg.deb --packages-file registry.json
+```
+
+### Remove package from registry
+
+```
+> reaper-man package remove my_pkg --packages-file registry.json
+```
+
+or remove a specific version
+
+```
+> reaper-man package remove my_pkg 1.0.0 --packages-file registry.json
+```
+
+### Create a repository
+
+```
+> reaper-man repo create --packages-file registry.json --package-system apt --output-directory /tmp/test-repo
+```
+
+This can also be used to update an existing repository structure.
+
+## Infos
+* Repository: https://github.com/hw-labs/reaper-man
+* IRC: Freenode @ #heavywater

data/bin/reaper-man

@@ -0,0 +1,83 @@
+#!/usr/bin/env ruby
+
+require 'bogo-cli'
+require 'reaper-man'
+
+Bogo::Cli::Setup.define do
+
+  on :v, :version, 'Print version' do
+    puts "reaper-man - Repository generation tool - [Version: #{ReaperMan::VERSION}]"
+    exit
+  end
+
+  common_options = lambda do
+    on :c, :config=, 'Path to configuration file'
+    on :k, :'signing-key=', 'Path to key for signing generated files'
+    on :t, :'signing-type=', 'Signing type name'
+    on :s, :sign, 'Enable file signing'
+    on :S, :'package-system=', 'Packaging system to generate repository (apt/yum/rubygems/supermarket)'
+    on :p, :'packages-file=', 'Path to packages file', :required => true
+  end
+
+  command :repo do
+    description 'Package repository generation'
+
+    repo_options = lambda do
+      on :o, :output_directory=, 'Path to output repository structure', :required => true
+    end
+
+    command :generate do
+      instance_exec(&common_options)
+      instance_exec(&repo_options)
+      description 'Generate repository'
+
+      run do |opts, args|
+        ReaperMan::Command::Repository::Generate.new(opts, args).execute!
+      end
+    end
+
+  end
+
+  command :package do
+    description 'Update packages within repositories'
+
+    package_options = lambda do
+      on :O, :origin=, 'Name of repository origin', :default => 'Default'
+      on :N, :codename=, 'Code name to add package', :default => 'all'
+      on :C, :component=, 'Component name to add package', :default => 'main'
+    end
+
+    command :add do
+      description 'Add package to repository manifest'
+      instance_exec(&common_options)
+      instance_exec(&package_options)
+
+      run do |opts, args|
+        ReaperMan::Command::Package::Add.new(opts, args).execute!
+      end
+    end
+
+    command :remove do
+      description 'Remove package from repository manifest'
+      instance_exec(&common_options)
+      instance_exec(&package_options)
+
+      run do |opts, args|
+        ReaperMan::Command::Package::Remove.new(opts, args).execute!
+      end
+    end
+
+  end
+
+  command :sign do
+    description 'Sign package(s)'
+
+    on :S, :'package-system=', 'Packaging system to generate repository (apt/yum/rubygems/supermarket)', :required => true
+    on :Z, :'signing-chunk-size=', 'Number of packages to sign at once', :default => 20, :as => Integer
+
+    run do |opts, args|
+      ReaperMan::Command::Sign.new(opts, args).execute!
+    end
+  end
+
+end

data/lib/reaper-man/command/package/add.rb

@@ -0,0 +1,24 @@
+require 'reaper-man'
+
+module ReaperMan
+  class Command
+    class Package
+
+      class Add < Package
+
+        def execute!
+          arguments.each do |path|
+            run_action "Adding package to repository manifest: #{path}" do
+              list = ReaperMan::PackageList.new(config[:packages_file], config)
+              list.add_package(path)
+              list.write!
+              nil
+            end
+          end
+        end
+
+      end
+
+    end
+  end
+end

data/lib/reaper-man/command/package/remove.rb

@@ -0,0 +1,24 @@
+require 'reaper-man'
+
+module ReaperMan
+  class Command
+    class Package
+
+      class Remove < Package
+
+        def execute!
+          arguments.each do |pkg|
+            run_action "Remove package from repository manifest: #{pkg}" do
+              list = ReaperMan::PackageList.new(config[:packages_file], config)
+              list.remove_package(pkg)
+              list.write!
+              nil
+            end
+          end
+        end
+
+      end
+
+    end
+  end
+end

data/lib/reaper-man/command/package.rb

@@ -0,0 +1,12 @@
+require 'reaper-man'
+
+module ReaperMan
+  class Command
+    class Package < Command
+
+      autoload :Add, 'reaper-man/command/package/add'
+      autoload :Remove, 'reaper-man/command/package/remove'
+
+    end
+  end
+end

data/lib/reaper-man/command/repository/generate.rb

@@ -0,0 +1,28 @@
+require 'reaper-man'
+
+module ReaperMan
+  class Command
+    class Repository
+
+      class Generate < Repository
+
+        def execute!
+          run_action 'Generating repository' do
+            ReaperMan::Generator.new(
+              config.merge(
+                Smash.new(
+                  :package_config => MultiJson.load(
+                    File.read(config[:packages_file])
+                  ).to_smash
+                )
+              )
+            ).generate!
+            nil
+          end
+        end
+
+      end
+
+    end
+  end
+end

data/lib/reaper-man/command/repository.rb

@@ -0,0 +1,11 @@
+require 'reaper-man'
+
+module ReaperMan
+  class Command
+    class Repository < Command
+
+      autoload :Generate, 'reaper-man/command/repository/generate'
+
+    end
+  end
+end

data/lib/reaper-man/command/sign.rb

@@ -0,0 +1,21 @@
+require 'reaper-man'
+
+module ReaperMan
+  class Command
+    class Sign < Command
+
+      def execute!
+        run_action 'Signing file(s)' do
+          signer = Signer.new(config)
+          files = Dir.glob(File.join(arguments.first, '**', '*'))
+          files.delete_if do |path|
+            !File.file?(path)
+          end
+          signer.package(*files)
+          nil
+        end
+      end
+
+    end
+  end
+end

data/lib/reaper-man/command.rb

@@ -0,0 +1,11 @@
+require 'reaper-man'
+
+module ReaperMan
+  class Command < Bogo::Cli::Command
+
+    autoload :Repository, 'reaper-man/command/repository'
+    autoload :Package, 'reaper-man/command/package'
+    autoload :Sign, 'reaper-man/command/sign'
+
+  end
+end

data/lib/reaper-man/error.rb

@@ -0,0 +1,20 @@
+require 'reaper-man'
+
+module ReaperMan
+  # Standard error for reaper errors
+  class Error < StandardError
+    # exit code for exception
+    CODE = -1
+
+    # @return [Integer]
+    def exit_code
+      self.class.const_get(:CODE)
+    end
+
+    # Define errors here
+    ['UnknownCommand', 'FileNotFound'].each_with_index do |klass_name, idx|
+      self.class_eval("class #{klass_name} < Error; CODE=#{idx + 1}; end;")
+    end
+
+  end
+end

data/lib/reaper-man/generator/apt.rb

@@ -0,0 +1,117 @@
+require 'reaper-man'
+
+module ReaperMan
+  class Generator
+    # Generator methods for apt
+    module Apt
+
+      # Generate the repository
+      def generate!
+        generate_dists(package_config[:apt])
+      end
+
+      # Generate the repository dists
+      #
+      # @param pkg_hash [Hash] repository description
+      # @return [TrueClass]
+      def generate_dists(pkg_hash)
+        pkg_hash.each do |origin_name, dists|
+          dists.each do |dist_name, dist_args|
+            dist_args[:components].each do |component_name, arches|
+              arches.each do |arch_name, packages|
+                package_file(origin_name, dist_name, component_name, arch_name, packages)
+                release_headers = Smash.new
+                release_headers['Label'] = dist_args['label']
+                release_headers['Archive'] = dist_name
+                sign_file_if_setup do
+                  release_file(origin_name, dist_name, component_name, arch_name, release_headers)
+                end
+              end
+            end
+            release_headers = Smash[
+              %w(Codename Suite Label Description Version).map do |field_name|
+                if(val = dist_args[field_name.downcase])
+                  [field_name, val]
+                end
+              end.compact
+            ]
+            release_headers['Components'] = dist_args[:components].keys.join(' ')
+            sign_file_if_setup do
+              release_file(origin_name, dist_name, release_headers)
+            end
+          end
+        end
+        true
+      end
+
+      # Sign file if configured for signing
+      #
+      # @yield block returning file path
+      # @return [String] file path
+      def sign_file_if_setup
+        path = yield
+        if(signer && options[:sign])
+          signer.file(path)
+        end
+        path
+      end
+
+      # Create Packages file
+      #
+      # @param args [String] argument list for file path
+      # @return [String] path to compressed Packages file
+      def package_file(*args)
+        pkgs = args.pop
+        args.insert(1, 'dists')
+        create_file(*args.push('Packages')) do |file|
+          pkgs.each do |pkg_name, pkgs|
+            pkgs.each do |pkg_version, pkg_meta|
+              pkg_meta.each do |field_name, field_value|
+                if(field_value)
+                  file.puts "#{field_name}: #{field_value}"
+                end
+              end
+              file.puts
+            end
+          end
+        end
+        compress_file(*args)
+      end
+
+      # Create Release file
+      #
+      # @param args [String] argument list for file path
+      # @return [TrueClass]
+      def release_file(*args)
+        header = args.detect{|a| a.is_a?(Hash)}
+        header ? args.delete(header) : header = Smash.new
+        header.merge(Smash[%w(Origin Codename Component Architecture).zip(args)])
+        header['Date'] = Time.now.utc.strftime('%a, %d %b %Y %H:%M:%S %Z')
+        args.insert(1, 'dists')
+        create_file(*args.dup.push('Release')) do |file|
+          contents = Dir.glob(File.join(File.dirname(file.path), '**', '*'))
+          header_content = header.map do |key, value|
+            next unless value
+            "#{key.to_s.capitalize}: #{value}"
+          end.compact.join("\n")
+          file.puts header_content
+          [['MD5Sum', :md5], ['SHA1', :sha1], ['SHA256', :sha256]].each do |field, digest|
+            file.puts "#{field}:"
+            contents.each do |content|
+              next if File.expand_path(content) == File.expand_path(file.path) || File.directory?(content)
+              File.open(content, 'r') do |content_file|
+                line = [' ']
+                line << checksum(content_file, digest)
+                line << content_file.size
+                line << content_file.path.sub(File.dirname(file.path), '').sub(/^\//, '')
+                file.puts line.join(' ')
+              end
+            end
+          end
+          true
+        end
+      end
+
+    end
+  end
+end

data/lib/reaper-man/generator/rubygems.rb

@@ -0,0 +1,91 @@
+require 'reaper-man'
+
+module ReaperMan
+  class Generator
+    # Generator methods for rubygems
+    module Rubygems
+
+      # Generate the rubygems repository
+      #
+      def generate!
+        generate_gemstore(package_config[:rubygem])
+      end
+
+      def generate_gemstore(gems)
+        generate_indexing(gems)
+        write_quick_specs(gems.fetch(:release, {}))
+        write_quick_specs(gems.fetch(:prerelease, {}))
+      end
+
+      def generate_indexing(gems)
+        build_spec_file('specs', gems.fetch(:release, {}))
+        build_spec_file('latest_specs', gems.fetch(:release, {}))
+        build_spec_file('prerelease', gems.fetch(:prerelease, {}))
+      end
+
+      def create_index(gems)
+        [].tap do |list|
+          gems.each do |name, all|
+            all.each do |version, info|
+              list << [name, Gem::Version.new(version.dup), info[:platform]]
+            end
+          end
+        end
+      end
+
+      def marshal_path
+        ['Marshal', marshal_version].join('.')
+      end
+
+      def marshal_version
+        [Marshal::MAJOR_VERSION, Marshal::MINOR_VERSION].join('.')
+      end
+
+      def build_spec_file(name, gems)
+        index = create_index(gems)
+        create_file("#{name}.#{marshal_version}") do |file|
+          file.write(Marshal.dump(index))
+        end
+        compress_file("#{name}.#{marshal_version}")
+      end
+
+      def write_quick_specs(gems)
+        gems.each do |name, list|
+          list.each do |version, info|
+            spec = Gem::Specification.new(name)
+            info.each do |var, value|
+              if(spec.respond_to?("#{var}="))
+                begin
+                  spec.send("#{var}=", value)
+                rescue Gem::InvalidSpecificationException => e
+                  # TODO: Do we have a logger in this project?
+                end
+              end
+            end
+            spec.version = Gem::Version.new(info[:version])
+            spec.date = Time.parse(info[:date])
+            info[:dependencies].each do |dep|
+              spec.add_dependency(*dep)
+            end
+            create_file('quick', marshal_path, "#{name}-#{version}.gemspec.rz") do |file|
+              file.write(Marshal.dump(spec))
+            end
+          end
+        end
+      end
+
+      # Sign file if configured for signing
+      #
+      # @yield block returning file path
+      # @return [String] file path
+      def sign_file_if_setup
+        path = yield
+        if(signer && options[:sign])
+          signer.file(path)
+        end
+        path
+      end
+
+    end
+  end
+end

data/lib/reaper-man/generator/yum.rb

@@ -0,0 +1,10 @@
+require 'reaper-man'
+require 'time'
+
+module ReaperMan
+  class Generator
+    # Generator methods for yum
+    module Yum
+    end
+  end
+end

data/lib/reaper-man/generator.rb

@@ -0,0 +1,98 @@
+require 'time'
+require 'zlib'
+require 'fileutils'
+
+require 'reaper-man'
+
+module ReaperMan
+  # Repository generator
+  class Generator
+
+    autoload :Apt, 'reaper-man/generator/apt'
+    autoload :Rpm, 'reaper-man/generator/rpm'
+    autoload :Rubygems, 'reaper-man/generator/rubygems'
+
+    include Utils::Checksum
+
+    # @return [String]
+    attr_reader :package_system
+    # @return [Hash]
+    attr_reader :package_config
+    # @return [Signer, NilClass]
+    attr_reader :signer
+    # @return [Hash]
+    attr_reader :options
+
+    # Create new instance
+    #
+    # @param args [Hash]
+    # @option args [String] :package_system apt/gem/etc...
+    # @option args [Hash] :package_config
+    # @option args [Signer] :signer
+    def initialize(args={})
+      @package_system = args[:package_system]
+      @package_config = args.fetch(:package_config, Smash.new)
+      @signer = args[:signer]
+      @options = args
+      extend self.class.const_get(package_system.to_s.split('_').map(&:capitalize).join.to_sym)
+    end
+
+    # Generate new repository
+    def generate!
+      raise NoMethodError.new 'Not implemented'
+    end
+
+    # Create new file
+    #
+    # @param name [String] argument list joined to output directory
+    # @yield block executed with file
+    # @yieldparam [String] path to file
+    # @return [String] path to file
+    def create_file(*name)
+      path = File.join(options[:output_directory], *name)
+      FileUtils.mkdir_p(File.dirname(path))
+      file = File.open(path, 'wb+')
+      if(block_given?)
+        yield file
+      end
+      file.close unless file.closed?
+      path
+    end
+
+    # Updates a file
+    #
+    # @param name [String] argument list joined to output directory
+    # @yield block executed with file
+    # @yieldparam [String] path to file
+    # @return [String] path to file
+    def for_file(*name)
+      path = File.join(options[:output_directory], *name)
+      FileUtils.mkdir_p(File.dirname(path))
+      if(block_given?)
+        file = File.open(path, 'a+')
+        yield file
+        file.close
+      end
+      path
+    end
+
+    # Compress a file (gzip)
+    #
+    # @param name [String] argument list joined to output directory
+    # @return [String] path to compressed file
+    def compress_file(*path)
+      compressed_path = path.dup
+      compressed_path.push("#{compressed_path.pop}.gz")
+      base_file = File.open(for_file(path))
+      create_file(compressed_path) do |file|
+        compressor = Zlib::GzipWriter.new(file)
+        while(data = base_file.read(2048))
+          compressor.write(data)
+        end
+        compressor.close
+      end
+    end
+
+  end
+
+end