boshify 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 301facaa8c05504dbec23d0896f196a1714a7551
+  data.tar.gz: c37f62978214b529975c01c8c1ef3a84549ed5a0
+SHA512:
+  metadata.gz: 3fe9bb319022d84106081a18ad7e341c90cb256d3e670b5c9eb9f70d4c56ee7c2012d35eb2e8eb68c8fc702e31ef22b3021c44b764d3ab4f8ed91fa156749cb7
+  data.tar.gz: 813e29e859b7945191cc061298038923ac9f76153610f886b6303a1c684748e2a463237ae001350d6191f957b8ba656f5a955f9a63678897bc9ec7a9fd9af4ed

data/bin/boshify

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+require 'boshify'
+require 'bosh_lite_helpers'
+
+# Boshify generates BOSH releases
+module Boshify
+  cr = BoshLiteHelpers::CommandRunner.new
+  fs = Filesystem.new
+  dl = Downloader.new(filesystem: fs)
+  release_creator = ReleaseCreator.new(filesystem: fs,
+                                       release_dir: Dir.pwd,
+                                       cmd_runner: cr)
+  pkg_source = UbuntuPackages.new(downloader: dl, cmd_runner: cr)
+  converter = PackageConverter.new(package_source: pkg_source,
+                                   downloader: dl,
+                                   release_creator: release_creator)
+  cmd_line = CommandLine.new(program_name: $PROGRAM_NAME,
+                             package_converter: converter)
+  result = cmd_line.run(ARGV)
+  puts result[:stdout] unless result[:stdout].empty?
+  STDERR.puts result[:stderr] unless result[:stderr].empty?
+  exit result[:exit_code]
+end

data/lib/boshify/command_line.rb

@@ -0,0 +1,81 @@
+require 'optparse'
+require 'pathname'
+
+module Boshify
+  # Command line argument processing
+  class CommandLine
+    def initialize(options)
+      unless options[:program_name]
+        fail ArgumentError, 'Program name must be specified'
+      end
+      unless options[:package_converter]
+        fail ArgumentError, 'Package converter must be specified'
+      end
+      @program_name = Pathname.new(options[:program_name]).basename
+      @package_converter = options[:package_converter]
+    end
+
+    def run(args)
+      with_options(args) do |options|
+        begin
+          use_mirror_if_specified(options[:mirror])
+          @package_converter.create_release_for(name: options[:package])
+          { exit_code: 0, stdout: "Package #{options[:package]} converted",
+            stderr: '' }
+        rescue => e
+          { exit_code: 1, stdout: '', stderr: e.message }
+        end
+      end
+    end
+
+    private
+
+    def with_options(args)
+      options = parse(args)
+      if options[:package]
+        yield options
+      else
+        { exit_code: 0, stdout: @parser.help, stderr: '' }
+      end
+      rescue OptionParser::MissingArgument
+        { exit_code: 1, stdout: @parser.help, stderr: '' }
+    end
+
+    def use_mirror_if_specified(mirror_url)
+      return unless mirror_url
+      @package_converter.package_source.mirror_url = mirror_url
+    end
+
+    def parse(args)
+      options = {}
+      @parser = OptionParser.new do |cmd_opts|
+        cmd_opts.banner = "#{@program_name} [options]"
+        add_package_option(cmd_opts, options)
+        add_mirror_option(cmd_opts, options)
+        add_help_option(cmd_opts, options)
+      end
+      @parser.parse!(args)
+      options
+    end
+
+    def add_package_option(cmd_opts, options)
+      cmd_opts.on('-p', '--package PACKAGE', 'Ubuntu source package') do |p|
+        options[:package] = p
+      end
+    end
+
+    def add_mirror_option(cmd_opts, options)
+      cmd_opts.on('-m',
+                  '--mirror [MIRROR]',
+                  'Alternate Ubuntu mirror URL') do |m|
+        options[:mirror] = m
+      end
+    end
+
+    def add_help_option(cmd_opts, options)
+      cmd_opts.on('-h', '--help', 'Print help') do
+        options[:help] = true
+      end
+    end
+  end
+end

data/lib/boshify/downloader.rb

@@ -0,0 +1,22 @@
+require 'httparty'
+require 'uri'
+
+module Boshify
+  class DownloadError < StandardError; end
+
+  # Downloads remote resources to disk
+  class Downloader
+    def initialize(options = {})
+      @filesystem = options[:filesystem]
+    end
+
+    def get(url)
+      bn = Pathname.new(URI.parse(url.to_s).path).basename
+      r = HTTParty.get(url)
+      unless r.ok?
+        fail DownloadError, "The resource could not be retrieved: #{url}"
+      end
+      @filesystem.write_file(basename: bn, content: r.body)
+    end
+  end
+end

data/lib/boshify/filesystem.rb

@@ -0,0 +1,44 @@
+require 'fileutils'
+require 'tmpdir'
+require 'pathname'
+
+module Boshify
+  # Wrapper around filesystem operations
+  class Filesystem
+    def copy(from, to)
+      FileUtils.copy(from, to)
+    end
+
+    def mkdir_p(path)
+      path.mkpath
+    end
+
+    def write_file(options)
+      check_file_options!(options)
+      file = determine_file_path(options)
+      File.open(file.cleanpath, 'w') { |f| f.write(options[:content]) }
+      file
+    end
+
+    private
+
+    def check_file_options!(options)
+      unless options[:content]
+        fail ArgumentError, 'File content must be specified'
+      end
+
+      # rubocop:disable GuardClause
+      unless options[:path] || options[:basename]
+        fail ArgumentError, 'Either basename or path must be specified'
+      end
+    end
+
+    def determine_file_path(options)
+      if options[:path]
+        options[:path]
+      else
+        Pathname.new(Dir.mktmpdir) + options[:basename].basename
+      end
+    end
+  end
+end

data/lib/boshify/package_converter.rb

@@ -0,0 +1,22 @@
+module Boshify
+  # Converts an operating system package to a BOSH release
+  class PackageConverter
+    attr_reader :package_source
+
+    def initialize(options)
+      @package_source = options[:package_source]
+      @downloader = options[:downloader]
+      @release_creator = options[:release_creator]
+    end
+
+    def create_release_for(package = {})
+      @package_source.refresh
+      local_path = @downloader.get(
+        @package_source.source_tarball_url(package[:name]))
+      @release_creator.create_release(name: package[:name], packages: [
+        name: package[:name],
+        source_tarball: local_path
+      ])
+    end
+  end
+end

data/lib/boshify/release_creator.rb

@@ -0,0 +1,154 @@
+require 'pathname'
+require 'yaml'
+
+module Boshify
+  # Responsible for generating BOSH releases
+  # rubocop:disable ClassLength
+  class ReleaseCreator
+    def initialize(options)
+      check_options!(options)
+      @fs = options[:filesystem]
+      @release_dir = Pathname.new(options[:release_dir])
+      @cmd_runner = options[:cmd_runner]
+    end
+
+    def create_release(release)
+      create_empty_release
+      create_placeholder_blobstore_config(release)
+      create_job(release)
+      create_packages(release)
+    end
+
+    private
+
+    def check_options!(options)
+      unless options[:filesystem]
+        fail ArgumentError, 'Filesystem must be provided'
+      end
+      unless options[:release_dir]
+        fail ArgumentError, 'Release directory must be provided'
+      end
+      # rubocop:disable GuardClause
+      unless options[:cmd_runner]
+        fail ArgumentError, 'Command runner must be provided'
+      end
+    end
+
+    def create_empty_release
+      create_release_dirs
+      generate_empty_blobs_yaml
+    end
+
+    def create_release_dirs
+      %w(blobs config jobs packages src).each do |dir|
+        @fs.mkdir_p(@release_dir + dir)
+      end
+    end
+
+    def generate_empty_blobs_yaml
+      @fs.write_file(path: @release_dir + 'config' + 'blobs.yml',
+                     content: YAML.dump({}))
+    end
+
+    # rubocop:disable MethodLength
+    def create_placeholder_blobstore_config(release)
+      @fs.write_file(path: @release_dir + 'config' + 'final.yml',
+                     content: YAML.dump(
+        'blobstore' => {
+          'provider' => 's3',
+          'options' => {
+            'bucket_name' => "#{release[:name]}-release",
+            'access_key_id' => 'MY_ACCESS_KEY_ID',
+            'secret_acces_key' => 'MY_SECRET_ACCESS_KEY',
+            'encryption_key' => 'MY_ENCRYPTION_KEY'
+          }
+        },
+        'final_name' => release[:name]
+      ))
+    end
+
+    def create_job(release)
+      job_dir = @release_dir + 'jobs' + release[:name]
+      @fs.mkdir_p(job_dir)
+      @fs.write_file(path: job_dir + 'monit', content: '')
+      @fs.write_file(path: job_dir + 'spec', content: job_spec(release))
+    end
+
+    def job_spec(release)
+      YAML.dump(
+        'name' => release[:name],
+        'packages' => release[:packages].map { |p| p[:name] },
+        'templates' => {}
+      )
+    end
+
+    def build_path(source_tarball)
+      files = @cmd_runner.run("tar -ztf #{source_tarball}",
+                              quiet: true)[:stdout].split("\n")
+      directory_with_configure(files.map { |p| Pathname.new(p) })
+    end
+
+    def directory_with_configure(paths)
+      paths.find { |p| p.basename == Pathname.new('configure') }.dirname
+    end
+
+    def create_packages(release)
+      release[:packages].each do |pkg|
+        pkg_dir = make_package_dir(pkg[:name])
+        bp = blob_path(pkg)
+
+        generate_package_spec(pkg, pkg_dir, bp)
+        copy_blob_into_place(pkg, bp)
+        generate_package_script(pkg_dir, bp, pkg[:source_tarball])
+      end
+    end
+
+    def generate_package_script(pkg_dir, blob_path, source_tarball)
+      pkg_script = packaging_script(blob_path, build_path(source_tarball))
+      @fs.write_file(path: pkg_dir + 'packaging', content: pkg_script)
+    end
+
+    def blob_path(pkg)
+      "#{pkg[:name]}/#{pkg[:source_tarball].basename}"
+    end
+
+    def make_package_dir(pkg_name)
+      pkg_dir = @release_dir + 'packages' + pkg_name
+      @fs.mkdir_p(pkg_dir)
+      pkg_dir
+    end
+
+    def generate_package_spec(pkg, pkg_dir, blob_path)
+      @fs.write_file(path: pkg_dir + 'spec',
+                     content: package_spec(pkg, blob_path))
+    end
+
+    def copy_blob_into_place(pkg, blob_path)
+      @fs.mkdir_p(@release_dir + 'blobs' + pkg[:name])
+      @fs.copy(pkg[:source_tarball],
+               Pathname.new(@release_dir + 'blobs' + blob_path))
+    end
+
+    def packaging_script(blob_path, build_dir_path)
+      "#!/bin/bash
+set -e
+set -u
+
+tar zxvf #{blob_path}
+cd #{build_dir_path}
+
+./configure --prefix=${BOSH_INSTALL_TARGET}
+
+make
+make install"
+    end
+
+    def package_spec(pkg, blob_path)
+      YAML.dump(
+        'name' => pkg[:name],
+        'dependencies' => [],
+        'files' => [blob_path]
+      )
+    end
+  end
+end

data/lib/boshify/ubuntu_packages.rb

@@ -0,0 +1,111 @@
+require 'uri'
+module Boshify
+  class PackageNotFoundError < StandardError; end
+  class InvalidPackageMetadataError < StandardError; end
+
+  # Ubuntu package source
+  class UbuntuPackages
+    attr_reader :mirror_url
+
+    def initialize(options = {})
+      @downloader = options[:downloader]
+      @cmd_runner = options[:cmd_runner]
+      self.mirror_url = options[:mirror_url] || 'http://us.archive.ubuntu.com/ubuntu'
+    end
+
+    def mirror_url=(mirror)
+      @mirror_url = URI.parse("#{mirror}/")
+    end
+
+    def refresh
+      @all_packages = packages_hash(
+        parse(decompress(download_sources_metadata)))
+    end
+
+    def source_tarball_url(package_name)
+      unless @all_packages[package_name]
+        fail PackageNotFoundError, "Package #{package_name} was not found"
+      end
+      pkg = @all_packages[package_name]
+      mirror_url + "#{pkg['Directory']}/#{original_tarball(pkg)}"
+    end
+
+    def parse(input)
+      group_package_values(
+        as_a_hash(
+          parse_multiline_values(
+            group_by_whether_pairs(
+              split_key_value_pairs(input)))))
+    end
+
+    private
+
+    def original_tarball(pkg)
+      pkg['Files'].find { |f| f[:name].end_with?('orig.tar.gz') }[:name]
+    end
+
+    def download_sources_metadata
+      @downloader.get(mirror_url + 'dists/lucid/main/source/Sources.bz2')
+    end
+
+    def decompress(local_path)
+      result = @cmd_runner.run("bzcat #{local_path}", quiet: true)
+      if result[:exit_code] != 0
+        fail InvalidPackageMetadataError,
+             "Could not decompress: #{result[:stderr]}"
+      end
+      result[:stdout]
+    end
+
+    FILE_KEYS = %w(Files Checksums-Sha1 Checksums-Sha256)
+
+    def split_key_value_pairs(input)
+      input.lines.map { |line| line.split(':', 2).map { |f| f.strip } }
+    end
+
+    def group_by_whether_pairs(pairs)
+      pairs.chunk { |p| p.size == 2 }.to_a
+    end
+
+    def parse_multiline_values(pairs)
+      # rubocop:disable Next
+      pairs.each_with_index do |v, i|
+        if !v[0] && FILE_KEYS.include?(pairs[i - 1][1].last[0])
+          pairs[i - 1][1].last[1] = remove_empty(v[1]).map do |line|
+            file = line[0].split(' ')
+            { name: file[2], size_bytes: file[1].to_i, checksum: file[0] }
+          end
+          pairs[i] = nil
+        end
+      end
+      pairs.compact
+    end
+
+    def remove_empty(lines)
+      lines.reject { |line| line[0].empty? }
+    end
+
+    def as_a_hash(pairs)
+      pairs.map { |b, p| Hash[p] if b }.compact
+    end
+
+    def group_package_values(pkgs)
+      last_pkg_index = -1
+      pkgs.each_with_index do |pkg, i|
+        if pkg.key?('Package')
+          last_pkg_index = i
+        else
+          pkgs[last_pkg_index].merge!(pkg)
+          pkgs[i] = nil
+        end
+      end
+      pkgs.compact
+    end
+
+    def packages_hash(pkgs)
+      Hash[pkgs.map do |pkg|
+        [pkg['Package'], pkg]
+      end]
+    end
+  end
+end

data/lib/boshify.rb

@@ -0,0 +1,6 @@
+require_relative 'boshify/command_line'
+require_relative 'boshify/downloader'
+require_relative 'boshify/filesystem'
+require_relative 'boshify/package_converter'
+require_relative 'boshify/release_creator'
+require_relative 'boshify/ubuntu_packages'

metadata

@@ -0,0 +1,66 @@
+--- !ruby/object:Gem::Specification
+name: boshify
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Andrew Crump
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-08-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.13.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.13.1
+description: Generates BOSH releases
+email: andrew@cloudcredo.com
+executables:
+- boshify
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/boshify
+- lib/boshify.rb
+- lib/boshify/command_line.rb
+- lib/boshify/downloader.rb
+- lib/boshify/filesystem.rb
+- lib/boshify/package_converter.rb
+- lib/boshify/release_creator.rb
+- lib/boshify/ubuntu_packages.rb
+homepage: https://github.com/cloudcredo/boshify
+licenses:
+- Apache
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Generates BOSH releases
+test_files: []