bosh-release 1.5.0.pre.1113

data/README

@@ -0,0 +1,3 @@
+Package compiler for compiling micro bosh (or in the future micro cloud) packages.
+
+NOTE: It does not start the agent. It expects the agent to be up and running.

data/bin/bosh-release

@@ -0,0 +1,54 @@
+#!/usr/bin/env ruby
+
+require "trollop"
+require "bosh/release"
+
+CPIS = %w[aws openstack vcloud vsphere]
+COMMANDS = %w[apply compile]
+
+def usage(msg=nil)
+  $stderr.puts "error: #{msg}" if msg
+  $stderr.puts "usage: bosh-release <options> <command> <arguments>"
+  $stderr.puts "  options:"
+  $stderr.puts "    --cpi <cloud provider interface> (#{CPIS.join(', ')})"
+  $stderr.puts "    --job <job to compile> (defaults to 'micro_<cpi>')"
+  $stderr.puts "  commands:"
+  $stderr.puts "    apply <apply_spec> <mbus_uri>"
+  $stderr.puts "    compile <manifest> <release_tgz> <blobstore_path> <mbus_uri>"
+  exit 1
+end
+
+options = Trollop::options do
+  opt :cpi, "Cloud Provider Interface", :type => String, :required => true
+  opt :job, "Job to compile", :type => String
+  stop_on COMMANDS
+end
+
+unless CPIS.include?(options[:cpi])
+  Trollop::die :cpi, "unknown CPI '#{options[:cpi]}'"
+end
+
+unless options[:job_given]
+  options[:job] = "micro_#{options[:cpi]}"
+end
+
+options["command"] = ARGV.shift
+
+case options["command"]
+when "apply"
+  usage "apply requires 2 arguments" unless ARGV.size == 2
+  options["apply_spec"] = ARGV.shift
+  options["agent_uri"] = ARGV.shift
+when "compile"
+  usage "compile requires 4 arguments" unless ARGV.size == 4
+  options["manifest"] = ARGV.shift
+  options["release"] = ARGV.shift
+  options["blobstore_options"] = {"blobstore_path" => ARGV.shift}
+  options["agent_uri"] = ARGV.shift
+when nil
+  usage "no command given"
+else
+  usage "unknown command: #{options["command"]}"
+end
+
+Bosh::Release::Runner.start(options)

data/lib/bosh/release.rb

@@ -0,0 +1,22 @@
+module Bosh; module Release; end; end
+
+require 'logger'
+require 'yaml'
+require 'blobstore_client'
+require 'common/common'
+require 'common/properties'
+require 'agent_client'
+require 'bosh/release/compiler'
+require 'fileutils'
+
+module Bosh
+  module Release
+    class Runner
+      class << self
+        def start(options)
+          Compiler.new(options).start
+        end
+      end
+    end
+  end
+end

data/lib/bosh/release/compiler.rb

@@ -0,0 +1,266 @@
+module Bosh
+  module Release
+    class Compiler
+      include Bosh::Common::PropertyHelper
+
+      OPTIONS = {
+        "blobstore_options" => { "blobstore_path" => "/var/vcap/micro_bosh/data/cache" },
+        "blobstore_provider" => "local",
+        "base_dir"  => "/var/vcap",
+        "platform_name" => "ubuntu",
+        "agent_uri" => "https://vcap:vcap@localhost:6969"
+      }
+
+      AGENT_START_RETRIES=16
+
+      def initialize(options)
+        @options = OPTIONS.merge(options)
+        @logger = Logger.new(@options["logfile"] || STDOUT)
+
+        FileUtils.mkdir_p(File.join(@options["base_dir"], "packages"))
+        bsc_provider = @options["blobstore_provider"]
+        bsc_options = @options["blobstore_options"]
+        @logger.info("Creating Blobstore client with #{bsc_provider} provider and options #{bsc_options}")
+        @blobstore_client = Bosh::Blobstore::Client.create(bsc_provider, bsc_options)
+      end
+
+      def start
+        # Start the "compile" or "apply"
+        send(@options["command"].to_sym)
+      end
+
+      def apply_spec
+        File.join(@options["base_dir"], "micro/apply_spec.yml")
+      end
+
+      def connect_to_agent
+        num_tries = 0
+        begin
+          @agent = Bosh::Agent::Client.create(@options["agent_uri"], "user" => "vcap", "password" => "vcap")
+          @agent.ping
+        rescue => e
+          num_tries += 1
+          sleep 0.1
+          # Dont retry forever
+          retry if num_tries < AGENT_START_RETRIES
+          @logger.warn("Error connecting to agent #{e.inspect}")
+          raise
+        end
+      end
+
+      def compile
+        @logger.info("Compiling #{@options["manifest"]} with tarball #{@options["release"]}")
+        connect_to_agent
+        deployment_mf = Psych.load_file(File.expand_path(@options["manifest"]))
+        @spec = prep_spec(deployment_mf)
+
+        @packages = {}
+        @spec["job"] = { "name" => @options[:job] }
+
+        untar(@options["release"]) do |dir|
+          release_mf = Psych.load_file("release.MF")
+          jobs = []
+
+          jobs_to_compile(@options[:job], deployment_mf).each do |spec_job|
+            job = find_by_name(release_mf["jobs"], spec_job)
+            job_path = File.expand_path("jobs/#{job["name"]}.tgz")
+            jobs << apply_spec_job(job, job_path)
+
+            if job["name"] == @options[:job]
+              @spec["job"]["version"] = job["version"].to_s
+              @spec["job"]["template"] = @options[:job]
+              @spec["job"]["sha1"] = job["sha1"]
+              @spec["job"]["blobstore_id"] = @blobstore_client.create(File.new(job_path))
+            end
+
+            untar(job_path) do
+              job = Psych.load_file("job.MF")
+
+              # add default job spec properties to apply spec
+              add_default_properties(@spec["properties"], job["properties"])
+
+              # Compile job packages
+              compile_packages(dir, release_mf, job["packages"])
+            end
+          end
+
+          @spec["job"]["templates"] = jobs
+        end
+        cleanup
+
+        # save apply spec
+        FileUtils.mkdir_p(File.dirname(apply_spec))
+        File.open(apply_spec, 'w') { |f| f.write(Psych.dump(@spec)) }
+
+        @spec["packages"]
+      rescue => e
+        @logger.error("Error #{e.message}, #{e.backtrace.join("\n")}")
+      end
+
+      def find_by_name(enum, name)
+        result = enum.find { |j| j["name"] == name }
+        if result
+          result
+        else
+          raise "Could not find name #{name} in #{enum}"
+        end
+      end
+
+      # Check manifest for job collocation
+      def jobs_to_compile(name, manifest)
+        compile_job = manifest["jobs"].find { |j| j["name"] == name } if manifest["jobs"]
+        if compile_job
+          compile_job["template"]
+        else
+          [name]
+        end
+      end
+
+      def apply_spec_job(job, job_path)
+        {
+          "name" => job["name"],
+          "version" => job["version"].to_s,
+          "sha1" => job["sha1"],
+          "blobstore_id" => @blobstore_client.create(File.new(job_path))
+        }
+      end
+
+      def cleanup
+        FileUtils.rm_rf("#{@options["base_dir"]}/data/compile")
+        FileUtils.rm_rf("#{@options["base_dir"]}/data/packages")
+        FileUtils.rm_rf("#{@options["base_dir"]}/data/tmp")
+        FileUtils.rm_rf("#{@options["base_dir"]}/packages")
+      end
+
+      def prep_spec(deployment)
+        spec = {}
+        spec["deployment"] = "micro"
+        spec["release"] = deployment["release"]
+        spec["properties"] = deployment["properties"]
+        spec["index"] = 0
+        spec["packages"] = {}
+        spec["configuration_hash"] = {}
+
+        case @options[:cpi]
+        when "vsphere", "vcloud"
+          spec["networks"] = {"local" => {"ip" => "127.0.0.1"}}
+        when "aws"
+          spec["networks"] = {"type" => "dynamic"}
+        when "openstack"
+          spec["networks"] = {"type" => "dynamic"}
+        else
+          puts "WARNING: no CPI specified"
+        end
+
+        spec
+      end
+
+      def compile_packages(dir, manifest, packages)
+        packages.each do |name|
+          package = find_package(manifest, name)
+          compile_packages(dir, manifest, package["dependencies"]) if package["dependencies"]
+
+          @logger.debug "compiling package #{name}"
+          compile_package(dir, package, name)
+        end
+      end
+
+      def find_package(manifest, name)
+        manifest["packages"].detect { |p| p["name"] == name }
+      end
+
+      def compile_package(dir, package, name)
+        # return if package is already compiled
+        return if @spec["packages"].has_key?(name)
+
+        src = "#{dir}/packages/#{name}.tgz"
+        version = package["version"]
+        dependencies = package["dependencies"]
+
+        # push source package into blobstore
+        file = File.new(src)
+        id = @blobstore_client.create(file)
+
+        sha1 = "sha1"
+        dependencies = {}
+        package["dependencies"].each do |name|
+          @logger.debug "dependency: #{name} = #{@spec["packages"][name]}"
+          dependencies[name] = @spec["packages"][name]
+        end
+
+        result = @agent.run_task(:compile_package, id, sha1, name, version, dependencies)
+        @logger.info("result is #{result}")
+
+        # remove source package from blobstore
+        @blobstore_client.delete(id)
+
+        id = result["result"]["blobstore_id"]
+        @logger.debug("stored package #{name} as #{id}")
+
+        @spec["packages"][name] = {
+          "name" => name,
+          "version" => version.to_s,
+          "sha1" => result["result"]["sha1"],
+          "blobstore_id" => id
+        }
+      end
+
+      def untar(file)
+        prev_dir = Dir.getwd
+        dir = Dir.mktmpdir
+        Dir.chdir(dir)
+        @logger.debug("untaring #{file} into #{dir}")
+        out = `tar xzf #{file} 2>&1`
+        raise RuntimeError, "untar of #{file} failed: #{out}" unless $? == 0
+        yield dir
+      ensure
+        Dir.chdir(prev_dir)
+        FileUtils.rm_rf dir
+      end
+
+      def apply
+        connect_to_agent
+        FileUtils.mkdir_p(File.join(@options["base_dir"], 'data/log'))
+        # Stop services
+        @logger.info("Stopping services")
+        begin
+          @agent.run_task(:stop)
+        rescue => e
+          @logger.warn("Ignoring error to stop services #{e.inspect}")
+        end
+
+        @spec = Psych.load_file(@options["apply_spec"])
+        @logger.info("#{@spec.inspect}")
+        update_bosh_spec
+        @agent.run_task(:apply, @spec)
+
+        @logger.info("Starting services")
+        @agent.run_task(:start)
+      end
+
+      def update_bosh_spec
+        uri = URI.parse(@options["agent_uri"])
+        ip = uri.host
+        properties = @spec["properties"]
+        properties["blobstore"]["address"] = ip
+        properties["postgres"]["address"] = ip
+        properties["director"]["address"] = ip
+        properties["redis"]["address"] = ip
+        properties["nats"]["address"] = ip
+        @spec["properties"] = properties
+      end
+
+      def add_default_properties(spec_properties, job_properties)
+        return unless job_properties
+
+        job_properties.each_pair do |name, definition|
+          unless definition["default"].nil?
+            copy_property(spec_properties, spec_properties, name, definition["default"])
+          end
+        end
+      end
+
+    end
+  end
+end
+

data/lib/bosh/release/version.rb

@@ -0,0 +1,6 @@
+module Bosh
+   module Release
+      VERSION = '1.5.0.pre.1113'
+   end
+end
+

metadata

@@ -0,0 +1,133 @@
+--- !ruby/object:Gem::Specification
+name: bosh-release
+version: !ruby/object:Gem::Version
+  version: 1.5.0.pre.1113
+  prerelease: 6
+platform: ruby
+authors:
+- VMware
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-10-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: agent_client
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.5.0.pre.1113
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.5.0.pre.1113
+- !ruby/object:Gem::Dependency
+  name: blobstore_client
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.5.0.pre.1113
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.5.0.pre.1113
+- !ruby/object:Gem::Dependency
+  name: bosh_common
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.5.0.pre.1113
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.5.0.pre.1113
+- !ruby/object:Gem::Dependency
+  name: yajl-ruby
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+- !ruby/object:Gem::Dependency
+  name: trollop
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.16'
+description: ! 'Bosh package compiler
+
+  cfd471'
+email: support@cloudfoundry.com
+executables:
+- bosh-release
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/bosh/release.rb
+- lib/bosh/release/compiler.rb
+- lib/bosh/release/version.rb
+- README
+- bin/bosh-release
+homepage: https://github.com/cloudfoundry/bosh
+licenses:
+- Apache 2.0
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.9.3
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>'
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Bosh package compiler
+test_files: []