bosh-bootstrap 0.5.0

data/lib/bosh-bootstrap/commander.rb

@@ -0,0 +1,9 @@
+module Bosh::Bootstrap::Commander
+end
+
+require "bosh-bootstrap/commander/command"
+require "bosh-bootstrap/commander/remote_script_command"
+require "bosh-bootstrap/commander/upload_command"
+require "bosh-bootstrap/commander/commands"
+require "bosh-bootstrap/commander/local_server"
+require "bosh-bootstrap/commander/remote_server"

data/lib/bosh-bootstrap/commander/README.md

@@ -0,0 +1,47 @@
+# Bosh::Bootstrap::Commander
+
+The sequence of commands that are run on an Inception VM can be either invoked locally on the Inception VM or from a remote machine. The `Commander` provides a DSL for describing the commands to be run and allows them to be sequentially run against a local or remote server.
+
+Remote servers are accessed via SSH commands.
+
+Example commands:
+``` ruby
+commands = Bosh::Bootstrap::Commander::Commands.new do |server|
+  server.create "vcap user", <<-BASH
+    #!/usr/bin/env bash
+    
+    groupadd vcap 
+    useradd vcap -m -g vcap
+    mkdir -p /home/vcap/.ssh
+    chown -R vcap:vcap /home/vcap/.ssh
+  BASH
+
+  server.install "rvm & ruby", <<-BASH
+    #!/usr/bin/env bash
+    
+    if [[ -x rvm ]]
+    then
+      rvm get stable
+    else
+      curl -L get.rvm.io | bash -s stable
+      source /etc/profile.d/rvm.sh
+    fi
+    command rvm install 1.9.3 # oh god this takes a long time
+    rvm 1.9.3
+    rvm alias create default 1.9.3
+  BASH
+end
+
+@local_machine.run(commands)
+@server.run(commands)
+```
+
+There is a set of predefined command methods which give nicer text output. You can also invoke any method upon `server` and it will be supported as the command name. The name of the method invoked is semantically meaningless; it is a convenience. 
+
+* `assign`
+* `create`
+* `download`
+* `install`
+* `provision`
+* `store`
+* `validate`

data/lib/bosh-bootstrap/commander/command.rb

@@ -0,0 +1,25 @@
+# A single command/script to be run on a local/remote server
+# For the display, it has an active ("installing") and 
+# past tense ("installed") verb and a noub/description ("packages")
+module Bosh::Bootstrap::Commander
+  class Command
+    attr_reader :command       # verb e.g. "install"
+    attr_reader :description   # noun phrase, e.g. "packages"
+
+    attr_reader :full_present_tense  # e.g. "installing packages"
+    attr_reader :full_past_tense    # e.g. "installed packages"
+  
+    def initialize(command, description, full_present_tense=nil, full_past_tense=nil)
+      @command            = command
+      @description        = description
+      @full_present_tense = full_present_tense || "#{command} #{description}"
+      @full_past_tense    = full_past_tense || "#{command} #{description}"
+    end
+
+    # Invoke this command (subclass) to call back upon
+    # +server+ to perform a server helper
+    def perform(server)
+      raise "please implement this method to call back upon `server`"
+    end
+  end
+end

data/lib/bosh-bootstrap/commander/commands.rb

@@ -0,0 +1,80 @@
+module Bosh::Bootstrap::Commander
+  class Commands
+    attr_reader :commands
+
+    def initialize(&block)
+      @commands = []
+      yield self
+    end
+
+    def upload_file(target_path, file_contents)
+      @commands << UploadCommand.new(target_path, file_contents)
+    end
+
+    #
+    # Generic remote script commands with custom phrases
+    #
+
+    def assign(description, script, options={})
+      @commands << RemoteScriptCommand.new(
+        "assign", description, script,
+        "assigning #{description}", "assigned #{description}", options)
+    end
+
+    # Runs a script on target server, and stores the (stripped) STDOUT into
+    # settings.
+    # 
+    # Usage:
+    # server.capture_value "salted password", script("convert_salted_password", "PASSWORD" => settings.bosh.password),
+    #   :settings => "bosh.salted_password"
+    #
+    # Would store the returned STDOUT into settings[:bosh][:salted_password]
+    def capture_value(description, script, options)
+      @commands << RemoteScriptCommand.new(
+        "capture value", description, script,
+        "captures value of #{description}", "captured value of #{description}", options)
+    end
+
+    def create(description, script, options={})
+      @commands << RemoteScriptCommand.new(
+        "create", description, script,
+        "creating #{description}", "created #{description}", options)
+    end
+
+    def download(description, script, options={})
+      @commands << RemoteScriptCommand.new(
+        "download", description, script,
+        "downloading #{description}", "downloaded #{description}", options)
+    end
+
+    def install(description, script, options={})
+      @commands << RemoteScriptCommand.new(
+        "install", description, script,
+        "installing #{description}", "installed #{description}", options)
+    end
+
+    def provision(description, script, options={})
+      @commands << RemoteScriptCommand.new(
+        "provision", description, script,
+        "provisioning #{description}", "provisioned #{description}", options)
+    end
+
+    def store(description, script, options={})
+      @commands << RemoteScriptCommand.new(
+        "store", description, script,
+        "storing #{description}", "stored #{description}", options)
+    end
+
+    def validate(description, script, options={})
+      @commands << RemoteScriptCommand.new(
+        "validate", description, script,
+        "validating #{description}", "validated #{description}", options)
+    end
+
+    # catch-all for commands with generic active/past tense phrases
+    def method_missing(command, *args, &blk)
+      description, script = args[0..1]
+      @commands << RemoteScriptCommand.new(command.to_s, description, script)
+    end
+  end
+end

data/lib/bosh-bootstrap/commander/local_server.rb

@@ -0,0 +1,68 @@
+require "popen4"
+
+class Bosh::Bootstrap::Commander::LocalServer
+  attr_reader :logfile
+
+  def initialize(logfile=STDERR)
+    @logfile = logfile
+  end
+
+  # Execute the +Command+ objects, in sequential order
+  # upon the local server
+  # +commands+ is a +Commands+ container
+  #
+  # Returns false once any subcommand fails to execute successfully
+  def run(commands)
+    commands.commands.each do |command|
+      puts command.full_present_tense
+      if command.perform(self)
+        Thor::Base.shell.new.say "Successfully #{command.full_past_tense}", :green
+      else
+        Thor::Base.shell.new.say_status "error", "#{command.full_present_tense}", :red
+        return false
+      end
+    end
+  end
+
+  #
+  # Commands performed on local server
+  # These map to Command subclasses, which then callback to these
+  # local server specific implementations
+  #
+
+  # Run a script
+  def run_script(command, script, options={})
+    command.as_file do |execution_command|
+      `chmod +x #{execution_command}`
+      status = POpen4::popen4(execution_command) do |stdout, stderr, stdin, pid|
+        out = stdout.read
+        logfile.puts out unless out.strip == ""
+        err = stderr.read
+        if err.strip != ""
+          logfile.puts err
+          STDERR.puts err if logfile != STDERR
+        end
+        logfile.flush
+      end
+      status.success?
+    end
+  rescue StandardError => e
+    logfile.puts e.message
+    false
+  end
+
+  # Upload a file (put a file into local filesystem)
+  def upload_file(command, path, contents, upload_as_user=nil)
+    basedir = File.dirname(path)
+    unless File.directory?(basedir)
+      logfile.puts "creating micro-bosh manifest folder: #{basedir}"
+      FileUtils.mkdir_p(basedir)
+    end
+    logfile.puts "creating micro-bosh manifest: #{path}"
+    File.open(path, "w") { |file| file << contents }
+    true
+  rescue StandardError => e
+    logfile.puts e.message
+    false
+  end
+end

data/lib/bosh-bootstrap/commander/remote_script_command.rb

@@ -0,0 +1,48 @@
+# A single command/script to be run on a local/remote server
+# For the display, it has an active ("installing") and 
+# past tense ("installed") verb and a noub/description ("packages")
+module Bosh::Bootstrap::Commander
+  class RemoteScriptCommand < Command
+    attr_reader :command       # verb e.g. "install"
+    attr_reader :description   # noun phrase, e.g. "packages"
+    attr_reader :script
+
+    attr_reader :full_present_tense  # e.g. "installing packages"
+    attr_reader :full_past_tense    # e.g. "installed packages"
+
+    # Optional:
+    attr_reader :specific_run_as_user # e.g. ubuntu
+    attr_reader :settings             # settings manifest (result of script might get stored back)
+    attr_reader :save_output_to_settings_key # e.g. bosh.salted_password
+
+    def initialize(command, description, script, full_present_tense=nil, full_past_tense=nil, options={})
+      super(command, description, full_present_tense, full_past_tense)
+      @script = script
+      @specific_run_as_user = options[:user]
+      @settings = options[:settings]
+      @save_output_to_settings_key = options[:save_output_to_settings_key]
+    end
+
+    # Invoke this command to call back upon +server.run_script+ 
+    def perform(server)
+      server.run_script(self, script, :user => specific_run_as_user,
+        :settings => settings, :save_output_to_settings_key => save_output_to_settings_key)
+    end
+
+    # Provide a filename that represents this Command
+    def to_filename
+      @to_filename ||= "#{command} #{description}".gsub(/\W+/, '_')
+    end
+
+    # Stores the script on the local filesystem in a temporary directory
+    # Returns path
+    def as_file(&block)
+      tmpdir = ENV['TMPDIR'] || "/tmp"
+      script_path = File.join(tmpdir, to_filename)
+      File.open(script_path, "w") do |f|
+        f << @script
+      end
+      yield script_path
+    end
+  end
+end

data/lib/bosh-bootstrap/commander/remote_server.rb

@@ -0,0 +1,121 @@
+require "net/scp"
+require "tempfile"
+
+class Bosh::Bootstrap::Commander::RemoteServer
+
+  attr_reader :host
+  attr_reader :default_username
+  attr_reader :logfile
+
+  def initialize(host, logfile=STDERR)
+    @host, @logfile = host, logfile
+    @default_username = "vcap" # unless overridden by a Command (before vcap exists)
+  end
+
+  # Execute the +Command+ objects, in sequential order
+  # upon the local server
+  # +commands+ is a +Commands+ container
+  #
+  # Returns false once any subcommand fails to execute successfully
+  def run(commands)
+    commands.commands.each do |command|
+      puts command.full_present_tense
+      if command.perform(self)
+        Thor::Base.shell.new.say "Successfully #{command.full_past_tense}", :green
+      else
+        Thor::Base.shell.new.say_status "error", "#{command.full_present_tense}", :red
+        return false
+      end
+    end
+  end
+
+  #
+  # Commands performed on remote server
+  # These map to Command subclasses, which then callback to these
+  # remote server specific implementations
+  #
+
+  # Run a script
+  #
+  # Uploads it to the remote server, makes it executable, then executes
+  # Stores the last line of stripped STDOUT/STDERR into a settings field, 
+  #   if :settings & :save_output_to_settings_key => "x.y.z" provided
+  def run_script(command, script, options={})
+    run_as_user  = options[:user] || default_username
+    settings     = options[:settings]
+    settings_key = options[:save_output_to_settings_key]
+
+    remote_path = remote_tmp_script_path(command)
+    upload_file(command, remote_path, script, run_as_user)
+    output, status = run_remote_script(remote_path, run_as_user)
+    output =~ /^(.*)\Z/
+    last_line = $1
+    # store output into a settings field, if requested
+    if settings_key
+      settings_key_portions = settings_key.split(".")
+      parent_key_portions, final_key = settings_key_portions[0..-2], settings_key_portions[-1]
+      target_settings_field = settings
+      parent_key_portions.each do |key_portion|
+        target_settings_field[key_portion] ||= {}
+        target_settings_field = target_settings_field[key_portion]
+      end
+      target_settings_field[final_key] = last_line.strip
+    end
+    status
+  rescue StandardError => e
+    logfile.puts e.message
+    false
+  end
+
+  # Upload a file (put a file into the remote server's filesystem)
+  def upload_file(command, remote_path, contents, upload_as_user=nil)
+    upload_as_user ||= default_username
+    run_remote_command("mkdir -p #{File.dirname(remote_path)}", upload_as_user)
+    Tempfile.open("remote_script") do |file|
+      file << contents
+      file.flush
+      logfile.puts "uploading #{remote_path} to Inception VM"
+      Net::SCP.upload!(host, upload_as_user, file.path, remote_path)
+    end
+    true
+  rescue StandardError => e
+    logfile.puts e.message
+    false
+  end
+
+  private
+  def remote_tmp_script_path(command)
+    "/tmp/remote_script_#{command.to_filename}"
+  end
+
+  # Makes +remote_path+ executable, then runs it
+  # Returns:
+  # * a String of all STDOUT/STDERR; which is also appended to +logfile+
+  # * status (true = success)
+  #
+  # TODO catch exceptions http://learnonthejob.blogspot.com/2010/08/exception-handling-for-netssh.html
+  def run_remote_script(remote_path, username)
+    commands = [
+      "chmod +x #{remote_path}",
+      "bash -lc 'sudo /usr/bin/env PATH=$PATH #{remote_path}'"
+    ]
+    script_output = ""
+    results = Fog::SSH.new(host, username).run(commands) do |stdout, stderr|
+      [stdout, stderr].flatten.each do |data|
+        logfile << data
+        script_output << data
+      end
+    end
+    result = results.last
+    result_success = result.status == 0
+    [script_output, result_success]
+  end
+
+  def run_remote_command(command, username)
+    Net::SSH.start(host, username) do |ssh|
+      ssh.exec!("bash -lc '#{command}'") do |channel, stream, data|
+        logfile << data
+      end
+    end
+  end
+end

data/lib/bosh-bootstrap/commander/upload_command.rb

@@ -0,0 +1,17 @@
+# A single command/script to be run on a local/remote server
+# For the display, it has an active ("installing") and 
+# past tense ("installed") verb and a noub/description ("packages")
+module Bosh::Bootstrap::Commander
+  class UploadCommand < Command
+    def initialize(target_path, file_contents)
+      super("upload", "file", "uploading file", "uploaded file")
+      @target_path = target_path
+      @file_contents = file_contents
+    end
+
+    # Invoke this command to call back upon +server.upload_file+ 
+    def perform(server)
+      server.upload_file(self, @target_path, @file_contents)
+    end
+  end
+end

data/lib/bosh-bootstrap/helpers.rb

@@ -0,0 +1,2 @@
+require "bosh-bootstrap/helpers/settings"
+require "bosh-bootstrap/helpers/fog_setup"

data/lib/bosh-bootstrap/helpers/fog_setup.rb

@@ -0,0 +1,50 @@
+# Copyright (c) 2012-2013 Stark & Wayne, LLC
+
+require "fog"
+module Bosh; module Bootstrap; module Helpers; end; end; end
+
+# A collection of methods related to getting fog_compute
+# credentials for creating the inception VM
+# and to provide to the MicroBOSH for its uses.
+#
+# Attempts to look in +settings.fog_path+ to see if there is a .fog file.
+module Bosh::Bootstrap::Helpers::FogSetup
+  
+  # fog connection object to Compute tasks (VMs, IP addresses)
+  def fog_compute
+    @fog_compute ||= begin
+      # Fog::Compute.new requires Hash with keys that are symbols
+      # but Settings converts all keys to strings
+      # So create a version of settings.fog_credentials with symbol keys
+      credentials_with_symbols = settings.fog_credentials.inject({}) do |creds, key_pair|
+        key, value = key_pair
+        creds[key.to_sym] = value
+        creds
+      end
+      Fog::Compute.new(credentials_with_symbols)
+    end
+  end
+
+  def reset_fog_compute
+    @fog_compute = nil
+    @provider = nil # in cli.rb - I don't like this; need one wrapper for all CPI/compute calls
+    # or don't create fog_compute until we know all IaaS details
+  end
+
+  def fog_config
+    @fog_config ||= begin
+      if File.exists?(fog_config_path)
+        say "Found infrastructure API credentials at #{fog_config_path} (override with --fog)"
+        YAML.load_file(fog_config_path)
+      else
+        say "No existing #{fog_config_path} fog configuration file", :yellow
+        {}
+      end
+    end
+  end
+
+  def fog_config_path
+    settings.fog_path
+  end
+
+end