bosh_agent 1.5.0.pre.1113

data/lib/bosh_agent/state.rb

@@ -0,0 +1,100 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+# This is a thin abstraction on top of a state.yml file that is managed by agent.
+# It can be used for getting the whole state as a hash or for querying
+# particular keys. Flushing the state to a file only happens on write call.
+
+# It aims to be threadsafe but doesn' use file locking so as long as everyone agrees on using the same
+# singleton of this class to read and write state it should be fine. However two agent processes using the same state file
+# could still be stepping on each other as well as two classes using different instances of this.
+
+module Bosh::Agent
+  class State
+
+    def initialize(state_file)
+      @state_file = state_file
+      @lock = Mutex.new
+      @data = nil
+      read
+    end
+
+    # Fetches the state from file (unless it's been already fetched)
+    # and returns the value of a given key.
+    # @param key Key that will be looked up in state hash
+    def [](key)
+      @lock.synchronize { @data[key] }
+    end
+
+    def to_hash
+      @lock.synchronize { @data.dup }
+    end
+
+    def ips
+      result = []
+      networks = self["networks"] || {}
+      return [] unless networks.kind_of?(Hash)
+
+      networks.each_pair do |name, network |
+        result << network["ip"] if network["ip"]
+      end
+
+      result
+    end
+
+    # Reads the current agent state from the state file and saves it internally.
+    # Empty file is fine but malformed file raises an exception.
+    def read
+      @lock.synchronize do
+        if File.exists?(@state_file)
+          state = Psych.load_file(@state_file) || default_state
+          unless state.kind_of?(Hash)
+            raise_format_error(state)
+          end
+          @data = state
+        else
+          @data = default_state
+        end
+      end
+
+      self
+    rescue SystemCallError => e
+      raise StateError, "Cannot read agent state file `#{@state_file}': #{e}"
+    rescue Psych::SyntaxError
+      raise StateError, "Malformed agent state: #{e}"
+    end
+
+    # Writes a new agent state into the state file.
+    # @param   new_state  Hash  New state
+    def write(new_state)
+      unless new_state.is_a?(Hash)
+        raise_format_error(new_state)
+      end
+
+      @lock.synchronize do
+        File.open(@state_file, "w") do |f|
+          f.puts(Psych.dump(new_state))
+        end
+        @data = new_state
+      end
+
+      true
+    rescue SystemCallError, Psych::SyntaxError => e
+      raise StateError, "Cannot write agent state file `#{@state_file}': #{e}"
+    end
+
+    private
+
+    def default_state
+      {
+        "deployment"    => "",
+        "networks"      => { },
+        "resource_pool" => { }
+      }
+    end
+
+    def raise_format_error(state)
+      raise StateError, "Unexpected agent state format: expected Hash, got #{state.class}"
+    end
+
+  end
+end

data/lib/bosh_agent/syslog_monitor.rb

@@ -0,0 +1,53 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+require 'eventmachine'
+require 'syslog_protocol'
+require 'uuidtools'
+
+module Bosh::Agent::SyslogMonitor
+
+  PORT = 33331
+
+  # severity level we publish in the nats alert
+  SEVERITY = 4
+
+  class Server < EventMachine::Connection
+    include EM::Protocols::LineText2
+
+    def initialize(nats, agent_id)
+      @nats = nats
+      @agent_id = agent_id
+    end
+
+    def receive_line(data)
+      parsed = SyslogProtocol.parse(data)
+
+      if parsed.content.end_with?('disconnected by user')
+        title = 'SSH Logout'
+      elsif parsed.content.include?('Accepted publickey for')
+        title = 'SSH Login'
+      else
+        return
+      end
+
+      json = Yajl::Encoder.encode(
+        {
+          'id' => UUIDTools::UUID.random_create,
+          'severity' => SEVERITY,
+          'title' => title,
+          'summary' => parsed.content,
+          'created_at' => Time.now.to_i
+        }
+      )
+      @nats.publish("hm.agent.alert.#{@agent_id}", json)
+    end
+  end
+
+  def self.start(nats, agent_id)
+    unless EM.reactor_running?
+      raise Error, 'Cannot start syslog monitor as event loop is not running'
+    end
+
+    EventMachine::start_server '127.0.0.1', PORT, Server, nats, agent_id
+  end
+end

data/lib/bosh_agent/template.rb

@@ -0,0 +1,50 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Agent
+
+  class Template
+
+    class TemplateDataError < StandardError; end
+
+    def self.write(&block)
+      self.new(block).write
+    end
+
+    def initialize(block)
+      raise ArgumentError unless block.arity == 1
+      @block = block
+      @block.call(self)
+      raise TemplateDataError if @template_data.nil?
+    end
+
+    def src(template_src)
+      case template_src
+      when String
+        if template_src.match(%r{\A/})
+          template_path = template_src
+        else
+          template_base_dir = File.dirname(__FILE__)
+          template_path = File.join(template_base_dir, template_src)
+        end
+        fh = File.new(template_path)
+      when IO, StringIO
+        fh = template_src
+      end
+      @template_data = fh.read
+    end
+
+    def dst(path)
+      @dst = path
+    end
+
+    def render
+      template = ERB.new(@template_data, 0, '%<>-')
+      template.result(@block.binding)
+    end
+
+    def write
+      File.open(@dst, 'w') { |fh| fh.write(self.render) }
+    end
+
+  end
+end

data/lib/bosh_agent/util.rb

@@ -0,0 +1,190 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Agent
+  class Util
+    include Bosh::Exec
+
+    class << self
+      def base_dir
+        Bosh::Agent::Config.base_dir
+      end
+
+      def logger
+        Bosh::Agent::Config.logger
+      end
+
+      def unpack_blob(blobstore_id, sha1, install_path)
+        bsc_options = Bosh::Agent::Config.blobstore_options
+        bsc_provider = Bosh::Agent::Config.blobstore_provider
+        blobstore_client = Bosh::Blobstore::Client.create(bsc_provider, bsc_options)
+
+        data_tmp = File.join(base_dir, 'data', 'tmp')
+        FileUtils.mkdir_p(data_tmp)
+
+        begin
+          tf = Tempfile.open(blobstore_id, data_tmp)
+          logger.info("Retrieving blob: #{blobstore_id}")
+
+          blobstore_client.get(blobstore_id, tf)
+          logger.info("Done retrieving blob")
+
+          tf.flush
+          blob_data_file = tf.path
+
+          logger.info("creating #{install_path}")
+          FileUtils.mkdir_p(install_path)
+
+          blob_sha1 = Digest::SHA1.file(blob_data_file).hexdigest
+          logger.info("hexdigest of #{blob_data_file}")
+
+          unless blob_sha1 == sha1
+            raise Bosh::Agent::MessageHandlerError,
+                  "Expected sha1: #{sha1}, Downloaded sha1: #{blob_sha1}, Blobstore ID: #{blobstore_id}, Install Path: #{install_path}"
+          end
+
+          logger.info("Installing to: #{install_path}")
+          Dir.chdir(install_path) do
+            output = `tar --no-same-owner -zxvf #{blob_data_file}`
+            raise Bosh::Agent::MessageHandlerError.new(
+              "Failed to unpack blob", output) unless $?.exitstatus == 0
+          end
+        rescue Exception => e
+          logger.info("Failure unpacking blob: #{e.inspect} #{e.backtrace}")
+          raise e
+        ensure
+          if tf
+            tf.close
+            tf.unlink
+          end
+        end
+
+      end
+
+      # @param [Hash] Instance spec
+      # @return [Binding] Template evaluation binding
+      def config_binding(spec)
+        Bosh::Common::TemplateEvaluationContext.new(spec).get_binding
+      end
+
+      def partition_disk(dev, sfdisk_input)
+        if File.blockdev?(dev)
+          sfdisk_cmd = "echo \"#{sfdisk_input}\" | sfdisk -uM #{dev}"
+          output = %x[#{sfdisk_cmd}]
+          unless $? == 0
+            logger.info("failed to parition #{dev}")
+            logger.info(output)
+          end
+        end
+      end
+
+      def lazy_itable_init_enabled?
+        File.exists?("/sys/fs/ext4/features/lazy_itable_init")
+      end
+
+      def block_device_size(block_device)
+        unless File.blockdev?(block_device)
+          raise Bosh::Agent::MessageHandlerError, "Not a blockdevice"
+        end
+
+        result = sh("/sbin/sfdisk -s #{block_device} 2>&1")
+        unless result.output.match(/\A\d+\Z/)
+          raise Bosh::Agent::MessageHandlerError,
+            "Unable to determine disk size"
+        end
+        result.output.to_i
+      end
+
+      def run_hook(hook, job_template_name)
+        hook_file = File.join(base_dir, 'jobs', job_template_name, 'bin', hook)
+
+        unless File.exists?(hook_file)
+          return nil
+        end
+
+        unless File.executable?(hook_file)
+          raise Bosh::Agent::MessageHandlerError, "`#{hook}' hook for `#{job_template_name}' job is not an executable file"
+        end
+
+        env = {
+          'PATH' => '/usr/sbin:/usr/bin:/sbin:/bin',
+        }
+
+        stdout_rd, stdout_wr = IO.pipe
+        stderr_rd, stderr_wr = IO.pipe
+        Process.spawn(env, hook_file, :out => stdout_wr, :err => stderr_wr, :unsetenv_others => true)
+        Process.wait
+        exit_status = $?.exitstatus
+        stdout_wr.close
+        stderr_wr.close
+        result = stdout_rd.read
+        error_output = stderr_rd.read
+
+        logger.info("Hook #{hook} for job #{job_template_name}: #{result}")
+
+        unless exit_status == 0
+          exception_message = "Hook #{hook} for #{job_template_name} failed "
+          exception_message += "(exit: #{exit_status}) "
+          exception_message += " stderr: #{error_output}, stdout: #{result}"
+          logger.info(exception_message)
+
+          raise Bosh::Agent::MessageHandlerError, exception_message
+        end
+        result
+      end
+
+      def create_symlink(src, dst)
+        # FileUtils doesn have 'no-deference' for links - causing ln_sf to
+        # attempt to create target link in dst rather than to overwrite it.
+        # BROKEN: FileUtils.ln_sf(src, dst)
+        out = %x(ln -nsf #{src} #{dst} 2>&1)
+        unless $?.exitstatus == 0
+          raise Bosh::Agent::MessageHandlerError,
+                "Failed to link '#{src}' to '#{dst}': #{out}"
+        end
+      end
+
+      # Poor mans idempotency
+      def update_file(data, path)
+        name = File.basename(path)
+        dir = File.dirname(path)
+
+        if_tmp_file = Tempfile.new(name, dir)
+        if_tmp_file.write(data)
+        if_tmp_file.flush
+
+        old = nil
+        begin
+          old = Digest::SHA1.file(path).hexdigest
+        rescue Errno::ENOENT
+          logger.debug("missing file: #{path}")
+        end
+        new = Digest::SHA1.file(if_tmp_file.path).hexdigest
+
+        updated = false
+        unless old == new
+          FileUtils.cp(if_tmp_file.path, path)
+          updated = true
+        end
+        updated
+      ensure
+        if if_tmp_file
+           if_tmp_file.close
+           FileUtils.rm_rf(if_tmp_file.path)
+        end
+      end
+
+      def get_network_info
+        sigar = SigarBox.create_sigar
+        net_info = sigar.net_info
+        ifconfig = sigar.net_interface_config(net_info.default_gateway_interface)
+
+        properties = {}
+        properties["ip"] = ifconfig.address
+        properties["netmask"] = ifconfig.netmask
+        properties["gateway"] = net_info.default_gateway
+        properties
+      end
+
+    end
+  end
+end

data/lib/bosh_agent/version.rb

@@ -0,0 +1,8 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh
+  module Agent
+    VERSION = '1.5.0.pre.1113'
+    BOSH_PROTOCOL = "1"
+  end
+end

data/lib/bosh_agent.rb

@@ -0,0 +1,92 @@
+module Bosh
+end
+
+require 'logger'
+require 'time'
+require 'yaml'
+require 'set'
+
+require 'nats/client'
+require 'yajl'
+require 'securerandom'
+require 'ostruct'
+require 'fileutils'
+require 'resolv'
+require 'ipaddr'
+require 'httpclient'
+require 'sigar'
+
+require 'common/exec'
+require 'common/properties'
+require 'bosh/core/encryption_handler'
+
+module Bosh::Agent
+  BOSH_APP = BOSH_APP_USER = BOSH_APP_GROUP = 'vcap'
+end
+
+require 'bosh_agent/ext'
+require 'bosh_agent/version'
+
+require 'bosh_agent/template'
+require 'bosh_agent/errors'
+require 'bosh_agent/remote_exception'
+
+require 'bosh_agent/sigar_box'
+require 'bosh_agent/config'
+require 'bosh_agent/util'
+require 'bosh_agent/monit'
+
+require 'bosh_agent/infrastructure'
+require 'bosh_agent/platform'
+
+require 'bosh_agent/platform/linux'
+require 'bosh_agent/platform/linux/adapter'
+require 'bosh_agent/platform/linux/disk'
+require 'bosh_agent/platform/linux/logrotate'
+require 'bosh_agent/platform/linux/password'
+require 'bosh_agent/platform/linux/network'
+
+require 'bosh_agent/platform/ubuntu'
+require 'bosh_agent/platform/ubuntu/network'
+
+require 'bosh_agent/platform/centos'
+require 'bosh_agent/platform/centos/disk'
+require 'bosh_agent/platform/centos/network'
+
+require 'bosh_agent/bootstrap'
+
+require 'bosh_agent/alert'
+require 'bosh_agent/alert_processor'
+require 'bosh_agent/smtp_server'
+require 'bosh_agent/heartbeat'
+require 'bosh_agent/heartbeat_processor'
+require 'bosh_agent/state'
+require 'bosh_agent/settings'
+require 'bosh_agent/file_matcher'
+require 'bosh_agent/file_aggregator'
+require 'bosh_agent/ntp'
+require 'bosh_agent/syslog_monitor'
+
+require 'bosh_agent/apply_plan/helpers'
+require 'bosh_agent/apply_plan/job'
+require 'bosh_agent/apply_plan/package'
+require 'bosh_agent/apply_plan/plan'
+
+require 'bosh_agent/disk_util'
+
+require 'bosh_agent/message/base'
+
+require 'bosh_agent/message/list_disk'
+require 'bosh_agent/message/migrate_disk'
+require 'bosh_agent/message/mount_disk'
+require 'bosh_agent/message/unmount_disk'
+
+require 'bosh_agent/message/state'
+require 'bosh_agent/message/drain'
+require 'bosh_agent/message/apply'
+require 'bosh_agent/message/compile_package'
+require 'bosh_agent/message/logs'
+require 'bosh_agent/message/ssh'
+
+require 'bosh_agent/handler'
+require 'bosh_agent/runner'