bosh_cli_plugin_micro 1.2089.0 → 1.2121.0

data/lib/bosh/cli/commands/micro.rb

@@ -52,7 +52,7 @@ module Bosh::Cli::Command
       if manifest['network'].blank?
         err 'network is not defined in deployment manifest'
       end
-      ip = deployer(manifest_filename).discover_bosh_ip || name
+      ip = deployer(manifest_filename).client_services_ip
 
       if target
         old_director_ip = URI.parse(target).host
@@ -298,7 +298,8 @@ AGENT_HELP
     usage 'micro apply'
     desc 'Apply spec'
     def apply(spec)
-      deployer.apply(Bosh::Deployer::Specification.new(load_yaml_file(spec)))
+      config = Bosh::Deployer::Config
+      deployer.apply(Bosh::Deployer::Specification.new(load_yaml_file(spec), config))
     end
 
     private
@@ -354,16 +355,12 @@ AGENT_HELP
 
     # rubocop:disable MethodLength
     def update_target
-      if deployer.exists?
-        bosh_ip = deployer.discover_bosh_ip
-        if URI.parse(target).host != bosh_ip
-          set_current(deployment)
-        end
-
-        director = Bosh::Cli::Client::Director.new(target)
+      set_target(deployer.client_services_ip)
 
+      if deployer.exists?
         if options[:director_checks]
           begin
+            director = Bosh::Cli::Client::Director.new(target)
             status = director.get_status
           rescue Bosh::Cli::AuthError
             status = {}

data/lib/bosh/deployer/configuration.rb

@@ -1,13 +1,17 @@
+require 'bosh/deployer/models/instance'
+require 'bosh/deployer/infrastructure_defaults'
+
 module Bosh::Deployer
   class Configuration
     attr_accessor :logger, :db, :uuid, :resources, :cloud_options,
-                  :spec_properties, :agent_properties, :bosh_ip, :env, :name, :net_conf
+                  :spec_properties, :agent_properties, :env, :name, :net_conf
+
+    attr_reader :base_dir
 
     # rubocop:disable MethodLength
     def configure(config)
       plugin = config['cloud']['plugin']
-
-      config = deep_merge(load_defaults(plugin), config)
+      config = InfrastructureDefaults.merge_for(plugin, config)
 
       @base_dir = config['dir']
       FileUtils.mkdir_p(@base_dir)
@@ -15,9 +19,9 @@ module Bosh::Deployer
       @name = config['name']
       @cloud_options = config['cloud']
       @net_conf = config['network']
-      @bosh_ip = @net_conf['ip']
       @resources = config['resources']
       @env = config['env']
+      @deployment_network = config['deployment_network']
 
       @logger = Logger.new(config['logging']['file'] || STDOUT)
       @logger.level = Logger.const_get(config['logging']['level'].upcase)
@@ -46,8 +50,7 @@ module Bosh::Deployer
       Sequel::Model.plugin :validation_helpers
 
       Bosh::Clouds::Config.configure(self)
-
-      require 'bosh/deployer/models/instance'
+      Models.define_instance_from_table(db[:instances])
 
       @cloud_options['properties']['agent']['mbus'] ||=
         'https://vcap:b00tstrap@0.0.0.0:6868'
@@ -55,6 +58,8 @@ module Bosh::Deployer
       @disk_model = nil
       @cloud = nil
       @networks = nil
+
+      self
     end
     # rubocop:enable MethodLength
 
@@ -66,26 +71,12 @@ module Bosh::Deployer
       @cloud
     end
 
-    def agent
-      uri = URI.parse(agent_url)
-      user, password = uri.userinfo.split(':', 2)
-      uri.userinfo = nil
-      uri.host = bosh_ip
-      Bosh::Agent::HTTPClient.new(uri.to_s, {
-        'user' => user,
-        'password' => password,
-        'reply_to' => uuid,
-      })
-    end
-
     def agent_url
       @cloud_options['properties']['agent']['mbus']
     end
 
     def networks
-      return @networks if @networks
-
-      @networks = {
+      @networks ||= {
         'bosh' => {
           'cloud_properties' => @net_conf['cloud_properties'],
           'netmask' => @net_conf['netmask'],
@@ -95,16 +86,7 @@ module Bosh::Deployer
           'type' => @net_conf['type'],
           'default' => %w(dns gateway)
         }
-      }
-      if @net_conf['vip']
-        @networks['vip'] = {
-          'ip' => @net_conf['vip'],
-          'type' => 'vip',
-          'cloud_properties' => {}
-        }
-      end
-
-      @networks
+      }.merge(vip_network).merge(deployment_network)
     end
 
     def task_checkpoint
@@ -114,8 +96,44 @@ module Bosh::Deployer
       # NoMethodError exceptions.
     end
 
+    def agent_services_ip
+      if net_conf['type'] == 'dynamic'
+        net_conf['vip']
+      elsif @deployment_network
+        @deployment_network['ip']
+      else
+        net_conf['ip']
+      end
+    end
+
+    def client_services_ip
+      net_conf['vip'] || net_conf['ip']
+    end
+
+    def internal_services_ip
+      '127.0.0.1'
+    end
+
     private
 
+    def vip_network
+      return {} unless @net_conf['vip']
+      {
+        'vip' => {
+          'ip' => @net_conf['vip'],
+          'type' => 'vip',
+          'cloud_properties' => {}
+        }
+      }
+    end
+
+    def deployment_network
+      return {} unless @deployment_network
+      {
+        'deployment' => @deployment_network
+      }
+    end
+
     def migrate_cpi
       cpi = @cloud_options['plugin']
       require_path = File.join('cloud', cpi)
@@ -127,24 +145,5 @@ module Bosh::Deployer
         Sequel::TimestampMigrator.new(@db, migrations, table: "#{cpi}_cpi_schema").run
       end
     end
-
-    def deep_merge(src, dst)
-      src.merge(dst) do |key, old, new|
-        if new.respond_to?(:blank) && new.blank?
-          old
-        elsif old.kind_of?(Hash) && new.kind_of?(Hash)
-          deep_merge(old, new)
-        elsif old.kind_of?(Array) && new.kind_of?(Array)
-          old.concat(new).uniq
-        else
-          new
-        end
-      end
-    end
-
-    def load_defaults(provider)
-      file = File.expand_path("../../../../config/#{provider}_defaults.yml", __FILE__)
-      Psych.load_file(file)
-    end
   end
 end

data/lib/bosh/deployer/infrastructure_defaults/aws.rb

@@ -0,0 +1,65 @@
+module Bosh::Deployer::InfrastructureDefaults
+  AWS = {
+    'name' => nil,
+    'logging' => {
+      'level' => 'INFO'
+    },
+    'dir' => nil,
+    'network' => {
+      'type' => 'dynamic',
+      'cloud_properties' => {}
+    },
+    'env' => {
+      'bosh' => {
+        'password' => nil
+      }
+    },
+    'resources' => {
+      'persistent_disk' => 4096,
+      'cloud_properties' => {
+        'instance_type' => 'm1.small',
+        'availability_zone' => nil
+      }
+    },
+    'cloud' => {
+      'plugin' => 'aws',
+      'properties' => {
+        'aws' => {
+          'access_key_id' => nil,
+          'secret_access_key' => nil,
+          'ec2_endpoint' => nil,
+          'max_retries' => 2,
+          'default_key_name' => nil,
+          'default_security_groups' => [],
+          'ssh_user' => 'vcap'
+        },
+        'registry' => {
+          'endpoint' => 'http://admin:admin@localhost:25888',
+          'user' => 'admin',
+          'password' => 'admin'
+        },
+        'stemcell' => {
+          'kernel_id' => nil,
+          'disk' => 4096
+        },
+        'agent' => {
+          'ntp' => [],
+          'blobstore' => {
+            'provider' => 'local',
+            'options' => {
+              'blobstore_path' => '/var/vcap/micro_bosh/data/cache'
+            }
+          },
+          'mbus' => nil
+        }
+      }
+    },
+    'apply_spec' => {
+      'properties' => {},
+      'agent' => {
+        'blobstore' => {},
+        'nats' => {}
+      }
+    }
+  }
+end

data/lib/bosh/deployer/infrastructure_defaults/openstack.rb

@@ -0,0 +1,63 @@
+module Bosh::Deployer::InfrastructureDefaults
+  OPENSTACK = {
+    'name' => nil,
+    'logging' => {
+      'level' => 'INFO'
+    },
+    'dir' => nil,
+    'network' => {
+      'type' => 'dynamic',
+      'cloud_properties' => {}
+    },
+    'env' => {
+      'bosh' => {
+        'password' => nil
+      }
+    },
+    'resources' => {
+      'persistent_disk' => 4096,
+      'cloud_properties' => {
+        'instance_type' => 'm1.small',
+        'availability_zone' => nil
+      }
+    },
+    'cloud' => {
+      'plugin' => 'openstack',
+      'properties' => {
+        'openstack' => {
+          'auth_url' => nil,
+          'username' => nil,
+          'api_key' => nil,
+          'tenant' => nil,
+          'region' => nil,
+          'endpoint_type' => 'publicURL',
+          'default_key_name' => nil,
+          'default_security_groups' => [],
+          'ssh_user' => 'vcap'
+        },
+        'registry' => {
+          'endpoint' => 'http://admin:admin@localhost:25889',
+          'user' => 'admin',
+          'password' => 'admin'
+        },
+        'agent' => {
+          'ntp' => [],
+          'blobstore' => {
+            'provider' => 'local',
+            'options' => {
+              'blobstore_path' => '/var/vcap/micro_bosh/data/cache'
+            }
+          },
+          'mbus' => nil
+        }
+      }
+    },
+    'apply_spec' => {
+      'properties' => {},
+      'agent' => {
+        'blobstore' => {},
+        'nats' => {}
+      }
+    }
+  }
+end

data/lib/bosh/deployer/infrastructure_defaults/vcloud.rb

@@ -0,0 +1,51 @@
+module Bosh::Deployer::InfrastructureDefaults
+  VCLOUD = {
+    'name' => nil,
+    'logging' => {
+      'level' => 'INFO'
+    },
+    'dir' => nil,
+    'network' => {
+      'dns' => [],
+      'cloud_properties' => {
+        'name' => nil
+      }
+    },
+    'env' => {
+      'bosh' => {
+        'password' => nil
+      }
+    },
+    'resources' => {
+      'persistent_disk' => 4096,
+      'cloud_properties' => {
+        'ram' => 1024,
+        'disk' => 4096,
+        'cpu' => 1
+      }
+    },
+    'cloud' => {
+      'plugin' => 'vcloud',
+      'properties' => {
+        'agent' => {
+          'ntp' => [],
+          'blobstore' => {
+            'provider' => 'local',
+            'options' => {
+              'blobstore_path' => '/var/vcap/micro_bosh/data/cache'
+            }
+          },
+          'mbus' => nil
+        },
+        'vcds' => []
+      }
+    },
+    'apply_spec' => {
+      'properties' => {},
+      'agent' => {
+        'blobstore' => {},
+        'nats' => {}
+      }
+    }
+  }
+end

data/lib/bosh/deployer/infrastructure_defaults/vsphere.rb

@@ -0,0 +1,54 @@
+module Bosh::Deployer::InfrastructureDefaults
+  VSPHERE = {
+    'name' => nil,
+    'logging' => {
+      'level' => 'INFO'
+    },
+    'dir' => nil,
+    'network' => {
+      'ip' => nil,
+      'netmask' => nil,
+      'gateway' => nil,
+      'dns' => [],
+      'cloud_properties' => {
+        'name' => nil
+      }
+    },
+    'env' => {
+      'bosh' => {
+        'password' => nil
+      }
+    },
+    'resources' => {
+      'persistent_disk' => 4096,
+      'cloud_properties' => {
+        'ram' => 1024,
+        'disk' => 4096,
+        'cpu' => 1
+      }
+    },
+    'cloud' => {
+      'plugin' => 'vsphere',
+      'properties' => {
+        'agent' => {
+          'ntp' => [],
+          'blobstore' => {
+            'provider' => 'local',
+            'options' => {
+              'blobstore_path' => '/var/vcap/micro_bosh/data/cache'
+            }
+          },
+          'mbus' => nil
+        },
+        'vcenters' => []
+      }
+    },
+    'apply_spec' => {
+      'properties' => {},
+      'agent' => {
+        'blobstore' => {},
+        'nats' => {}
+      }
+    }
+  }
+end

data/lib/bosh/deployer/infrastructure_defaults.rb

@@ -0,0 +1,39 @@
+require 'bosh/deployer/infrastructure_defaults/aws'
+require 'bosh/deployer/infrastructure_defaults/openstack'
+require 'bosh/deployer/infrastructure_defaults/vcloud'
+require 'bosh/deployer/infrastructure_defaults/vsphere'
+
+module Bosh::Deployer::InfrastructureDefaults
+  def self.merge_for(plugin, config)
+    case plugin
+      when 'aws'
+        defaults = AWS
+      when 'openstack'
+        defaults = OPENSTACK
+      when 'vcloud'
+        defaults = VCLOUD
+      when 'vsphere'
+        defaults = VSPHERE
+      else
+        raise "Infrastructure '#{plugin}' not found"
+    end
+    deep_merge(defaults, config)
+  end
+
+  private
+
+  def self.deep_merge(src, dst)
+    src.merge(dst) do |key, old, new|
+      if new.respond_to?(:blank) && new.blank?
+        old
+      elsif old.kind_of?(Hash) && new.kind_of?(Hash)
+        deep_merge(old, new)
+      elsif old.kind_of?(Array) && new.kind_of?(Array)
+        old.concat(new).uniq
+      else
+        new
+      end
+    end
+  end
+end
+

data/lib/bosh/deployer/instance_manager/aws.rb

@@ -5,10 +5,11 @@ require 'bosh/deployer/ssh_server'
 module Bosh::Deployer
   class InstanceManager
     class Aws
-      def initialize(instance_manager, logger)
+      def initialize(instance_manager, config, logger)
         @instance_manager = instance_manager
         @logger = logger
-        properties = Config.cloud_options['properties']
+        @config = config
+        properties = config.cloud_options['properties']
 
         @registry = Registry.new(
           properties['registry']['endpoint'],
@@ -25,7 +26,7 @@ module Bosh::Deployer
       end
 
       def remote_tunnel
-        @remote_tunnel.create(instance_manager.bosh_ip, registry.port)
+        @remote_tunnel.create(instance_manager.client_services_ip, registry.port)
       end
 
       def disk_model
@@ -40,11 +41,11 @@ module Bosh::Deployer
         # and if it doesn't exist, use the bosh deployer
         # aws properties (cloud.properties.aws)
         properties['aws'] =
-          Config.spec_properties['aws'] ||
-          Config.cloud_options['properties']['aws'].dup
+          config.spec_properties['aws'] ||
+            config.cloud_options['properties']['aws'].dup
 
-        properties['aws']['registry'] = Config.cloud_options['properties']['registry']
-        properties['aws']['stemcell'] = Config.cloud_options['properties']['stemcell']
+        properties['aws']['registry'] = config.cloud_options['properties']['registry']
+        properties['aws']['stemcell'] = config.cloud_options['properties']['stemcell']
 
         spec.delete('networks')
       end
@@ -62,29 +63,16 @@ module Bosh::Deployer
         instance_manager.save_state
       end
 
-      def discover_bosh_ip
-        if instance_manager.state.vm_cid
-          # choose elastic IP over public, as any agent connecting to the
-          # deployed micro bosh will be cut off from the public IP when
-          # we re-deploy micro bosh
-          instance = instance_manager.cloud.ec2.instances[instance_manager.state.vm_cid]
-          if instance.has_elastic_ip?
-            ip = instance.elastic_ip.public_ip
-          else
-            ip = instance.public_ip_address
-          end
-
-          if ip && ip != instance_manager.bosh_ip
-            instance_manager.bosh_ip = ip
-            logger.info("discovered bosh ip=#{instance_manager.bosh_ip}")
-          end
-        end
+      def client_services_ip
+        discover_client_services_ip
+      end
 
-        instance_manager.bosh_ip
+      def agent_services_ip
+        discover_client_services_ip
       end
 
-      def service_ip
-        instance_manager.cloud.ec2.instances[instance_manager.state.vm_cid].private_ip_address
+      def internal_services_ip
+        config.internal_services_ip
       end
 
       # @return [Integer] size in MiB
@@ -98,13 +86,13 @@ module Bosh::Deployer
         # is a risk of conversion errors which lead to an unnecessary
         # disk migration, so we need to do a double conversion
         # here to avoid that
-        requested = (Config.resources['persistent_disk'] / 1024.0).ceil * 1024
+        requested = (config.resources['persistent_disk'] / 1024.0).ceil * 1024
         requested != disk_size(instance_manager.state.disk_cid)
       end
 
       private
 
-      attr_reader :registry, :instance_manager, :logger
+      attr_reader :registry, :instance_manager, :logger, :config
 
       def ssh_properties(properties)
         ssh_user = properties['aws']['ssh_user']
@@ -113,6 +101,7 @@ module Bosh::Deployer
 
         key = properties['aws']['ec2_private_key']
         err 'Missing properties.aws.ec2_private_key' unless key
+
         ssh_key = File.expand_path(key)
         unless File.exists?(ssh_key)
           err "properties.aws.ec2_private_key '#{key}' does not exist"
@@ -120,6 +109,25 @@ module Bosh::Deployer
 
         [ssh_key, ssh_port, ssh_user, ssh_wait]
       end
+
+      def discover_client_services_ip
+        if instance_manager.state.vm_cid
+          # choose elastic IP over public, as any agent connecting to the
+          # deployed micro bosh will be cut off from the public IP when
+          # we re-deploy micro bosh
+          instance = instance_manager.cloud.ec2.instances[instance_manager.state.vm_cid]
+          if instance.has_elastic_ip?
+            ip = instance.elastic_ip.public_ip
+          else
+            ip = instance.public_ip_address
+          end
+
+          logger.info("discovered bosh ip=#{ip}")
+          ip
+        else
+          config.client_services_ip
+        end
+      end
     end
   end
 end