fhcap-cli 0.3.0

data/lib/fhcap/tasks/chef/server/provision.rb

@@ -0,0 +1,45 @@
+require 'fhcap/tasks/chef/chef_task_base'
+
+module Fhcap
+  module Tasks
+    module Chef
+      module Server
+        class Provision < ChefTaskBase
+
+          attr_reader :environments, :node_names, :roles
+
+          def initialize(options)
+            super
+            @environments = options[:environments]
+            @node_names = options[:nodes]
+            @roles = options[:roles]
+            @chef_server = options[:chef_server]
+          end
+
+          def run
+            thor.say "Chef::Server::Provision: environments = #{environments}", :yellow
+
+            chef_server_environments = {}
+            environments.each do |environment|
+              chef_server = select_chef_server(environment, @chef_server)
+              chef_server_environments[chef_server] = chef_server_environments[chef_server] || []
+              chef_server_environments[chef_server] << environment
+            end
+
+            chef_server_environments.each do |server, environments|
+              environments.each do |environment|
+                query = ["chef_environment:#{environment}"]
+                query << "AND name:#{node_names}" if node_names
+                query << "AND roles:#{roles}" if roles
+                query = query.join(' ')
+                cmd = "sudo chef-client"
+                run_knife_ssh_cmd(query, cmd, server, options)
+              end
+            end
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/fhcap/tasks/clean.rb

@@ -0,0 +1,34 @@
+require 'fhcap/tasks/task_base'
+
+module Fhcap
+  module Tasks
+    class Clean < TaskBase
+
+      def initialize(options)
+        super
+      end
+
+      def run
+        if config.exists?
+          repos = config[:repos]
+          delete_files = repos.collect do |name, cfg|
+            repo_dir(name)
+          end.compact
+          delete_files << config.config_file
+
+          delete_files.each do |file|
+            thor.say_status 'removing', file, :yellow
+          end
+          if thor.yes? "Continue (y/n)"
+            delete_files.each do |file|
+              thor.remove_file file
+            end
+          end
+        else
+          thor.say_status 'error', "No config file found #{config.config_file}", :red
+        end
+      end
+
+    end
+  end
+end

data/lib/fhcap/tasks/cluster/cluster_task_base.rb

@@ -0,0 +1,57 @@
+require 'fhcap/tasks/task_base'
+
+module Fhcap
+  module Tasks
+    module Cluster
+      class ClusterTaskBase < TaskBase
+
+        attr_reader :name, :cluster_config, :cluster_file
+
+        def initialize(options, exit_on_error=true)
+          super(options)
+          @name = options[:name]
+          @cluster_config = options[:'cluster-config']
+
+          unless @cluster_config
+            @cluster_file = find_cluster(@name)
+            if @cluster_file
+              @cluster_config = JSON.parse(IO.read(@cluster_file), {:symbolize_names => true})
+            else
+              exit_with_error("Unknown cluster #{@name}") if exit_on_error
+              @cluster_config = {}
+            end
+          end
+        end
+
+        def template_as_object(template_file, config)
+          template = File.read(template_file)
+          eruby = Erubis::Eruby.new(template)
+          result = eruby.result({config: config})
+          JSON.parse(result, {:symbolize_names => true})
+        end
+
+        def checkout_repos
+          repos = ['fhcap', cluster_config[:repo]].compact.uniq
+          repos.each do |name|
+            name = name.to_s
+            if @options[:'git-ref']
+              fallback_git_ref = name == 'fhcap' ? nil : 'master'
+            else
+              fallback_git_ref = nil
+            end
+            Repo::Checkout.new(@options.dup.merge({:repo => name, :'fallback-git-ref' => fallback_git_ref})).run
+          end if repos
+        end
+
+        private
+
+        def cluster_environments
+          @cluster_config[:environments].collect do |env_name, cfg|
+            [@name.to_s, env_name.to_s].uniq.join('-')
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/fhcap/tasks/cluster/create.rb

@@ -0,0 +1,243 @@
+require 'fhcap/tasks/cluster/cluster_task_base'
+require 'fhcap/tasks/cluster/create_environment'
+require 'fhcap/tasks/chef/provisioning/create'
+require 'fhcap/tasks/chef/environments/promote_cookbooks'
+require 'fhcap/tasks/chef/environments/create'
+require 'fhcap/tasks/chef/server/bootstrap'
+require 'fhcap/tasks/repo/checkout'
+require "fhcap/tasks/misc/create_dns_record"
+
+module Fhcap
+  module Tasks
+    module Cluster
+      class Create < ClusterTaskBase
+
+        attr_reader :name, :cluster_config
+
+        def initialize(options)
+          super(options, false)
+          @template = options[:template]
+          @repo = options[:'repo']
+          @provider_config = options[:'provider-config'] || {}
+          @gitref = options[:'git-ref']
+          @domain = options[:domain]
+          @ssl_cert = options[:'ssl-cert']
+          @dns_provider_id = options[:'dns-provider-id']
+          @chef_server = options[:'chef-server']
+          @provider_id = options[:'provider-id']
+          @skip_repo_checkout = options[:'skip-repo-checkout']
+          @skip_cookbook_promote = options[:'skip-cookbook-promote']
+          @skip_server_bootstrap = options[:'skip-server-bootstrap']
+          @skip_dns_record = options[:'skip-dns-record']
+          @skip_create = options[:'skip-create']
+
+          @cluster_config = {
+              id: @name,
+              template: @template,
+              driver: "",
+              domain: @domain,
+              repo: @repo,
+              chef_server: @chef_server,
+              provider_id: @provider_id
+          }.merge(@cluster_config)
+
+          @cluster_config[:provider_config] = @cluster_config[:provider_config] || {}
+          @cluster_config[:provider_config].merge!(@provider_config)
+        end
+
+        def run
+          thor.say "Cluster::Create: name = #{@name}", :yellow
+
+          #Checkout repos
+          checkout_repos unless @skip_repo_checkout
+
+          #Generate cluster config
+          generate_cluster_config
+
+          #Create environment files
+          create_cluster_environments
+
+          #Write config to disk
+          cluster_file = find_cluster(name) || File.join(repo_dir(cluster_config[:repo]), repo_clusters_dir(cluster_config[:repo]), "#{name}.json")
+          thor.create_file(cluster_file, JSON.pretty_generate(cluster_config))
+
+          #Chef Stuff
+          #Promote cookbooks
+          Chef::Environments::PromoteCookbooks.new(@options.dup.merge({:environments => cluster_environments})).run unless @skip_cookbook_promote
+
+          #Bootstrap Server
+          Chef::Server::Bootstrap.new(@options.dup.merge({:environments => cluster_environments, :chef_server => @cluster_config[:chef_server]})).run unless @skip_server_bootstrap
+
+          #Provisioning create
+          Chef::Provisioning::Create.new(options).run unless @skip_create
+
+          create_dns_record unless @skip_dns_record
+        end
+
+        private
+
+        def required_config
+          {
+              repo: {
+                  msg: "In which repo should this clusters config and environment files be stored?",
+                  options: {:limited_to => repo_names}
+              },
+              domain: {
+              },
+              chef_server: {
+                  options: {:limited_to => chef_server_names}
+              },
+              provider_id: {
+                  options: {:limited_to => provider_names_for('compute')}
+              },
+              template: {
+                  options: {:limited_to => cluster_template_names}
+              }
+          }
+        end
+
+        def aws_required_config
+          aws_regions = provider_config(@cluster_config[:provider_id])[:regions]
+          {
+              region: {
+                  options: {:limited_to => aws_regions.keys.collect { |k| k.to_s }}
+              },
+              cidr: {}
+          }
+        end
+
+        def generate_cluster_config
+          ask_config(required_config, cluster_config)
+          cluster_config[:driver] = provider_type(cluster_config[:provider_id])
+          send(:"generate_cluster_config_#{cluster_config[:driver]}")
+          driver_template_file = File.join('templates', 'cluster', cluster_config[:driver], "common.json.erb")
+          driver_template_config = template_as_object(driver_template_file, cluster_config)
+          cluster_config.merge!(driver_template_config)
+        end
+
+        def generate_cluster_config_aws
+          ask_config(aws_required_config, cluster_config[:provider_config])
+          aws_regions = provider_config(cluster_config[:provider_id])[:regions]
+          cluster_config[:default_instance_options] = cluster_config[:default_instance_options] || {}
+          cluster_config[:default_instance_options][:image_id] = aws_regions[cluster_config[:provider_config][:region].to_sym][:base_image]
+        end
+
+        def generate_cluster_config_openstack
+          cluster_config[:default_instance_options] = cluster_config[:default_instance_options] || {}
+          cluster_config[:default_instance_options][:image_ref] = provider_config(@cluster_config[:provider_id])[:image_ref]
+          cluster_config[:default_instance_options][:flavor_ref] = provider_config(@cluster_config[:provider_id])[:flavor_ref]
+          cluster_config[:default_instance_options][:floating_ip_pool] = provider_config(@cluster_config[:provider_id])[:floating_ip_pool]
+          cluster_config[:default_instance_options][:ssh_username] = provider_config(@cluster_config[:provider_id])[:ssh_username]
+        end
+
+        def create_cluster_environments
+          env_template_file = File.join('templates', 'cluster', "#{cluster_config[:template]}.json.erb")
+          template_config = template_as_object(env_template_file, cluster_config)
+
+          template_config[:environments].each do |env|
+            env_template_file = File.join('templates', 'cluster', cluster_config[:driver], "#{env[:template]}.json.erb")
+            template_config = template_as_object(env_template_file, cluster_config)
+
+            env_domain = [env[:'domain-prefix'], cluster_config[:domain]].compact.join('.')
+            env_cfg = {
+                :name => name,
+                :'environment-name' => env[:name],
+                :domain => env_domain,
+                :template => env[:template],
+                :'cluster-config' => cluster_config,
+                :'skip-create-cluster-file' => true
+            }
+
+            CreateEnvironment.new(@options.dup.merge(env_cfg)).run
+          end
+        end
+
+        def create_dns_record
+          send(:"create_dns_record_#{cluster_config[:driver]}")
+        end
+
+        def create_dns_record_aws
+          cluster_config[:environments].each do |env_name, env_cfg|
+            env_cfg[:load_balancers].each do |lb_name, lb_cfg|
+              if lb_cfg[:scheme] == "internet-facing"
+                begin
+                  require 'aws-sdk'
+
+                  cfg = provider_config(cluster_config[:provider_id])
+                  creds = cfg[:credentials]
+
+                  elb = Aws::ElasticLoadBalancing::Client.new(
+                      region: cluster_config[:provider_config][:region],
+                      access_key_id: creds[:'aws-access-key'],
+                      secret_access_key: creds[:'aws-secret-key']
+                  )
+
+                  lb = load_balancer_name_for(name, env_name, lb_name)
+                  resp = elb.describe_load_balancers({
+                                                         load_balancer_names: [lb],
+                                                         page_size: 1,
+                                                     })
+
+                  elb = resp.load_balancer_descriptions.first
+                  thor.say "Found ELB name: #{elb.load_balancer_name}, dns_name: #{elb.dns_name}, zone_id: #{elb.canonical_hosted_zone_name_id}"
+
+                  dns_record_cfg = {
+                      domain: "*.#{env_cfg[:domain]}",
+                      :"alias-target" => {
+                          dns_name: elb.dns_name,
+                          hosted_zone_id: elb.canonical_hosted_zone_name_id
+                      }
+                  }
+                  Misc::CreateDNSRecord.new(@options.dup.merge(dns_record_cfg)).run
+                rescue Aws::ElasticLoadBalancing::Errors::LoadBalancerNotFound => e
+                  thor.say_status 'error', "LoadBalancer #{lb} not found, unable to create DNS Entry", :red
+                rescue Aws::ElasticLoadBalancing::Errors::ServiceError => e
+                  thor.say_status 'error', e.message, :red
+                end
+              end
+            end if env_cfg[:load_balancers]
+          end
+        end
+
+        def create_dns_record_openstack
+          cluster_config[:environments].each do |env_name, env_cfg|
+            env_name = "#{name}-#{env_name}"
+            knife_config_file = knife_config_file_for(cluster_config[:chef_server])
+            nodes = JSON.parse(`knife search "chef_environment:#{env_name} AND recipes:nginx_feedhenry\\:\\:loadbalancer" -c #{knife_config_file} -F json -a name -a cloud.public_ipv4`)
+
+            lb_node = nodes['rows'].collect do |row|
+              name, attrs = row.first
+              attrs
+            end.first
+
+            if lb_node
+              if lb_node['cloud.public_ipv4']
+                dns_record_cfg = {
+                    domain: "*.#{env_cfg[:domain]}",
+                    ipaddress: lb_node['cloud.public_ipv4']
+                }
+                Misc::CreateDNSRecord.new(@options.dup.merge(dns_record_cfg)).run
+              else
+                thor.say "Found lb node '#{lb_node['name']}', but was unable to retrieve it's IP!!}"
+              end
+            else
+              thor.say "Unable to locate lb node in cluster!!"
+            end
+          end
+        end
+
+        def load_balancer_name_for(org_name, environment, lb_name)
+          fh_name_for('fh-lb', org_name, environment, lb_name)
+        end
+
+        def fh_name_for(*args)
+          args.collect do |str|
+            str.to_s.split('-')
+          end.flatten.uniq.join('-')
+        end
+
+
+      end
+    end
+  end
+end

data/lib/fhcap/tasks/cluster/create_environment.rb

@@ -0,0 +1,171 @@
+require 'fhcap/tasks/cluster/cluster_task_base'
+require 'fhcap/tasks/chef/environments/create'
+require "fhcap/tasks/misc/create_ssl_cert"
+
+module Fhcap
+  module Tasks
+    module Cluster
+      class CreateEnvironment < ClusterTaskBase
+
+        attr_reader :template, :environment_name, :domain, :environment_config
+
+        def initialize(options)
+          super(options)
+          @skip_create_cluster_file = options[:'skip-create-cluster-file']
+          @template = options[:template]
+          @environment_name = options[:'environment-name']
+          @domain = options[:domain]
+
+          @cluster_config[:environments] = @cluster_config[:environments] || {}
+          @environment_config = @cluster_config[:environments][@environment_name.to_sym] || {}
+
+          template_filepath = File.join(Fhcap.source_root, 'templates', 'cluster', @cluster_config[:driver], "#{template}.json.erb")
+          eruby = Erubis::Eruby.new(File.read(template_filepath))
+          result = eruby.result(
+              {
+                  config: {
+                      domain: @domain,
+                      environment_name: @environment_name
+                  }
+              }
+          )
+          @environment_config.merge!(JSON.parse(result, {:symbolize_names => true}))
+
+          @availability_zones = provider_availability_zones(@cluster_config[:provider_id], @cluster_config[:provider_config][:region])
+          @cidr = @cluster_config[:provider_config][:cidr]
+          @subnets = {}
+          @allocated_ips = {}
+          @next_subnet = 0
+        end
+
+        def run
+          thor.say "Cluster::CreateEnvironment: cluster = #{name}, environment = #{environment_name}, domain = #{domain}", :yellow
+          env_cfg = {
+              name: "#{name}-#{environment_name}",
+              domain: domain,
+              type: get_env_type(environment_name),
+              repo: cluster_config[:repo]
+          }
+          generate_environment_config
+          create_chef_environment(env_cfg)
+          create_ssl_cert(env_cfg)
+          thor.create_file(cluster_file, JSON.pretty_generate(cluster_config)) unless @skip_create_cluster_file
+        end
+
+
+        private
+
+        def generate_environment_config
+          environment_config[:subnets].each do |subnet_id, subnet_cfg|
+            subnet_cfg[:cidr] = get_subnet_cidr(subnet_id, subnet_cfg[:cidr])
+            subnet_cfg[:availability_zone] = get_availability_zone(subnet_id)
+          end if environment_config[:subnets]
+
+          environment_config[:instances].each do |instance_id, instance_cfg|
+            if instance_cfg[:aws]
+              instance_cfg[:aws][:private_ip_address] = get_subnet_private_ip(instance_cfg[:aws][:subnet]) if instance_cfg[:aws].has_key?(:private_ip_address)
+            end
+          end if environment_config[:instances]
+
+          if environment_config[:security_groups]
+            cluster_config[:security_groups] = cluster_config[:security_groups] || {}
+
+            environment_config[:security_groups].each do |name, cfg|
+              cluster_config[:security_groups][name.to_sym] = cfg
+            end
+            environment_config.delete(:security_groups)
+          end
+
+          cluster_config[:environments][environment_name.to_sym] = cluster_config[:environments][environment_name.to_sym] || {}
+          cluster_config[:environments][environment_name.to_sym].merge!(environment_config)
+        end
+
+        def create_chef_environment(env_cfg)
+          Chef::Environments::Create.new(@options.dup.merge(env_cfg)).run
+        end
+
+        def create_ssl_cert(env_cfg)
+          if provider_type(cluster_config[:provider_id]) == 'aws'
+            cert_format = 'pem'
+            cert_directory = File.join(repo_dir(cluster_config[:repo]), repo_clusters_dir(cluster_config[:repo]), 'ssl_certs')
+          else
+            cert_format = 'json'
+            cert_directory = File.join(repo_dir(cluster_config[:repo]), 'data_bags', 'nginx_ssl_certs')
+          end
+          cert_name = "wildcard.#{env_cfg[:domain]}"
+          cert_domain = "*.#{env_cfg[:domain]}"
+          domain_cert_cfg = {
+              name: cert_name,
+              domain: cert_domain,
+              format: cert_format,
+              directory: cert_directory
+          }
+          cluster_config[:environments].each do |env_name, env_cfg|
+            env_cfg[:load_balancers].each do |lb_name, lb_cfg|
+              lb_cfg[:listeners].each do |listener|
+                if listener[:server_certificate]
+                  listener[:server_certificate] = cert_name
+                end
+              end
+            end if env_cfg[:load_balancers]
+          end
+          Misc::CreateSslCert.new(@options.dup.merge(domain_cert_cfg)).run
+        end
+
+        def get_subnet(subnet)
+          subnet = subnet.to_sym
+          unless @subnets[subnet]
+            @subnets[subnet] = {}
+            @allocated_ips[subnet] = []
+          end
+          @subnets[subnet]
+        end
+
+        def get_subnet_cidr(subnet, range)
+          subnet = subnet.to_sym
+          unless get_subnet(subnet)[:cidr]
+            @subnets[subnet][:cidr] = "#{@cidr.split('.')[0..2].join('.')}.#{range}"
+          end
+          @subnets[subnet][:cidr]
+        end
+
+        def get_availability_zone(subnet)
+          subnet = subnet.to_sym
+          unless get_subnet(subnet)[:avilability_zone]
+            @subnets[subnet][:avilability_zone] = @availability_zones[@next_subnet]
+            @next_subnet = @next_subnet < (@availability_zones.length - 1) ? @next_subnet + 1 : 0
+          end
+          @subnets[subnet][:avilability_zone]
+        end
+
+        def get_subnet_private_ip(subnet)
+          subnet = subnet.to_sym
+          require 'ipaddr'
+          range = IPAddr.new(@subnets[subnet][:cidr]).to_range
+          private_ip = nil
+          range.each do |ip|
+            next if @allocated_ips[subnet].include? ip
+            private_ip = ip
+            @allocated_ips[subnet] << private_ip
+            break
+          end
+          private_ip.to_s
+        end
+
+        def get_env_type(name)
+          case name
+            when /core/
+              'core'
+            when /mbaas/
+              'mbaas'
+            when /single/
+              'single'
+            else
+              'empty'
+          end
+        end
+
+      end
+    end
+  end
+end