fhcap-cli 0.3.0

data/lib/cookbooks/provision/recipes/aws_cluster_create.rb

@@ -0,0 +1,59 @@
+include_recipe 'provision::common'
+include_recipe "provision::aws"
+
+cluster_config = cluster_config_for(node)
+org_name = cluster_config[:id]
+
+key_pair_name = key_pair_name_for(org_name)
+aws_key_pair key_pair_name
+
+#aws_key_pair 'my-aws-key' do
+#  private_key_path "~boiardi/.ssh/my-aws-key.pem"
+#  public_key_path "~boiardi/.ssh/my-aws-key.pub"
+#  allow_overwrite false # Set to true if you want to regenerate this each chef run
+#end
+
+#Create VPC
+vpc_name = vpc_name_for(org_name)
+
+aws_vpc vpc_name do
+  cidr_block cluster_config[:vpc][:cidr]
+  internet_gateway true
+  main_routes '0.0.0.0/0' => :internet_gateway
+end
+
+# Create Security Groups
+cluster_config[:security_groups].each do |sg_name, sg_config|
+  aws_security_group security_group_name_for(org_name, sg_name) do
+    vpc vpc_name
+    inbound_rules inbound_rules_for(org_name, sg_config[:authorize_ingress])
+  end
+end
+
+# Create Route Tables
+cluster_config[:environments].each do |env, env_config|
+
+  env_route_table = route_table_name_for(org_name, env)
+  aws_route_table env_route_table do
+    vpc vpc_name
+    routes '0.0.0.0/0' => :internet_gateway
+  end
+
+  env_config[:subnets].each do |subnet_suffix, subnet_config|
+
+    subnet_name = subnet_name_for(org_name, env, subnet_suffix)
+    aws_subnet subnet_name do
+      vpc vpc_name
+      cidr_block subnet_config[:cidr]
+      availability_zone subnet_config[:availability_zone]
+      route_table env_route_table
+      map_public_ip_on_launch true
+    end
+
+  end
+
+end
+
+include_recipe "provision::cluster_provision_instances"
+
+include_recipe "provision::aws_cluster_create_elb"

data/lib/cookbooks/provision/recipes/aws_cluster_create_elb.rb

@@ -0,0 +1,61 @@
+include_recipe 'provision::common'
+include_recipe "provision::aws"
+
+cluster_config = cluster_config_for(node)
+org_name = cluster_config[:id]
+
+# Create Load Balancers
+cluster_config[:environments].each do |env, env_config|
+
+  env_config[:load_balancers].each do |lb_name, lb_config|
+
+    lb_config[:security_groups] = lb_config[:security_groups].collect do |security_group|
+      security_group_name_for(org_name, security_group)
+    end
+
+    lb_config[:subnets] = lb_config[:subnets].collect do |subnet_name|
+      subnet_name_for(org_name, env, subnet_name)
+    end
+
+    lb_certs = {}
+    lb_config[:listeners].each_with_index do |listener, index|
+      if listener[:server_certificate]
+        certificate_file = File.join(node[:local_repo_path], 'clusters', 'ssl_certs', "#{listener[:server_certificate]}-certificate.pem")
+        private_key_file = File.join(node[:local_repo_path], 'clusters', 'ssl_certs', "#{listener[:server_certificate]}-private-key.pem")
+
+        if File.exists?(certificate_file) && File.exists?(private_key_file)
+          certificate_body = File.open(certificate_file, "rb").read
+          private_key = File.open(private_key_file, "rb").read
+
+          cert_name = listener[:server_certificate]
+          lb_certs[index] = aws_server_certificate cert_name do
+            certificate_body certificate_body
+            private_key private_key
+          end
+        else
+          Chef::Log.warn("Could not find #{listener[:server_certificate]} locally to upload. Is this certificate already registered? ELB create will fail if it does not exist!!")
+        end
+      end
+    end
+
+    lb_machines = env_config[:instances].select do |instance_name, instance_config|
+      instance_config[:load_balancers] && instance_config[:load_balancers].include?(lb_name.to_s)
+    end.collect do |instance_name, instance_config|
+      instance_name_for(org_name, env, instance_name)
+    end
+
+    lb_name = load_balancer_name_for(org_name, env, lb_name)
+    load_balancer lb_name do
+      machines lb_machines
+      load_balancer_options(
+          lazy do
+            lb_certs.each do |index, cert|
+              lb_config[:listeners][index][:server_certificate] = cert.aws_object.arn
+            end
+            lb_config
+          end
+      )
+    end
+
+  end if env_config[:load_balancers]
+end

data/lib/cookbooks/provision/recipes/aws_cluster_destroy.rb

@@ -0,0 +1,52 @@
+include_recipe 'provision::common'
+include_recipe "provision::aws"
+
+cluster_config = cluster_config_for(node)
+
+org_name = cluster_config[:id]
+
+node_names = node_names_for(cluster_config)
+
+machine_batch do
+  machines node_names
+  action :destroy
+end
+
+# Clear all rules from Security Groups
+cluster_config[:security_groups].each do |sg_name, sg_config|
+  aws_security_group security_group_name_for(org_name, sg_name) do
+    inbound_rules []
+  end
+end
+
+vpc_name = vpc_name_for(org_name)
+aws_vpc vpc_name do
+  action :purge
+end
+
+cluster_config[:environments].each do |env, env_config|
+
+  aws_route_table route_table_name_for(org_name, env) do
+    action :destroy
+  end
+
+  env_config[:subnets].each do |subnet_suffix, subnet_config|
+    aws_subnet  subnet_name_for(org_name, env, subnet_suffix) do
+      action :destroy
+    end
+  end
+
+end
+
+cluster_config[:environments].each do |env, env_config|
+  env_config[:load_balancers].each do |lb_name, lb_config|
+    lb_name = load_balancer_name_for(org_name, env, lb_name)
+    load_balancer lb_name do
+      action :destroy
+    end
+  end if env_config[:load_balancers]
+end
+
+aws_key_pair key_pair_name_for(org_name) do
+  action :destroy
+end

data/lib/cookbooks/provision/recipes/cluster_create.rb

@@ -0,0 +1,2 @@
+
+include_recipe "provision::#{node['driver']}_cluster_create"

data/lib/cookbooks/provision/recipes/cluster_destroy.rb

@@ -0,0 +1,2 @@
+
+include_recipe "provision::#{node['driver']}_cluster_destroy"

data/lib/cookbooks/provision/recipes/cluster_destroy_instances.rb

@@ -0,0 +1,11 @@
+include_recipe "provision::common"
+include_recipe "provision::#{node['driver']}"
+
+cluster_config = cluster_config_for(node)
+org_name = cluster_config[:id]
+node_names = node_names_for(cluster_config)
+
+machine_batch do
+  machines node_names
+  action :destroy
+end

data/lib/cookbooks/provision/recipes/cluster_provision.rb

@@ -0,0 +1,4 @@
+
+include_recipe 'provision::common'
+include_recipe "provision::#{node['driver']}"
+include_recipe "provision::cluster_provision_instances"

data/lib/cookbooks/provision/recipes/cluster_provision_instances.rb

@@ -0,0 +1,55 @@
+cluster_config = cluster_config_for(node)
+org_name = cluster_config[:id]
+
+# Create/Provision Instances
+default_instance_options = cluster_config[:default_instance_options]
+cluster_config[:environments].each do |env, env_config|
+  chef_environment = fh_name_for(org_name, env)
+
+  instances = {}
+  env_config[:instances].each do |instance_name, instance_config|
+    instances[instance_name_for(org_name, env, instance_name)] = {}
+    instances[instance_name_for(org_name, env, instance_name)][:instance_config] = instance_config
+    instances[instance_name_for(org_name, env, instance_name)][:machine_options] = machine_options_for(cluster_config[:driver], org_name, env, default_instance_options.merge(instance_config[cluster_config[:driver].to_sym] || {}))
+  end
+
+  current_nodes = search(:node, "chef_environment:#{chef_environment}").map { |n| n.name }
+  required_nodes = instances.keys
+  missing_nodes = required_nodes - current_nodes
+
+  with_chef_environment chef_environment do
+
+    machine_batch do
+      missing_nodes.each do |instance_name|
+        machine_options = instances[instance_name][:machine_options]
+        machine instance_name do
+          chef_environment chef_environment
+          run_list ["recipe[feedhenry_common::hostname]"]
+          machine_options(machine_options)
+        end
+      end
+      not_if { missing_nodes.empty? }
+    end
+
+    if required_nodes.length > 1
+      converge_times = missing_nodes.empty? ? 1 : 3
+    else
+      converge_times = 1
+    end
+
+    converge_times.times do
+      machine_batch do
+        required_nodes.each do |instance_name|
+          machine_options = instances[instance_name][:machine_options]
+          instance_config = instances[instance_name][:instance_config]
+          machine instance_name do
+            chef_environment chef_environment
+            run_list instance_config[:run_list]
+            machine_options(machine_options)
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/cookbooks/provision/recipes/cluster_status.rb

@@ -0,0 +1,24 @@
+include_recipe 'provision::common'
+include_recipe "provision::#{node['driver']}"
+
+cluster_config = cluster_config_for(node)
+org_name = cluster_config[:id]
+
+cluster_config[:environments].each do |env, env_config|
+  chef_environment = fh_name_for(org_name, env)
+
+  nodes = search(:node, "chef_environment:#{chef_environment}")
+
+  with_chef_environment chef_environment do
+
+    nodes.each do |n|
+      machine_execute 'sudo fhctl status' do
+        live_stream true
+        machine n.name
+        only_if {n.recipes.include? 'feedhenry_common::fhctl'}
+      end
+    end
+
+  end
+
+end

data/lib/cookbooks/provision/recipes/common.rb

@@ -0,0 +1,9 @@
+
+raise "Missing cluster id" unless node['id']
+raise "Missing driver" unless node['driver']
+raise "Missing chef server config" unless node['chef_server_config']
+
+with_chef_server node['chef_server_config']['chef_server_url'],
+                 :client_name => node['chef_server_config']['node_name'],
+                 :signing_key_filename => node['chef_server_config']['client_key'],
+                 :ssl_verify_mode => :verify_none

data/lib/cookbooks/provision/recipes/default.rb

@@ -0,0 +1,5 @@
+#
+# Cookbook Name:: provision
+# Recipe:: default
+#
+# Copyright (c) 2015 The Authors, All Rights Reserved.

data/lib/cookbooks/provision/recipes/openstack.rb

@@ -0,0 +1,11 @@
+raise "Missing openstack credentials" unless node['provider_credentials']['os-username'] && node['provider_credentials']['os-password']
+
+require 'chef/provisioning/fog_driver/driver'
+
+os_auth_url = File.join(node['provider_credentials']['os-auth-url'], 'tokens')
+with_driver "fog:OpenStack:#{os_auth_url}", :compute_options => {
+    :openstack_username => node['provider_credentials']['os-username'],
+    :openstack_api_key => node['provider_credentials']['os-password'],
+    :openstack_auth_url => os_auth_url,
+    :openstack_tenant => node['provider_credentials']['os-tenant-name']
+}

data/lib/cookbooks/provision/recipes/openstack_cluster_create.rb

@@ -0,0 +1,11 @@
+
+include_recipe 'provision::common'
+include_recipe "provision::openstack"
+
+cluster_config = cluster_config_for(node)
+org_name = cluster_config[:id]
+
+key_pair_name = key_pair_name_for(org_name)
+fog_key_pair key_pair_name
+
+include_recipe "provision::cluster_provision_instances"

data/lib/cookbooks/provision/recipes/openstack_cluster_destroy.rb

@@ -0,0 +1,4 @@
+include_recipe 'provision::common'
+include_recipe "provision::openstack"
+
+include_recipe "provision::cluster_destroy_instances"

data/lib/cookbooks/provision/recipes/reset_rabbitmq.rb

@@ -0,0 +1,49 @@
+include_recipe 'provision::common'
+include_recipe "provision::#{node['driver']}"
+
+cluster_config = cluster_config_for(node)
+org_name = cluster_config[:id]
+
+default_instance_options = cluster_config[:default_instance_options]
+cluster_config[:environments].each do |env, env_config|
+  chef_environment = fh_name_for(org_name, env)
+
+  instances = {}
+  env_config[:instances].each do |instance_name, instance_config|
+    instances[instance_name_for(org_name, env, instance_name)] = {}
+    instances[instance_name_for(org_name, env, instance_name)][:instance_config] = instance_config
+    instances[instance_name_for(org_name, env, instance_name)][:machine_options] = machine_options_for(cluster_config[:driver], org_name, env, default_instance_options.merge(instance_config[cluster_config[:driver].to_sym] || {}))
+  end
+
+  rabbit_nodes = search(:node, "chef_environment:#{chef_environment} AND roles:rabbitmq_server").map { |n| n.name }
+
+  with_chef_environment chef_environment do
+
+    rabbit_nodes.each do |instance_name|
+      machine_execute 'sudo rabbitmqctl stop_app' do
+        machine instance_name
+      end
+
+      machine_execute 'sudo rabbitmqctl reset' do
+        machine instance_name
+      end
+
+      machine_execute 'sudo rabbitmqctl start_app' do
+        machine instance_name
+      end
+    end
+
+    machine_batch do
+      rabbit_nodes.each do |instance_name|
+        machine_options = instances[instance_name][:machine_options]
+        instance_config = instances[instance_name][:instance_config]
+        machine instance_name do
+          chef_environment chef_environment
+          run_list instance_config[:run_list]
+          machine_options(machine_options)
+        end
+      end
+    end
+
+  end
+end

data/lib/cookbooks/provision/recipes/restart_services.rb

@@ -0,0 +1,24 @@
+include_recipe 'provision::common'
+include_recipe "provision::#{node['driver']}"
+
+cluster_config = cluster_config_for(node)
+org_name = cluster_config[:id]
+
+cluster_config[:environments].each do |env, env_config|
+  chef_environment = fh_name_for(org_name, env)
+
+  nodes = search(:node, "chef_environment:#{chef_environment}")
+
+  with_chef_environment chef_environment do
+
+    nodes.each do |n|
+      machine_execute 'sudo fhctl restart' do
+        live_stream true
+        machine n.name
+        only_if {n.recipes.include? 'feedhenry_common::fhctl'}
+      end
+    end
+
+  end
+
+end

data/lib/extensions/chef/provisioning.rb

@@ -0,0 +1,21 @@
+class Chef
+  module Provisioning
+
+    def self.connect_to_machine(machine_spec, config = Cheffish.profiled_config, driver=nil)
+
+      chef_server = Cheffish.default_chef_server(config)
+      if machine_spec.is_a?(String)
+        machine_spec = chef_managed_entry_store(chef_server).get(:machine, machine_spec)
+      end
+      driver = driver_for_url(machine_spec.driver_url, config) unless driver
+      if driver
+        machine_options = {:convergence_options => {:chef_server => chef_server}}
+        machine_options = Cheffish::MergedConfig.new(config[:machine_options], machine_options) if config[:machine_options]
+        driver.connect_to_machine(machine_spec, machine_options)
+      else
+        nil
+      end
+    end
+
+  end
+end

data/lib/extensions/chef/provisioning/aws_driver/driver.rb

@@ -0,0 +1,46 @@
+class Chef
+  module Provisioning
+    module AWSDriver
+      class Driver < Chef::Provisioning::Driver
+
+        def initialize(driver_url, config)
+          super
+
+          _, profile_name, region = driver_url.split(':')
+          profile_name = nil if profile_name && profile_name.empty?
+          region = nil if region && region.empty?
+
+          credentials = profile_name ? aws_credentials[profile_name] : aws_credentials.default
+          @aws_config = AWS.config(
+              access_key_id:     credentials[:aws_access_key_id],
+              secret_access_key: credentials[:aws_secret_access_key],
+              region: region || credentials[:region],
+              proxy_uri: credentials[:proxy_uri] || nil,
+              session_token: credentials[:aws_session_token] || nil,
+              logger: Chef::Log.logger
+          )
+
+          # TODO document how users could add something to the Aws.config themselves if they want to
+          # Right now we are supporting both V1 and V2, so we create 2 config sets
+          credentials2 = ::Aws::Credentials.new(
+              credentials[:aws_access_key_id],
+              credentials[:aws_secret_access_key],
+              ENV["AWS_SESSION_TOKEN"]
+          )
+
+          Chef::Config.chef_provisioning ||= {}
+          ::Aws.config.update(
+              credentials: credentials2,
+              region: region || ENV["AWS_DEFAULT_REGION"] || credentials[:region],
+              # TODO when we get rid of V1 replace the credentials class with something that knows how
+              # to read ~/.aws/config
+              :http_proxy => credentials[:proxy_uri] || nil,
+              logger: Chef::Log.logger,
+              retry_limit: Chef::Config.chef_provisioning[:aws_retry_limit] || 5
+          )
+        end
+
+      end
+    end
+  end
+end