fhcap-cli 0.4.5 → 0.4.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2404936a843382f70df3f7f0c834b3ed5891ab4f
-  data.tar.gz: b3edd1294c2b6ea919a7d81642bf464adf29808a
+  metadata.gz: 78bf8ef12f1a57a0cea0fb7d43f449ebaa837d3e
+  data.tar.gz: 20bd845bfbed5c9adf26e8d4b5785737c5ed7a73
 SHA512:
-  metadata.gz: 78843714ea979e07b5dffc9a4dec0f75e3fc626480bab516f27879e2ce106ca6cfeef9867fa3862ddd9a43dd1546055967a9d1e01be585f4d1d49fe1e01cc2d8
-  data.tar.gz: 1d8872f549528b47f72ec8f465763b94beb8d907c7d9d43a6ab4d38606f306aa3a84db13284c729891546ec83f225f35405a0aa0cb255c504d6bdbac621cf6f9
+  metadata.gz: 114792fc44c7c251413a224373f8ad1d196742eabac60c3dd65fb221194f2afa2be27c0e39d70dce77a9c99c631212aacd3001d2c79947bc37908acbcd5b5ea5
+  data.tar.gz: 57d6da1fe18c0233a7cfeb02643c2c7a045b3fd3308f972fd013aaae72ee356d30f934c4a3b9e7aeedb1984c36019c53c6d6e99ea6a3c162a0ba2a1084af4f67

data/.gitignore

@@ -8,3 +8,4 @@
 /spec/reports/
 /tmp/
 /lib/nodes/
+.idea

data/CHANGELOG.md

@@ -1,4 +1,10 @@
 
+## 0.4.6
+
+* [RHMAP-3031] - Add aws-tags (Name, Organisationm, Environment and Role) to all instances
+* [RHMAP-3815] - Add dry-run option to all chef_provisions tasks, sets why-run option on chef run (chef-provisioning)
+* [RHMAP-4038] - Fix aws templates to set correct required security group protocols/ports for environment.
+
 ## 0.4.5
 
 * [RHMAP-3789] - Implement series option for cluster provision task to allow switching between parallel/series provisions of cluster nodes.

data/README.md

@@ -133,7 +133,7 @@ fhcap provider add --name os1internal --type openstack --credentials os-auth-url
 fhcap provider add --name fheng --type aws
 ```
 
-A prompt will be displayed for any credentaisl that are required but not passed to the command directly
+A prompt will be displayed for any credentials that are required but not passed to the command directly
 
 ## Usage
 

data/fhcap-cli.gemspec

@@ -25,6 +25,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'public_suffix', '= 1.5.2'
   spec.add_dependency 'erubis', '= 2.7.0'
   spec.add_dependency 'launchy', '= 2.4.3'
+  spec.add_dependency 'deep_merge', '= 1.0.1'
   spec.add_dependency 'librarian-chef', '= 0.0.4'
   spec.add_dependency 'knife-block', '= 0.2.1'
   spec.add_dependency 'knife-cookbook-readme', '= 0.2.1'
@@ -38,7 +39,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'chef-provisioning-vagrant', '= 0.10.0'
   spec.add_dependency 'chef-provisioning-fog', '= 0.15.0'
   spec.add_dependency 'test-kitchen', '= 1.4.2'
-  spec.add_dependency 'kitchen-sync', '= 1.1.1'
+  spec.add_dependency 'kitchen-sync', '= 2.1.1'
   spec.add_dependency 'kitchen-vagrant', '= 0.14.0'
   spec.add_dependency 'kitchen-ec2', '= 0.8.0'
   spec.add_dependency 'kitchen-docker', '= 2.1.0'

data/lib/cookbooks/provision/libraries/provision.rb

@@ -1,3 +1,5 @@
+require 'deep_merge'
+
 def node_names_for(cluster_config)
   org_name = cluster_config[:id]
   #If we are using chef-client in local mode (-z) we can't search :-(
@@ -25,8 +27,8 @@ def inbound_rules_for(org_name, ingress)
   end.flatten
 end
 
-def machine_options_for(provider, org_name, environment, instance_options)
-  send(:"#{provider}_machine_options", org_name, environment, instance_options)
+def machine_options_for(provider, org_name, environment, instance_name, instance_options)
+  send(:"#{provider}_machine_options", org_name, environment, instance_name, instance_options)
 end
 
 def bootstrap_options_for(provider, org_name, environment, instance_options)
@@ -39,10 +41,11 @@ end
 
 #AWS
 
-def aws_machine_options(org_name, environment, instance_options)
+def aws_machine_options(org_name, environment, instance_name, instance_options)
   {
       :bootstrap_options => bootstrap_options_for('aws', org_name, environment, instance_options),
       :convergence_options => converge_options_for('aws', org_name, environment, instance_options),
+      :aws_tags => {'Name' => instance_name, 'fh-tag-org' => org_name, 'fh-tag-environment' => [org_name,environment].join('-'), 'fh-tag-role' => instance_name.split('-').last.gsub(/\d+/, "")}
   }
 end
 
@@ -67,7 +70,7 @@ end
 
 #OpenStack
 
-def openstack_machine_options(org_name, environment, instance_options)
+def openstack_machine_options(org_name, environment, instance_name, instance_options)
   {
       :bootstrap_options => bootstrap_options_for('openstack', org_name, environment, instance_options),
       :convergence_options => converge_options_for('openstack', org_name, environment, instance_options),
@@ -144,19 +147,31 @@ def with_cluster_config(node, &block)
   block.call cluster_config
 end
 
-def with_cluster_instances(node, &block)
+def with_cluster_instances(node,  &block)
+  env_regexp = node[:'env-regexp'] || '.'
+  node_regexp = node[:'node-regexp'] || '.'
   with_cluster_config(node) do |cluster_config|
     org_name = cluster_config[:id]
     default_instance_options = cluster_config[:default_instance_options]
     instances = {}
     cluster_config[:environments].each do |env, env_config|
       chef_environment = fh_name_for(org_name, env)
-      instances[chef_environment] = {}
-
-      env_config[:instances].each do |instance_name, instance_config|
-        instances[chef_environment][instance_name_for(org_name, env, instance_name)] = {}
-        instances[chef_environment][instance_name_for(org_name, env, instance_name)][:instance_config] = instance_config
-        instances[chef_environment][instance_name_for(org_name, env, instance_name)][:machine_options] = machine_options_for(cluster_config[:driver], org_name, env, default_instance_options.merge(instance_config[cluster_config[:driver].to_sym] || {}))
+      if chef_environment =~ /#{env_regexp}/
+        instances[chef_environment] = {}
+
+        env_config[:instances].each do |instance_name, instance_config|
+          instance_name = instance_name_for(org_name, env, instance_name)
+          if instance_name =~ /#{node_regexp}/
+            instances[chef_environment][instance_name] = {}
+            instances[chef_environment][instance_name][:instance_config] = instance_config
+
+            machine_options = machine_options_for(cluster_config[:driver], org_name, env, instance_name, default_instance_options.merge(instance_config[cluster_config[:driver].to_sym] || {}))
+            machine_options.deep_merge!(JSON.parse(node[:machine_options].to_json, {:symbolize_names => true})) if node[:machine_options]
+            machine_options.deep_merge!(JSON.parse(instance_config[:machine_options].to_json, {:symbolize_names => true})) if instance_config[:machine_options]
+
+            instances[chef_environment][instance_name][:machine_options] = machine_options
+          end
+        end
       end
     end
     block.call instances

data/lib/cookbooks/provision/recipes/aws_cluster_create.rb

@@ -21,6 +21,13 @@ aws_vpc vpc_name do
   main_routes '0.0.0.0/0' => :internet_gateway
 end
 
+# Create Security Groups with empty rules first to avoid group dependent rule conflicts
+cluster_config[:security_groups].each do |sg_name, sg_config|
+  aws_security_group security_group_name_for(org_name, sg_name) do
+    vpc vpc_name
+  end
+end
+
 # Create Security Groups
 cluster_config[:security_groups].each do |sg_name, sg_config|
   aws_security_group security_group_name_for(org_name, sg_name) do

data/lib/cookbooks/provision/recipes/aws_cluster_create_elb.rb

@@ -23,16 +23,21 @@ cluster_config[:environments].each do |env, env_config|
         certificate_file = File.join(node[:local_repo_path], node[:local_repo_clusters_dir], 'ssl_certs', "#{listener[:server_certificate]}-certificate.pem")
         private_key_file = File.join(node[:local_repo_path], node[:local_repo_clusters_dir], 'ssl_certs', "#{listener[:server_certificate]}-private-key.pem")
 
+        cert_name = listener[:server_certificate]
+
         if File.exists?(certificate_file) && File.exists?(private_key_file)
           certificate_body = File.open(certificate_file, "rb").read
           private_key = File.open(private_key_file, "rb").read
 
-          cert_name = listener[:server_certificate]
           lb_certs[index] = aws_server_certificate cert_name do
             certificate_body certificate_body
             private_key private_key
           end
         else
+          cert_name = listener[:server_certificate]
+          lb_certs[index] = aws_server_certificate cert_name do
+            action :nothing
+          end
           Chef::Log.warn("Could not find #{listener[:server_certificate]} locally to upload. Is this certificate already registered? ELB create will fail if it does not exist!!")
         end
       end

data/lib/cookbooks/provision/recipes/aws_cluster_destroy.rb

@@ -5,27 +5,38 @@ cluster_config = cluster_config_for(node)
 
 org_name = cluster_config[:id]
 
-node_names = node_names_for(cluster_config)
+include_recipe "provision::cluster_destroy_instances"
 
-machine_batch do
-  machines node_names
-  action :destroy
+cluster_config[:environments].each do |env, env_config|
+  env_config[:load_balancers].each do |lb_name, lb_config|
+    lb_name = load_balancer_name_for(org_name, env, lb_name)
+    load_balancer lb_name do
+      action :destroy
+    end
+  end if env_config[:load_balancers]
 end
 
-# Clear all rules from Security Groups
+# Clear all rules from Security Groups so that all dependent objects are cleared. This can potentially re-create security groups, so we have to force the destroy again below.
 cluster_config[:security_groups].each do |sg_name, sg_config|
   aws_security_group security_group_name_for(org_name, sg_name) do
     inbound_rules []
   end
 end
 
+# Destroy all security groups. vpc purge would do this, but since we could have potentially re-created the group above, we do it here to be sure its removed.
+cluster_config[:security_groups].each do |sg_name, sg_config|
+  aws_security_group security_group_name_for(org_name, sg_name) do
+    action :destroy
+    ignore_failure true
+  end
+end
+
 vpc_name = vpc_name_for(org_name)
 aws_vpc vpc_name do
   action :purge
 end
 
 cluster_config[:environments].each do |env, env_config|
-
   aws_route_table route_table_name_for(org_name, env) do
     action :destroy
   end
@@ -35,16 +46,6 @@ cluster_config[:environments].each do |env, env_config|
       action :destroy
     end
   end
-
-end
-
-cluster_config[:environments].each do |env, env_config|
-  env_config[:load_balancers].each do |lb_name, lb_config|
-    lb_name = load_balancer_name_for(org_name, env, lb_name)
-    load_balancer lb_name do
-      action :destroy
-    end
-  end if env_config[:load_balancers]
 end
 
 aws_key_pair key_pair_name_for(org_name) do

data/lib/cookbooks/provision/recipes/cluster_bootstrap.rb

@@ -0,0 +1,4 @@
+
+include_recipe 'provision::common'
+include_recipe "provision::#{node['driver']}"
+include_recipe "provision::cluster_bootstrap_instances"

data/lib/cookbooks/provision/recipes/cluster_bootstrap_instances.rb

@@ -0,0 +1,37 @@
+include_recipe 'provision::common'
+include_recipe "provision::#{node['driver']}"
+
+with_cluster_instances(node) do |cluster_instances|
+  machine_batch "cluster_bootstrap_instances-converge_missing_nodes" do
+    cluster_instances.each do |chef_environment, instances|
+      current_nodes = search(:node, "chef_environment:#{chef_environment} AND network:*").map { |n| n.name }
+      required_nodes = instances.keys
+      missing_nodes = required_nodes - current_nodes
+
+      missing_nodes.each do |instance_name|
+        machine_options = instances[instance_name][:machine_options]
+        machine instance_name do
+          chef_environment chef_environment
+          machine_options machine_options
+          run_list []
+        end
+      end
+    end
+    max_simultaneous node['max_simultaneous']
+    action :converge
+  end
+
+  machine_batch "cluster_bootstrap_instances-setup_nodes" do
+    cluster_instances.each do |chef_environment, instances|
+      instances.each do |name, cfg|
+        machine name do
+          chef_environment chef_environment
+          machine_options cfg[:machine_options]
+          run_list cfg[:instance_config][:run_list]
+        end
+      end
+    end
+    max_simultaneous node['max_simultaneous']
+    action :setup
+  end
+end

data/lib/cookbooks/provision/recipes/cluster_create_instances.rb

@@ -1,15 +1,17 @@
+include_recipe "provision::common"
+include_recipe "provision::#{node['driver']}"
+
 with_cluster_instances(node) do |cluster_instances|
-  cluster_instances.each do |chef_environment, instances|
-    with_chef_environment chef_environment do
-      machine_batch 'cluster_create_instances' do
-        instances.each do |name, cfg|
-          machine name do
-            chef_environment chef_environment
-            machine_options(cfg[:machine_options])
-            run_list []
-          end
+  machine_batch "cluster_create_instances" do
+    cluster_instances.each do |chef_environment, instances|
+      instances.each do |name, cfg|
+        machine name do
+          chef_environment chef_environment
+          machine_options cfg[:machine_options]
         end
       end
     end
+    max_simultaneous node['max_simultaneous']
+    action :ready
   end
 end

data/lib/cookbooks/provision/recipes/cluster_provision_instances.rb

@@ -1,17 +1,18 @@
+include_recipe 'provision::common'
+include_recipe "provision::#{node['driver']}"
+
 with_cluster_instances(node) do |cluster_instances|
-  cluster_instances.each do |chef_environment, instances|
-    with_chef_environment chef_environment do
-      machine_batch 'cluster_provision_instances' do
-        max_simultaneous node['max_simultaneous']
-        instances.each do |name, cfg|
-          machine name do
-            chef_environment chef_environment
-            run_list cfg[:instance_config][:run_list]
-            machine_options(cfg[:machine_options])
-            converge true
-          end
+  machine_batch "cluster_provision_instances" do
+    cluster_instances.each do |chef_environment, instances|
+      instances.each do |name, cfg|
+        machine name do
+          chef_environment chef_environment
+          machine_options cfg[:machine_options]
+          run_list cfg[:instance_config][:run_list]
         end
       end
     end
+    max_simultaneous node['max_simultaneous']
+    action :converge_only
   end
 end

data/lib/cookbooks/provision/recipes/galera_reset_cluster.rb

@@ -7,53 +7,57 @@ with_cluster_instances(node) do |cluster_instances|
   cluster_instances.each do |chef_environment, instances|
     with_chef_environment chef_environment do
 
-      search_role = 'galera.*_server'
+      # We have to deal with arbiters after the servers or things go all wrong :-/
+      %w{galera_server galera_arbiter_server}.each do |role|
 
-      search_nodes = search(:node, "chef_environment:#{chef_environment} AND roles:#{search_role}").map { |n| n.name }
+        search_nodes = search(:node, "chef_environment:#{chef_environment} AND roles:#{role}").map { |n| n.name }
 
-      run_list_query = /role\[#{search_role}\]/
+        run_list_query = /role\[#{role}\]/
 
-      galera_instances = instances.select do |name, cfg|
-        if cfg[:instance_config] && cfg[:instance_config][:run_list] && cfg[:instance_config][:run_list].index { |s| s =~ run_list_query }
-          tmp_run_list = []
-          tmp_run_list << cfg[:instance_config][:run_list][cfg[:instance_config][:run_list].index { |s| s =~ run_list_query }]
-          cfg[:instance_config][:tmp_run_list] = tmp_run_list
-          true
-        else
-          false
+        galera_instances = instances.select do |name, cfg|
+          if cfg[:instance_config] && cfg[:instance_config][:run_list] && cfg[:instance_config][:run_list].index { |s| s =~ run_list_query }
+            node.default['tmp_node_run_list'] = {} unless node['tmp_node_run_list']
+            tmp_run_list = node['tmp_node_run_list'][name] ? node['tmp_node_run_list'][name].dup : []
+            tmp_run_list << cfg[:instance_config][:run_list][cfg[:instance_config][:run_list].index { |s| s =~ run_list_query }]
+            cfg[:instance_config][:tmp_run_list] = tmp_run_list
+            node.default['tmp_node_run_list'][name] = tmp_run_list
+            true
+          else
+            false
+          end
         end
-      end
 
-      #Delete the installed flag to force a re-install of galera, Note DO NOT run this in a prod setup that is being used for real.
-      galera_instances.each do |name, cfg|
-        machine_execute "galera_reset_cluster - #{name}: Remove /root/.galera_installed" do
-          command 'sudo rm -rf /root/.galera_installed'
-          machine name
+        #Delete the installed flag to force a re-install of galera, Note DO NOT run this in a prod setup that is being used for real.
+        galera_instances.each do |name, cfg|
+          machine_execute "galera_reset_cluster - #{name}: Remove /root/.galera_installed" do
+            command 'sudo rm -rf /root/.galera_installed'
+            machine name
+          end
         end
-      end
 
-      # Chef run to force the install recipe for galera to run, depending on weather nodes exist on the server already or not, we will need to do this once or twice.
-      batch_provisions = search_nodes.length > 1 ? 1 : 2
-      batch_provisions.times do |index|
-        machine_batch "galera_reset_cluster: Batch Provison #{index + 1}/#{batch_provisions}" do
-          galera_instances.each do |name, cfg|
-            machine name do
-              chef_environment chef_environment
-              machine_options(cfg[:machine_options])
-              run_list cfg[:instance_config][:tmp_run_list]
-              converge true
+        # Chef run to force the install recipe for galera to run, depending on weather nodes exist on the server already or not, we will need to do this once or twice.
+        batch_provisions = search_nodes.length > 1 ? 1 : 2
+        batch_provisions.times do |index|
+          machine_batch "galera_reset_cluster: Batch Provison #{index + 1}/#{batch_provisions}" do
+            galera_instances.each do |name, cfg|
+              machine name do
+                chef_environment chef_environment
+                machine_options(cfg[:machine_options])
+                run_list cfg[:instance_config][:tmp_run_list]
+                converge true
+              end
             end
           end
         end
-      end
 
-      # Chef run on each node in series to setup clustering
-      galera_instances.each do |name, cfg|
-        machine name do
-          chef_environment chef_environment
-          machine_options(cfg[:machine_options])
-          run_list cfg[:instance_config][:tmp_run_list]
-          converge true
+        # Chef run on each node in series to setup clustering
+        galera_instances.each do |name, cfg|
+          machine name do
+            chef_environment chef_environment
+            machine_options(cfg[:machine_options])
+            run_list cfg[:instance_config][:tmp_run_list]
+            converge true
+          end
         end
       end
 

data/lib/cookbooks/provision/recipes/mongo_reset_cluster.rb

@@ -5,27 +5,28 @@ with_cluster_instances(node) do |cluster_instances|
   cluster_instances.each do |chef_environment, instances|
     with_chef_environment chef_environment do
 
-      search_role = '.*mongo.*_server'
+      # It always take 3 provisions for mongo to be working in a clustered setup.
+      #
+      # 1. install mongo and update chef server
+      # 2. configure mongo configs with all the other nodes and setup replicaset
+      # 3. Add any users to what should be a now working mongo cluster
 
+      search_role = '.*mongo.*_server'
       run_list_query = /role\[#{search_role}\]/
 
       mongo_instances = instances.select do |name, cfg|
         if cfg[:instance_config] && cfg[:instance_config][:run_list] && cfg[:instance_config][:run_list].index { |s| s =~ run_list_query }
-          tmp_run_list = []
+          node.default['tmp_node_run_list'] = {} unless node['tmp_node_run_list']
+          tmp_run_list = node['tmp_node_run_list'][name] ? node['tmp_node_run_list'][name].dup : []
           tmp_run_list << cfg[:instance_config][:run_list][cfg[:instance_config][:run_list].index { |s| s =~ run_list_query }]
           cfg[:instance_config][:tmp_run_list] = tmp_run_list
+          node.default['tmp_node_run_list'][name] = tmp_run_list
           true
         else
           false
         end
       end
 
-      # It always take 3 provisions for mongo to be working in a clustered setup.
-      #
-      # 1. install mongo and update chef server
-      # 2. configure mongo configs with all the other nodes and setup replicaset
-      # 3. Add any users to what should be a now working mongo cluster
-
       batch_provisions = 3
       batch_provisions.times do |index|
         machine_batch "mongo_reset_cluster: Batch Provison #{index + 1}/#{batch_provisions}" do
@@ -34,9 +35,9 @@ with_cluster_instances(node) do |cluster_instances|
               chef_environment chef_environment
               machine_options(cfg[:machine_options])
               run_list cfg[:instance_config][:tmp_run_list]
-              converge true
             end
           end
+          action :converge
         end
       end
 

data/lib/cookbooks/provision/recipes/rabbitmq_reset_cluster.rb

@@ -13,9 +13,11 @@ with_cluster_instances(node) do |cluster_instances|
 
       rabbitmq_instances = instances.select do |name, cfg|
         if cfg[:instance_config] && cfg[:instance_config][:run_list] && cfg[:instance_config][:run_list].index { |s| s =~ run_list_query }
-          tmp_run_list = []
+          node.default['tmp_node_run_list'] = {} unless node['tmp_node_run_list']
+          tmp_run_list = node['tmp_node_run_list'][name] ? node['tmp_node_run_list'][name].dup : []
           tmp_run_list << cfg[:instance_config][:run_list][cfg[:instance_config][:run_list].index { |s| s =~ run_list_query }]
           cfg[:instance_config][:tmp_run_list] = tmp_run_list
+          node.default['tmp_node_run_list'][name] = tmp_run_list
           true
         else
           false

data/lib/extensions/chef/solr/query/regexpable_query.rb

@@ -0,0 +1,17 @@
+#Fixes search issue https://github.com/chef/chef-zero/pull/158
+module ChefZero
+  module Solr
+    module Query
+      class RegexpableQuery
+
+        original_verbose, $VERBOSE = $VERBOSE, nil
+
+        WORD_CHARACTER = "[A-Za-z0-9@._':\-]"
+        NON_WORD_CHARACTER = "[^A-Za-z0-9@._':\-]"
+
+        $VERBOSE = original_verbose
+
+      end
+    end
+  end
+end

data/lib/fhcap/chef-dk/chef_runner.rb

@@ -18,14 +18,16 @@ module Fhcap
     attr_reader :run_list
     attr_reader :private_key_paths
     attr_reader :node_attrs
+    attr_reader :dry_run
 
-    def initialize(cookbook_path, run_list, node_attrs={}, private_key_paths=[])
+    def initialize(cookbook_path, run_list, dry_run=false, node_attrs={}, private_key_paths=[])
       @cookbook_path = File.expand_path(cookbook_path)
       @run_list = run_list
       @private_key_paths = private_key_paths
       @node_attrs = node_attrs
       @formatter = nil
       @ohai = nil
+      @dry_run = dry_run
     end
 
     def converge
@@ -70,6 +72,7 @@ module Fhcap
       Chef::Config.private_key_paths = (Chef::Config.private_key_paths + private_key_paths).compact.uniq
       Chef::Config.color = true
       Chef::Config.diff_disabled = true
+      Chef::Config.why_run = dry_run
 
       # atomic file operations on Windows require Administrator privileges to be able to read the SACL from a file
       # Using file_staging_uses_destdir(true) will get us inherited permissions indirectly on tempfile creation